Merge branch 'HADOOP-17377' of https://github.com/anmolanmol1234/hadoop…

… into HADOOP-17377

.gitignore

@@ -1,4 +1,4 @@
-.DS_Store
+*.DS_Store
 *.iml
 *.ipr
 *.iws

BUILDING.txt

@@ -163,13 +163,13 @@ Maven build goals:
  YARN Application Timeline Service V2 build options:
 
    YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
-   versions of Apache HBase are 1.2.6 (default) and 2.0.0-beta1.
+   versions of Apache HBase are 1.7.1 (default) and 2.2.4.
 
-  * HBase 1.2.6 is used by default to build Hadoop. The official releases are ready to use if you
-    plan on running Timeline Service v2 with HBase 1.2.6.
+  * HBase 1.7.1 is used by default to build Hadoop. The official releases are ready to use if you
+    plan on running Timeline Service v2 with HBase 1.7.1.
 
-  * Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.0.0-beta1. Provide this option if you plan
-    on running Timeline Service v2 with HBase 2.0.
+  * Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.2.4. Provide this option if you plan
+    on running Timeline Service v2 with HBase 2.x.
 
 
  Snappy build options:
@@ -311,6 +311,36 @@ Maven build goals:
     package. This option requires that -Dpmdk.lib is specified. With -Dbundle.pmdk provided,
     the build will fail if -Dpmdk.lib is not specified.
 
+Controlling the redistribution of the protobuf-2.5 dependency
+
+    The protobuf 2.5.0 library is used at compile time to compile the class
+    org.apache.hadoop.ipc.ProtobufHelper; this class known to have been used by
+    external projects in the past. Protobuf 2.5 is not used elsewhere in
+    the Hadoop codebase; alongside the move to Protobuf 3.x a private successor
+    class, org.apache.hadoop.ipc.internal.ShadedProtobufHelper is now used.
+
+    The hadoop-common module no longer exports its compile-time dependency on
+    protobuf-2.5. Hadoop distributions no longer include it.
+    Any application declaring a dependency on hadoop-commmon will no longer get
+    the artifact added to their classpath.
+    If is still required, then they must explicitly declare it:
+
+      <dependency>
+        <groupId>com.google.protobuf</groupId>
+        <artifactId>protobuf-java</artifactId>
+        <version>2.5.0</version>
+      </dependency>
+
+    In Hadoop builds the scope of the dependency can be set with the
+    option "common.protobuf2.scope".
+    This can be upgraded from "provided" to "compile" on the maven command line:
+
+           -Dcommon.protobuf2.scope=compile
+
+    If this is done then protobuf-2.5.0.jar will again be exported as a
+    hadoop-common dependency, and included in the share/hadoop/common/lib/
+    directory of any Hadoop distribution built.
+
 ----------------------------------------------------------------------------------
 Building components separately
 

LICENSE-binary

@@ -241,15 +241,13 @@ com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.31
-com.squareup.okhttp3:okhttp:4.10.0
-com.squareup.okio:okio:3.4.0
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.4
 commons-cli:commons-cli:1.5.0
 commons-codec:commons-codec:1.11
 commons-collections:commons-collections:3.2.2
 commons-daemon:commons-daemon:1.0.13
-commons-io:commons-io:2.8.0
+commons-io:commons-io:2.14.0
 commons-net:commons-net:3.9.0
 de.ruedigermoeller:fst:2.50
 io.grpc:grpc-api:1.26.0
@@ -259,36 +257,36 @@ io.grpc:grpc-netty:1.26.0
 io.grpc:grpc-protobuf:1.26.0
 io.grpc:grpc-protobuf-lite:1.26.0
 io.grpc:grpc-stub:1.26.0
-io.netty:netty-all:4.1.94.Final
-io.netty:netty-buffer:4.1.94.Final
-io.netty:netty-codec:4.1.94.Final
-io.netty:netty-codec-dns:4.1.94.Final
-io.netty:netty-codec-haproxy:4.1.94.Final
-io.netty:netty-codec-http:4.1.94.Final
-io.netty:netty-codec-http2:4.1.94.Final
-io.netty:netty-codec-memcache:4.1.94.Final
-io.netty:netty-codec-mqtt:4.1.94.Final
-io.netty:netty-codec-redis:4.1.94.Final
-io.netty:netty-codec-smtp:4.1.94.Final
-io.netty:netty-codec-socks:4.1.94.Final
-io.netty:netty-codec-stomp:4.1.94.Final
-io.netty:netty-codec-xml:4.1.94.Final
-io.netty:netty-common:4.1.94.Final
-io.netty:netty-handler:4.1.94.Final
-io.netty:netty-handler-proxy:4.1.94.Final
-io.netty:netty-resolver:4.1.94.Final
-io.netty:netty-resolver-dns:4.1.94.Final
-io.netty:netty-transport:4.1.94.Final
-io.netty:netty-transport-rxtx:4.1.94.Final
-io.netty:netty-transport-sctp:4.1.94.Final
-io.netty:netty-transport-udt:4.1.94.Final
-io.netty:netty-transport-classes-epoll:4.1.94.Final
-io.netty:netty-transport-native-unix-common:4.1.94.Final
-io.netty:netty-transport-classes-kqueue:4.1.94.Final
-io.netty:netty-resolver-dns-classes-macos:4.1.94.Final
-io.netty:netty-transport-native-epoll:4.1.94.Final
-io.netty:netty-transport-native-kqueue:4.1.94.Final
-io.netty:netty-resolver-dns-native-macos:4.1.94.Final
+io.netty:netty-all:4.1.100.Final
+io.netty:netty-buffer:4.1.100.Final
+io.netty:netty-codec:4.1.100.Final
+io.netty:netty-codec-dns:4.1.100.Final
+io.netty:netty-codec-haproxy:4.1.100.Final
+io.netty:netty-codec-http:4.1.100.Final
+io.netty:netty-codec-http2:4.1.100.Final
+io.netty:netty-codec-memcache:4.1.100.Final
+io.netty:netty-codec-mqtt:4.1.100.Final
+io.netty:netty-codec-redis:4.1.100.Final
+io.netty:netty-codec-smtp:4.1.100.Final
+io.netty:netty-codec-socks:4.1.100.Final
+io.netty:netty-codec-stomp:4.1.100.Final
+io.netty:netty-codec-xml:4.1.100.Final
+io.netty:netty-common:4.1.100.Final
+io.netty:netty-handler:4.1.100.Final
+io.netty:netty-handler-proxy:4.1.100.Final
+io.netty:netty-resolver:4.1.100.Final
+io.netty:netty-resolver-dns:4.1.100.Final
+io.netty:netty-transport:4.1.100.Final
+io.netty:netty-transport-rxtx:4.1.100.Final
+io.netty:netty-transport-sctp:4.1.100.Final
+io.netty:netty-transport-udt:4.1.100.Final
+io.netty:netty-transport-classes-epoll:4.1.100.Final
+io.netty:netty-transport-native-unix-common:4.1.100.Final
+io.netty:netty-transport-classes-kqueue:4.1.100.Final
+io.netty:netty-resolver-dns-classes-macos:4.1.100.Final
+io.netty:netty-transport-native-epoll:4.1.100.Final
+io.netty:netty-transport-native-kqueue:4.1.100.Final
+io.netty:netty-resolver-dns-native-macos:4.1.100.Final
 io.opencensus:opencensus-api:0.12.3
 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
 io.reactivex:rxjava:1.3.8
@@ -301,7 +299,7 @@ net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 org.apache.avro:avro:1.9.2
 org.apache.commons:commons-collections4:4.2
-org.apache.commons:commons-compress:1.21
+org.apache.commons:commons-compress:1.24.0
 org.apache.commons:commons-configuration2:2.8.0
 org.apache.commons:commons-csv:1.9.0
 org.apache.commons:commons-digester:1.8.1
@@ -319,8 +317,8 @@ org.apache.hbase:hbase-common:1.7.1
 org.apache.hbase:hbase-protocol:1.7.1
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
-org.apache.httpcomponents:httpclient:4.5.6
-org.apache.httpcomponents:httpcore:4.4.10
+org.apache.httpcomponents:httpclient:4.5.13
+org.apache.httpcomponents:httpcore:4.4.13
 org.apache.kafka:kafka-clients:2.8.2
 org.apache.kerby:kerb-admin:2.0.3
 org.apache.kerby:kerb-client:2.0.3
@@ -339,31 +337,30 @@ org.apache.kerby:kerby-xdr:2.0.3
 org.apache.kerby:token-provider:2.0.3
 org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
-org.apache.zookeeper:zookeeper:3.6.3
+org.apache.zookeeper:zookeeper:3.7.2
 org.codehaus.jettison:jettison:1.5.4
-org.eclipse.jetty:jetty-annotations:9.4.51.v20230217
-org.eclipse.jetty:jetty-http:9.4.51.v20230217
-org.eclipse.jetty:jetty-io:9.4.51.v20230217
-org.eclipse.jetty:jetty-jndi:9.4.51.v20230217
-org.eclipse.jetty:jetty-plus:9.4.51.v20230217
-org.eclipse.jetty:jetty-security:9.4.51.v20230217
-org.eclipse.jetty:jetty-server:9.4.51.v20230217
-org.eclipse.jetty:jetty-servlet:9.4.51.v20230217
-org.eclipse.jetty:jetty-util:9.4.51.v20230217
-org.eclipse.jetty:jetty-util-ajax:9.4.51.v20230217
-org.eclipse.jetty:jetty-webapp:9.4.51.v20230217
-org.eclipse.jetty:jetty-xml:9.4.51.v20230217
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217
+org.eclipse.jetty:jetty-annotations:9.4.53.v20231009
+org.eclipse.jetty:jetty-http:9.4.53.v20231009
+org.eclipse.jetty:jetty-io:9.4.53.v20231009
+org.eclipse.jetty:jetty-jndi:9.4.53.v20231009
+org.eclipse.jetty:jetty-plus:9.4.53.v20231009
+org.eclipse.jetty:jetty-security:9.4.53.v20231009
+org.eclipse.jetty:jetty-server:9.4.53.v20231009
+org.eclipse.jetty:jetty-servlet:9.4.53.v20231009
+org.eclipse.jetty:jetty-util:9.4.53.v20231009
+org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009
+org.eclipse.jetty:jetty-webapp:9.4.53.v20231009
+org.eclipse.jetty:jetty-xml:9.4.53.v20231009
+org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.53.v20231009
+org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.53.v20231009
 org.ehcache:ehcache:3.3.1
 org.ini4j:ini4j:0.5.4
-org.jetbrains.kotlin:kotlin-stdlib:1.4.10
-org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
-org.xerial.snappy:snappy-java:1.1.10.1
+org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
+software.amazon.awssdk:bundle:jar:2.20.160
 
 
 --------------------------------------------------------------------------------
@@ -395,7 +392,6 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
-com.google.protobuf:protobuf-java:2.5.0
 com.google.protobuf:protobuf-java:3.6.1
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.54

NOTICE-binary

@@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
------------------------------------------------------------------------
-
-This product contains a modified portion of 'OkHttp', an open source
-HTTP & SPDY client for Android and Java applications, which can be obtained
-at:
-
-  * LICENSE:
-    * okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/square/okhttp
-  * LOCATION_IN_GRPC:
-    * okhttp/third_party/okhttp
-
 This product contains a modified portion of 'Netty', an open source
 networking library, which can be obtained at:
 

dev-support/bin/hadoop.sh

@@ -21,18 +21,20 @@
 personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
 
 # These flags are needed to run Yetus against Hadoop on Windows.
-WINDOWS_FLAGS="-Pnative-win
-  -Dhttps.protocols=TLSv1.2
-  -Drequire.openssl
-  -Drequire.test.libhadoop
-  -Dshell-executable=${BASH_EXECUTABLE}
-  -Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}
-  -Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}
-  -Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}
-  -Dwindows.cmake.build.type=RelWithDebInfo
-  -Dwindows.build.hdfspp.dll=off
-  -Dwindows.no.sasl=on
-  -Duse.platformToolsetVersion=v142"
+WINDOWS_FLAGS=(
+  "-Pnative-win"
+  "-Dhttps.protocols=TLSv1.2"
+  "-Drequire.openssl"
+  "-Drequire.test.libhadoop"
+  "-Dshell-executable=${BASH_EXECUTABLE}"
+  "-Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}"
+  "-Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}"
+  "-Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}"
+  "-Dwindows.cmake.build.type=RelWithDebInfo"
+  "-Dwindows.build.hdfspp.dll=off"
+  "-Dwindows.no.sasl=on"
+  "-Duse.platformToolsetVersion=v142"
+)
 
 ## @description  Globals specific to this personality
 ## @audience     private
@@ -292,7 +294,7 @@ function hadoop_native_flags
         -Drequire.snappy \
         -Pdist \
         -Dtar \
-        "${WINDOWS_FLAGS}"
+        "${WINDOWS_FLAGS[@]}"
     ;;
     *)
       echo \
@@ -436,7 +438,7 @@ function personality_modules
   fi
 
   if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
-    extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}"
+    extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS[*]} ${extra}"
   fi
 
   for module in $(hadoop_order ${ordering}); do
@@ -557,14 +559,6 @@ function shadedclient_rebuild
   declare module
   declare -a modules=()
 
-  if [[ ${OSTYPE} = Windows_NT ||
-        ${OSTYPE} =~ ^CYGWIN.* ||
-        ${OSTYPE} =~ ^MINGW32.* ||
-        ${OSTYPE} =~ ^MSYS.* ]]; then
-    echo "hadoop personality: building on windows, skipping check of client artifacts."
-    return 0
-  fi
-
   yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
   for module in hadoop-client-modules/hadoop-client-check-invariants \
                 hadoop-client-modules/hadoop-client-check-test-invariants \
@@ -581,28 +575,47 @@ function shadedclient_rebuild
 
   big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
 
-  extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true"
+  extra=(
+    "-Dtest=NoUnitTests"
+    "-Dmaven.javadoc.skip=true"
+    "-Dcheckstyle.skip=true"
+    "-Dspotbugs.skip=true"
+  )
 
   if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
+    # shellcheck disable=SC2206
+    extra+=(${WINDOWS_FLAGS[*]})
+
+    # The shaded client integration tests require the Hadoop jars that were just built to be
+    # installed in the local maven repository.
+    # shellcheck disable=SC2086
+    echo_and_redirect "${logfile}" \
+      "${MAVEN}" "${MAVEN_ARGS[@]}" install -fae --batch-mode \
+        -DskipTests -DskipDocs -Pdist -Dtar ${extra[*]}
+
+    # The shaded client integration tests spawn a MiniDFS and MiniYARN cluster for testing. Both of
+    # them require winutils.exe to be found in the PATH and HADOOP_HOME to be set.
     if load_hadoop_version; then
       export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT"
+      WIN_HADOOP_HOME=$(cygpath -w -a "${HADOOP_HOME}")
+      export PATH="${PATH};${WIN_HADOOP_HOME}\bin"
     else
       yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail."
     fi
-
-    extra="${WINDOWS_FLAGS} ${extra}"
   fi
 
+  # shellcheck disable=SC2086
   echo_and_redirect "${logfile}" \
-    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}"
+    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" ${extra[*]}
 
   big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
 
+  # shellcheck disable=SC2086
   echo_and_redirect "${logfile}" \
     "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
       "${modules[@]}" \
       -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \
-      -Dspotbugs.skip=true "${extra}"
+      -Dspotbugs.skip=true ${extra[*]}
 
   count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
   if [[ ${count} -gt 0 ]]; then