Skip to content

Commit 56635ba

Browse files
sumangala17sumangala17
committed
Merge branch 'trunk' into HADOOP-17596 
2 parents 6611791 + b944084 commit 56635ba

File tree

511 files changed

+14344
-4220
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

511 files changed

+14344
-4220
lines changed

‎BUILDING.txt‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,13 @@ Known issues:
5151
and run your IDE and Docker etc inside that VM.
5252

5353
----------------------------------------------------------------------------------
54-
Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
54+
Installing required packages for clean install of Ubuntu 18.04 LTS Desktop.
55+
(For Ubuntu 20.04, gcc/g++ and cmake bundled with Ubuntu can be used.
56+
Refer to dev-support/docker/Dockerfile):
5557

56-
* Oracle JDK 1.8 (preferred)
57-
$ sudo apt-get purge openjdk*
58-
$ sudo apt-get install software-properties-common
59-
$ sudo add-apt-repository ppa:webupd8team/java
58+
* Open JDK 1.8
6059
$ sudo apt-get update
61-
$ sudo apt-get install oracle-java8-installer
60+
$ sudo apt-get -y install java-8-openjdk
6261
* Maven
6362
$ sudo apt-get -y install maven
6463
* Native libraries
@@ -86,12 +85,13 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
8685
$ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2
8786
$ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0
8887
$ ./bootstrap.sh --prefix=/usr/
89-
$ ./b2 --without-python install
88+
$ ./b2 --without-python
89+
$ sudo ./b2 --without-python install
9090

9191
Optional packages:
9292

9393
* Snappy compression (only used for hadoop-mapreduce-client-nativetask)
94-
$ sudo apt-get install snappy libsnappy-dev
94+
$ sudo apt-get install libsnappy-dev
9595
* Intel ISA-L library for erasure coding
9696
Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
9797
(OR https://github.com/01org/isa-l)

‎LICENSE-binary‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ com.google.guava:guava:20.0
240240
com.google.guava:guava:27.0-jre
241241
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
242242
com.microsoft.azure:azure-storage:7.0.0
243-
com.nimbusds:nimbus-jose-jwt:4.41.1
243+
com.nimbusds:nimbus-jose-jwt:9.8.1
244244
com.squareup.okhttp:okhttp:2.7.5
245245
com.squareup.okio:okio:1.6.0
246246
com.zaxxer:HikariCP-java7:2.4.12
@@ -283,7 +283,7 @@ javax.inject:javax.inject:1
283283
log4j:log4j:1.2.17
284284
net.java.dev.jna:jna:5.2.0
285285
net.minidev:accessors-smart:1.2
286-
net.minidev:json-smart:2.3
286+
net.minidev:json-smart:2.4.2
287287
org.apache.avro:avro:1.7.7
288288
org.apache.commons:commons-collections4:4.2
289289
org.apache.commons:commons-compress:1.19

‎dev-support/Jenkinsfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ pipeline {
3535
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
3636
YETUS='yetus'
3737
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
38-
YETUS_VERSION='11eb9b09786e401fbdeaa3be83a19a4066fd7813'
38+
YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
3939
}
4040

4141
parameters {

‎dev-support/docker/Dockerfile_centos_7‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Dockerfile for installing the necessary dependencies for building Hadoop.
18+
# See BUILDING.txt.
19+
20+
FROM centos:7
21+
22+
WORKDIR /root
23+
24+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
25+
26+
RUN yum update -y \
27+
&& yum install -y centos-release-scl \
28+
&& yum install -y devtoolset-9 \
29+
&& yum install -y \
30+
ant \
31+
build-essential \
32+
bzip2 \
33+
bzip2-devel \
34+
clang \
35+
curl \
36+
cyrus-sasl-devel \
37+
doxygen \
38+
fuse \
39+
fuse-libs \
40+
fuse-devel \
41+
git \
42+
libcurl-devel \
43+
libtirpc-devel \
44+
libpmem-devel \
45+
libtool \
46+
lz4-devel \
47+
make \
48+
openssl-devel \
49+
pinentry-curses \
50+
python3 \
51+
python3-pip \
52+
python3-setuptools \
53+
python3-wheel \
54+
rsync \
55+
snappy-devel \
56+
sudo \
57+
valgrind \
58+
zlib-devel
59+
60+
# Set GCC 9 as the default C/C++ compiler
61+
RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
62+
SHELL ["/bin/bash", "--login", "-c"]
63+
64+
####
65+
# Install Maven 3.6.3
66+
####
67+
RUN mkdir -p /opt/maven /tmp/maven \
68+
&& curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
69+
-o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \
70+
&& tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
71+
72+
####
73+
# Install CMake 3.19
74+
####
75+
# hadolint ignore=DL3003
76+
RUN mkdir -p /tmp/cmake /opt/cmake \
77+
&& curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \
78+
&& tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \
79+
&& cd /opt/cmake || exit && ./bootstrap \
80+
&& make "-j$(nproc)" \
81+
&& make install \
82+
&& cd /root || exit
83+
84+
####
85+
# Install zstandard
86+
####
87+
# hadolint ignore=DL3003
88+
RUN mkdir -p /opt/zstd /tmp/zstd \
89+
&& curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \
90+
&& tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \
91+
&& cd /opt/zstd || exit \
92+
&& make "-j$(nproc)" \
93+
&& make install \
94+
&& cd /root || exit
95+
96+
######
97+
# Set env vars required to build Hadoop
98+
######
99+
ENV MAVEN_HOME /opt/maven
100+
ENV PATH "${PATH}:${MAVEN_HOME}/bin"
101+
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
102+
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
103+
104+
#######
105+
# Install SpotBugs 4.2.2
106+
#######
107+
RUN mkdir -p /opt/spotbugs \
108+
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
109+
-o /opt/spotbugs.tgz \
110+
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
111+
&& chmod +x /opt/spotbugs/bin/*
112+
ENV SPOTBUGS_HOME /opt/spotbugs
113+
114+
#######
115+
# Install Boost 1.72 (1.71 ships with Focal)
116+
#######
117+
# hadolint ignore=DL3003
118+
RUN mkdir -p /opt/boost-library \
119+
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
120+
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
121+
&& cd /opt/boost-library \
122+
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
123+
&& cd /opt/boost-library/boost_1_72_0 \
124+
&& ./bootstrap.sh --prefix=/usr/ \
125+
&& ./b2 --without-python install \
126+
&& cd /root \
127+
&& rm -rf /opt/boost-library
128+
129+
######
130+
# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
131+
######
132+
# hadolint ignore=DL3003
133+
RUN mkdir -p /opt/protobuf-src \
134+
&& curl -L -s -S \
135+
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
136+
-o /opt/protobuf.tar.gz \
137+
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
138+
&& cd /opt/protobuf-src \
139+
&& ./configure --prefix=/opt/protobuf \
140+
&& make "-j$(nproc)" \
141+
&& make install \
142+
&& cd /root \
143+
&& rm -rf /opt/protobuf-src
144+
ENV PROTOBUF_HOME /opt/protobuf
145+
ENV PATH "${PATH}:/opt/protobuf/bin"
146+
147+
####
148+
# Install Node.js
149+
####
150+
# hadolint ignore=DL3003
151+
RUN mkdir -p /tmp/node \
152+
&& curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \
153+
&& tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \
154+
&& cd /tmp/node || exit \
155+
&& ./configure \
156+
&& make "-j$(nproc)" \
157+
&& make install \
158+
&& cd /root || exit
159+
160+
####
161+
# Install pylint and python-dateutil
162+
####
163+
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
164+
165+
####
166+
# Install bower
167+
####
168+
# hadolint ignore=DL3008
169+
RUN npm install -g bower@1.8.8

‎hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,13 +58,6 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
5858
allowed_expr+="|^java.policy$"
5959
# * Used by javax.annotation
6060
allowed_expr+="|^jndi.properties$"
61-
# * allowing native libraries from rocksdb. Leaving native libraries as it is.
62-
allowed_expr+="|^librocksdbjni-linux32.so"
63-
allowed_expr+="|^librocksdbjni-linux64.so"
64-
allowed_expr+="|^librocksdbjni-osx.jnilib"
65-
allowed_expr+="|^librocksdbjni-win64.dll"
66-
allowed_expr+="|^librocksdbjni-linux-ppc64le.so"
67-
6861

6962
allowed_expr+=")"
7063
declare -i bad_artifacts=0

‎hadoop-client-modules/hadoop-client-minicluster/pom.xml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -760,13 +760,6 @@
760760
<exclude>xml.xsd</exclude>
761761
</excludes>
762762
</filter>
763-
<!-- filtering HISTORY-JAVA.md from rocksdb jar -->
764-
<filter>
765-
<artifact>org.rocksdb:rocksdbjni</artifact>
766-
<excludes>
767-
<exclude>HISTORY-JAVA.md</exclude>
768-
</excludes>
769-
</filter>
770763
<filter>
771764
<!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
772765
<artifact>org.eclipse.jetty:*</artifact>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,9 @@
6060
<build>
6161
<plugins>
6262
<plugin>
63-
<groupId>org.codehaus.mojo</groupId>
64-
<artifactId>findbugs-maven-plugin</artifactId>
63+
<groupId>com.github.spotbugs</groupId>
64+
<artifactId>spotbugs-maven-plugin</artifactId>
6565
<configuration>
66-
<findbugsXmlOutput>true</findbugsXmlOutput>
6766
<xmlOutput>true</xmlOutput>
6867
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
6968
</excludeFilterFile>
@@ -116,7 +115,7 @@
116115
<dependency>
117116
<groupId>junit</groupId>
118117
<artifactId>junit</artifactId>
119-
<version>4.12</version>
118+
<version>${junit.version}</version>
120119
<scope>test</scope>
121120
</dependency>
122121
<dependency>

‎hadoop-common-project/hadoop-auth/pom.xml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -193,11 +193,7 @@
193193
<artifactId>guava</artifactId>
194194
<scope>test</scope>
195195
</dependency>
196-
<dependency>
197-
<groupId>org.eclipse.jetty</groupId>
198-
<artifactId>jetty-server</artifactId>
199-
</dependency>
200-
</dependencies>
196+
</dependencies>
201197

202198
<build>
203199
<plugins>

‎hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import org.apache.hadoop.security.authentication.client.AuthenticationException;
2020
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
2121
import org.apache.hadoop.security.authentication.util.*;
22-
import org.eclipse.jetty.server.Response;
2322
import org.slf4j.Logger;
2423
import org.slf4j.LoggerFactory;
2524

@@ -621,18 +620,15 @@ && getMaxInactiveInterval() > 0) {
621620
errCode = HttpServletResponse.SC_FORBIDDEN;
622621
}
623622
// After Jetty 9.4.21, sendError() no longer allows a custom message.
624-
// use setStatusWithReason() to set a custom message.
623+
// use setStatus() to set a custom message.
625624
String reason;
626625
if (authenticationEx == null) {
627626
reason = "Authentication required";
628627
} else {
629628
reason = authenticationEx.getMessage();
630629
}
631630

632-
if (httpResponse instanceof Response) {
633-
((Response)httpResponse).setStatusWithReason(errCode, reason);
634-
}
635-
631+
httpResponse.setStatus(errCode, reason);
636632
httpResponse.sendError(errCode, reason);
637633
}
638634
}

‎hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,16 @@ private static Oid getNumericOidInstance(String oidName) {
7979
*
8080
* @return Oid instance
8181
* @param oidName The oid Name
82+
* @throws ClassNotFoundException for backward compatibility.
83+
* @throws GSSException for backward compatibility.
8284
* @throws NoSuchFieldException if the input is not supported.
85+
* @throws IllegalAccessException for backward compatibility.
86+
*
8387
*/
8488
@Deprecated
8589
public static Oid getOidInstance(String oidName)
86-
throws NoSuchFieldException {
90+
throws ClassNotFoundException, GSSException, NoSuchFieldException,
91+
IllegalAccessException {
8792
switch (oidName) {
8893
case "GSS_SPNEGO_MECH_OID":
8994
return GSS_SPNEGO_MECH_OID;

0 commit comments

Comments
 (0)