Merge pull request #1 from apache/trunk

merge from upstream

Author: bitmybytes

BUILDING.txt

@@ -5,7 +5,7 @@ Requirements:
 
 * Unix System
 * JDK 1.8+
-* Maven 3.0 or later
+* Maven 3.3 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
 * CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac

LICENSE.txt

@@ -631,9 +631,9 @@ Azure Data Lake Store - Java client SDK 2.0.11
 JCodings 1.0.8
 Joni 2.1.2
 Mockito 1.8.5
-JUL to SLF4J bridge 1.7.10
-SLF4J API Module 1.7.10
-SLF4J LOG4J-12 Binding 1.7.10
+JUL to SLF4J bridge 1.7.25
+SLF4J API Module 1.7.25
+SLF4J LOG4J-12 Binding 1.7.25
 --------------------------------------------------------------------------------
 
 The MIT License (MIT)
@@ -736,7 +736,7 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE
 
 The binary distribution of this product bundles these dependencies under the
 following license:
-HSQLDB Database 2.0.0
+HSQLDB Database 2.3.4
 --------------------------------------------------------------------------------
 (HSQL License)
 "COPYRIGHTS AND LICENSES (based on BSD License)
@@ -1711,7 +1711,6 @@ Hamcrest Core 1.3
 ASM Core 5.0.4
 ASM Commons 5.0.2
 ASM Tree 5.0.2
-xmlenc Library 0.52
 --------------------------------------------------------------------------------
 (3-clause BSD)
 Redistribution and use in source and binary forms, with or without
@@ -2659,4 +2658,24 @@ available under the Creative Commons By Attribution 3.0 License.
         available upon request from time to time. For the avoidance of doubt,
         this trademark restriction does not form part of this License.
 
-        Creative Commons may be contacted at https://creativecommons.org/.
+        Creative Commons may be contacted at https://creativecommons.org/.
+--------------------------------------------------------------------------------
+
+For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs
+/server/datanode/checker/AbstractFuture.java and
+hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs
+/server/datanode/checker/TimeoutFuture.java
+
+Copyright (C) 2007 The Guava Authors
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.

dev-support/bin/yetus-wrapper

@@ -14,6 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# you must be this high to ride the ride
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+   || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+   || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+  echo "bash v3.2+ is required. Sorry."
+  exit 1
+fi
+
 set -o pipefail
 
 ## @description  Print a message to stderr
@@ -39,6 +47,7 @@ function yetus_abs
   declare obj=$1
   declare dir
   declare fn
+  declare dirret
 
   if [[ ! -e ${obj} ]]; then
     return 1
@@ -51,7 +60,8 @@ function yetus_abs
   fi
 
   dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
-  if [[ $? = 0 ]]; then
+  dirret=$?
+  if [[ ${dirret} = 0 ]]; then
     echo "${dir}${fn}"
     return 0
   fi
@@ -63,7 +73,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.3.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.4.0}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
@@ -85,7 +95,8 @@ if [[ ! -d "${HADOOP_PATCHPROCESS}" ]]; then
 fi
 
 mytmpdir=$(yetus_abs "${HADOOP_PATCHPROCESS}")
-if [[ $? != 0 ]]; then
+ret=$?
+if [[ ${ret} != 0 ]]; then
   yetus_error "yetus-dl: Unable to cwd to ${HADOOP_PATCHPROCESS}"
   exit 1
 fi
@@ -108,15 +119,13 @@ TARBALL="yetus-${HADOOP_YETUS_VERSION}-bin.tar"
 GPGBIN=$(command -v gpg)
 CURLBIN=$(command -v curl)
 
-pushd "${HADOOP_PATCHPROCESS}" >/dev/null
-if [[ $? != 0 ]]; then
+if ! pushd "${HADOOP_PATCHPROCESS}" >/dev/null; then
   yetus_error "ERROR: yetus-dl: Cannot pushd to ${HADOOP_PATCHPROCESS}"
   exit 1
 fi
 
 if [[ -n "${CURLBIN}" ]]; then
-  "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz"
-  if [[ $? != 0 ]]; then
+  if ! "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz"; then
     yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz"
     exit 1
   fi
@@ -126,40 +135,33 @@ else
 fi
 
 if [[ -n "${GPGBIN}" ]]; then
-  mkdir -p .gpg
-  if [[ $? != 0 ]]; then
+  if ! mkdir -p .gpg; then
     yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg"
     exit 1
   fi
-  chmod -R 700 .gpg
-  if [[ $? != 0 ]]; then
+  if ! chmod -R 700 .gpg; then
     yetus_error "ERROR: yetus-dl: Unable to chmod ${HADOOP_PATCHPROCESS}/.gpg"
     exit 1
   fi
-  "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS
-  if [[ $? != 0 ]]; then
+  if ! "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS; then
     yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS"
     exit 1
   fi
-  "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"
-  if [[ $? != 0 ]]; then
+  if ! "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"; then
     yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc"
     exit 1
   fi
-  "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1
-  if [[ $? != 0 ]]; then
+  if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1; then
     yetus_error "ERROR: yetus-dl: gpg unable to import ${HADOOP_PATCHPROCESS}/KEYS_YETUS"
     exit 1
   fi
-  "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1
-   if [[ $? != 0 ]]; then
+  if ! "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1; then
      yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HADOOP_PATCHPROCESS} failed"
      exit 1
    fi
 fi
 
-gunzip -c "${TARBALL}.gz" | tar xpf -
-if [[ $? != 0 ]]; then
+if ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then
   yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HADOOP_PATCHPROCESS} to try again."
   exit 1
 fi

dev-support/docker/Dockerfile

@@ -58,6 +58,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
     protobuf-c-compiler \
     python \
     python2.7 \
+    python2.7-dev \
     python-pip \
     rsync \
     snappy \
@@ -81,13 +82,17 @@ RUN apt-get -q install --no-install-recommends -y oracle-java8-installer
 ####
 # Apps that require Java
 ###
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
-    ant \
-    maven
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant
 
-# Fixing the Apache commons / Maven dependency problem under Ubuntu:
-# See http://wiki.apache.org/commons/VfsProblems
-RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
+######
+# Install Apache Maven
+######
+RUN mkdir -p /opt/maven && \
+    curl -L -s -S \
+         http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
+         -o /opt/maven.tar.gz && \
+    tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven
+ENV MAVEN_HOME /opt/maven
 
 ######
 # Install findbugs

hadoop-client-modules/hadoop-client-integration-tests/pom.xml

@@ -145,6 +145,11 @@
           <artifactId>hadoop-common</artifactId>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+          <scope>test</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-hdfs</artifactId>

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -218,10 +218,6 @@
           <groupId>javax.servlet</groupId>
           <artifactId>javax.servlet-api</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>xmlenc</groupId>
-          <artifactId>xmlenc</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <!-- Add optional runtime dependency on the in-development timeline server module

hadoop-client-modules/hadoop-client/pom.xml

@@ -85,10 +85,6 @@
           <groupId>net.sf.kosmosfs</groupId>
           <artifactId>kfs</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>net.java.dev.jets3t</groupId>
-          <artifactId>jets3t</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.jcraft</groupId>
           <artifactId>jsch</artifactId>

hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

@@ -93,10 +93,6 @@
           <groupId>net.sf.kosmosfs</groupId>
           <artifactId>kfs</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>net.java.dev.jets3t</groupId>
-          <artifactId>jets3t</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.jcraft</groupId>
           <artifactId>jsch</artifactId>

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java

@@ -15,8 +15,9 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
-import java.nio.charset.Charset;
+import java.security.SecureRandom;
 import java.util.Random;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -32,7 +33,7 @@ public class RandomSignerSecretProvider extends RolloverSignerSecretProvider {
 
   public RandomSignerSecretProvider() {
     super();
-    rand = new Random();
+    rand = new SecureRandom();
   }
 
   /**
@@ -48,6 +49,8 @@ public RandomSignerSecretProvider(long seed) {
 
   @Override
   protected byte[] generateNewSecret() {
-    return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8"));
+    byte[] secret = new byte[32]; // 32 bytes = 256 bits
+    rand.nextBytes(secret);
+    return secret;
   }
 }

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java

@@ -16,6 +16,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.security.SecureRandom;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -149,7 +150,7 @@ public class ZKSignerSecretProvider extends RolloverSignerSecretProvider {
 
   public ZKSignerSecretProvider() {
     super();
-    rand = new Random();
+    rand = new SecureRandom();
   }
 
   /**
@@ -342,8 +343,11 @@ private synchronized void pullFromZK(boolean isInit) {
     }
   }
 
-  private byte[] generateRandomSecret() {
-    return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8"));
+  @VisibleForTesting
+  protected byte[] generateRandomSecret() {
+    byte[] secret = new byte[32]; // 32 bytes = 256 bits
+    rand.nextBytes(secret);
+    return secret;
   }
 
   /**