Merge pull request #9 from apache/trunk

sync

LICENSE-binary

@@ -218,12 +218,12 @@ com.aliyun.oss:aliyun-sdk-oss:3.13.0
 com.amazonaws:aws-java-sdk-bundle:1.11.901
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.9.9
-com.fasterxml.jackson.core:jackson-core:2.9.9
-com.fasterxml.jackson.core:jackson-databind:2.9.9.2
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9
+com.fasterxml.jackson.core:jackson-annotations:2.13.0
+com.fasterxml.jackson.core:jackson-core:2.13.0
+com.fasterxml.jackson.core:jackson-databind:2.13.0
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.0
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.0
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.0
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.3.0
 com.github.davidmoten:rxjava-extras:0.8.0.17
@@ -494,6 +494,7 @@ javax.annotation:javax.annotation-api:1.3.2
 javax.servlet:javax.servlet-api:3.1.0
 javax.servlet.jsp:jsp-api:2.1
 javax.websocket:javax.websocket-api:1.0
+javax.ws.rs:javax.ws.rs-api:2.1.1
 javax.ws.rs:jsr311-api:1.1.1
 javax.xml.bind:jaxb-api:2.2.11
 

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -690,6 +690,7 @@
                       <exclude>org.bouncycastle:*</exclude>
                       <!-- Leave snappy that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <exclude>javax.ws.rs:javax.ws.rs-api</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -736,6 +737,12 @@
                         <exclude>testdata/*</exclude>
                       </excludes>
                     </filter>
+                    <filter>
+                      <artifact>com.fasterxml.jackson.*:*</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/11/module-info.class</exclude>
+                      </excludes>
+                    </filter>
 
                     <!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
                     <filter>

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -163,6 +163,7 @@
                       <exclude>org.bouncycastle:*</exclude>
                       <!-- Leave snappy that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <exclude>javax.ws.rs:javax.ws.rs-api</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -242,6 +243,12 @@
                         <exclude>google/protobuf/**/*.proto</exclude>
                       </excludes>
                     </filter>
+                    <filter>
+                      <artifact>com.fasterxml.jackson.*:*</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/11/module-info.class</exclude>
+                      </excludes>
+                    </filter>
                   </filters>
                   <relocations>
                     <relocation>

hadoop-common-project/hadoop-common/pom.xml

@@ -915,7 +915,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
               <forkCount>${testsThreadCount}</forkCount>
               <reuseForks>false</reuseForks>
               <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java

@@ -84,14 +84,14 @@ static int get(String padding) throws NoSuchPaddingException {
     String loadingFailure = null;
     try {
       if (!NativeCodeLoader.buildSupportsOpenssl()) {
-        PerformanceAdvisory.LOG.debug("Build does not support openssl");
+        PerformanceAdvisory.LOG.warn("Build does not support openssl");
         loadingFailure = "build does not support openssl.";
       } else {
         initIDs();
       }
     } catch (Throwable t) {
       loadingFailure = t.getMessage();
-      LOG.debug("Failed to load OpenSSL Cipher.", t);
+      LOG.warn("Failed to load OpenSSL Cipher.", t);
     } finally {
       loadingFailureReason = loadingFailure;
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -1324,6 +1324,14 @@ public String toString() {
     }
   }
 
+  @VisibleForTesting
+  @InterfaceAudience.Private
+  protected void configureSocketChannel(SocketChannel channel) throws IOException {
+    channel.configureBlocking(false);
+    channel.socket().setTcpNoDelay(tcpNoDelay);
+    channel.socket().setKeepAlive(true);
+  }
+
   /** Listens on the socket. Creates jobs for the handler threads*/
   private class Listener extends Thread {
 
@@ -1530,15 +1538,24 @@ private void closeCurrentConnection(SelectionKey key, Throwable e) {
     InetSocketAddress getAddress() {
       return (InetSocketAddress)acceptChannel.socket().getLocalSocketAddress();
     }
-    
+
     void doAccept(SelectionKey key) throws InterruptedException, IOException,  OutOfMemoryError {
       ServerSocketChannel server = (ServerSocketChannel) key.channel();
       SocketChannel channel;
       while ((channel = server.accept()) != null) {
 
-        channel.configureBlocking(false);
-        channel.socket().setTcpNoDelay(tcpNoDelay);
-        channel.socket().setKeepAlive(true);
+        try {
+          configureSocketChannel(channel);
+        } catch (IOException e) {
+          LOG.warn("Error in an accepted SocketChannel", e);
+          try {
+            channel.socket().close();
+            channel.close();
+          } catch (IOException ex) {
+            LOG.warn("Error in closing SocketChannel", ex);
+          }
+          continue;
+        }
 
         Reader reader = getReader();
         Connection c = connectionManager.register(channel,

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java

@@ -34,6 +34,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.DataInput;
 import java.io.DataOutput;
+import java.io.EOFException;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -48,6 +49,7 @@
 import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
+import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -616,6 +618,51 @@ public void testIOEOnWriteAfterPingClient() throws Exception {
       WRITABLE_FAULTS_SLEEP = 0;
     }
   }
+
+  /**
+   * Test for HADOOP-18024.
+   */
+  @Test(timeout=60000)
+  public void testIOEOnListenerAccept() throws Exception {
+    // start server
+    Server server = new TestServer(1, false,
+            LongWritable.class, LongWritable.class) {
+      @Override
+      protected void configureSocketChannel(SocketChannel channel) throws IOException {
+        maybeThrowIOE();
+        super.configureSocketChannel(channel);
+      }
+    };
+    InetSocketAddress addr = NetUtils.getConnectAddress(server);
+    server.start();
+
+    // start client
+    WRITABLE_FAULTS_ENABLED = true;
+    Client client = new Client(LongWritable.class, conf);
+    try {
+      LongWritable param = LongWritable.class.newInstance();
+
+      try {
+        call(client, param, addr, 0, conf);
+        fail("Expected an exception to have been thrown");
+      } catch (EOFException e) {
+        LOG.info("Got expected exception", e);
+      } catch (Throwable t) {
+        LOG.warn("Got unexpected error", t);
+        fail("Expected an EOFException to have been thrown");
+      }
+
+      // Doing a second call with faults disabled should return fine --
+      // ie the internal state of the client or server should not be broken
+      // by the failed call
+      WRITABLE_FAULTS_ENABLED = false;
+      call(client, param, addr, 0, conf);
+
+    } finally {
+      client.stop();
+      server.stop();
+    }
+  }
 
   private static void assertExceptionContains(
       Throwable t, String substring) {

hadoop-common-project/hadoop-kms/pom.xml

@@ -186,7 +186,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <forkCount>1</forkCount>
           <reuseForks>false</reuseForks>
           <threadCount>1</threadCount>

hadoop-common-project/hadoop-registry/pom.xml

@@ -231,7 +231,6 @@
       <groupId>org.apache.maven.plugins</groupId>
       <artifactId>maven-surefire-plugin</artifactId>
       <configuration>
-        <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
         <reuseForks>false</reuseForks>
         <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
         <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>

hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml

@@ -247,7 +247,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <threadCount>1</threadCount>
           <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
           <systemPropertyVariables>
@@ -361,7 +360,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
               <forkCount>1</forkCount>
               <reuseForks>true</reuseForks>
               <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt

@@ -23,6 +23,9 @@ add_executable(hdfs_tool_tests
                 hdfs-delete-snapshot-mock.cc
                 hdfs-create-snapshot-mock.cc
                 hdfs-cat-mock.cc
+                hdfs-chown-mock.cc
+                hdfs-chmod-mock.cc
+                hdfs-chgrp-mock.cc
                 hdfs-tool-test-fixtures.cc
                 hdfs-tool-tests.cc
                 hdfs-df-mock.cc
@@ -36,6 +39,9 @@ target_include_directories(hdfs_tool_tests PRIVATE
                             ../../tools/hdfs-delete-snapshot
                             ../../tools/hdfs-create-snapshot
                             ../../tools/hdfs-rename-snapshot
+                            ../../tools/hdfs-chown
+                            ../../tools/hdfs-chgrp
+                            ../../tools/hdfs-chmod
                             ../../tools/hdfs-cat)
 target_link_libraries(hdfs_tool_tests PRIVATE
                        gmock_main
@@ -45,5 +51,8 @@ target_link_libraries(hdfs_tool_tests PRIVATE
                        hdfs_deleteSnapshot_lib
                        hdfs_createSnapshot_lib
                        hdfs_renameSnapshot_lib
+                       hdfs_chown_lib
+                       hdfs_chgrp_lib
+                       hdfs_chmod_lib
                        hdfs_cat_lib)
 add_test(hdfs_tool_tests hdfs_tool_tests)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc

@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-chgrp-mock.h"
+#include "hdfs-chgrp.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+ChgrpMock::~ChgrpMock() = default;
+
+void ChgrpMock::SetExpectations(
+    std::function<std::unique_ptr<ChgrpMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<ChgrpMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<ChgrpMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassOwnerAndAPath<ChgrpMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+
+    EXPECT_CALL(*this, HandlePath(arg1, false, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursiveOwnerAndAPath<ChgrpMock>) {
+    const auto arg1 = args[1];
+    const auto arg2 = args[2];
+
+    EXPECT_CALL(*this, HandlePath(arg1, true, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test