From 28c533a582f8b6070508ba2e57662d4576ac05d2 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 31 Aug 2023 14:54:53 +0100 Subject: [PATCH 001/155] Revert "HADOOP-18860. Upgrade mockito version to 4.11.0 (#5977)" This reverts commit 1046f9cf9888155c27923f3f56efa107d908ad5b. --- .../org/apache/hadoop/ipc/TestServer.java | 2 +- .../security/http/TestCrossOriginFilter.java | 9 ++++--- .../http/TestRestCsrfPreventionFilter.java | 10 +++---- .../federation/router/TestRouterAdmin.java | 16 +++++------- .../router/TestRouterRpcMultiDestination.java | 2 +- .../server/namenode/TestCacheDirectives.java | 2 +- .../namenode/TestSnapshotPathINodes.java | 2 +- hadoop-project/pom.xml | 18 +------------ .../containerlaunch/TestAbstractLauncher.java | 4 +-- .../resources/TestCGroupsHandlerImpl.java | 8 +++--- .../gpu/TestGpuResourceAllocator.java | 4 +-- .../runtime/TestDockerContainerRuntime.java | 4 +-- .../com/nec/TestNECVEPlugin.java | 6 ++--- ...TestFSConfigToCSConfigArgumentHandler.java | 6 ++--- .../TestFSConfigToCSConfigConverter.java | 4 +-- .../TestQueuePlacementConverter.java | 26 +++++++++---------- hadoop-yarn-project/pom.xml | 5 ---- 17 files changed, 53 insertions(+), 75 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java index b6e696fddeefa..748d99e2a0d34 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java @@ -151,7 +151,7 @@ public Writable call( // Nothing should be logged for a suppressed exception. server.logException(logger, new TestException1(), dummyCall); - verifyNoInteractions(logger); + verifyZeroInteractions(logger); // No stack trace should be logged for a terse exception. server.logException(logger, new TestException2(), dummyCall); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java index 9f7a6114b0ac3..0b396be48f983 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java @@ -58,7 +58,8 @@ public void testSameOrigin() throws ServletException, IOException { CrossOriginFilter filter = new CrossOriginFilter(); filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -223,7 +224,7 @@ public void testDisallowedOrigin() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -251,7 +252,7 @@ public void testDisallowedMethod() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -282,7 +283,7 @@ public void testDisallowedHeader() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java index f125ffac1e191..6052ef059a732 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java @@ -75,7 +75,7 @@ public void testNoHeaderDefaultConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -110,7 +110,7 @@ public void testNoHeaderCustomAgentConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -228,7 +228,7 @@ public void testMissingHeaderWithCustomHeaderConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -260,7 +260,7 @@ public void testMissingHeaderNoMethodsToIgnoreConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -356,6 +356,6 @@ public void testMissingHeaderMultipleIgnoreMethodsConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index dac5be209ab11..c2eaddc17a2a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.HashMap; @@ -69,6 +68,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.util.reflection.FieldSetter; /** * The administrator interface of the {@link Router} implemented by @@ -118,20 +118,18 @@ public static void globalSetUp() throws Exception { * @throws IOException * @throws NoSuchFieldException */ - private static void setUpMocks() throws IOException, NoSuchFieldException, - IllegalAccessException { + private static void setUpMocks() throws IOException, NoSuchFieldException { RouterRpcServer spyRpcServer = Mockito.spy(routerContext.getRouter().createRpcServer()); - Field rpcServerField = Router.class.getDeclaredField("rpcServer"); - rpcServerField.setAccessible(true); - rpcServerField.set(routerContext.getRouter(), spyRpcServer); + FieldSetter.setField(routerContext.getRouter(), + Router.class.getDeclaredField("rpcServer"), spyRpcServer); Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString()); // mock rpc client for destination check when editing mount tables. mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient()); - Field rpcClientField = RouterRpcServer.class.getDeclaredField("rpcClient"); - rpcClientField.setAccessible(true); - rpcClientField.set(spyRpcServer, mockRpcClient); + FieldSetter.setField(spyRpcServer, + RouterRpcServer.class.getDeclaredField("rpcClient"), + mockRpcClient); RemoteLocation remoteLocation0 = new RemoteLocation("ns0", "/testdir", null); RemoteLocation remoteLocation1 = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index ab51a8224271a..336ea3913859e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.apache.hadoop.test.Whitebox.getInternalState; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 48c1527de1031..1331c50e80b3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -1575,7 +1575,7 @@ public void testNoLookupsWhenNotUsed() throws Exception { CacheManager cm = cluster.getNamesystem().getCacheManager(); LocatedBlocks locations = Mockito.mock(LocatedBlocks.class); cm.setCachedLocations(locations); - Mockito.verifyNoInteractions(locations); + Mockito.verifyZeroInteractions(locations); } @Test(timeout=120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index cda5b39f1d03e..b62a4180d43ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -447,6 +447,6 @@ public void testShortCircuitSnapshotSearch() throws SnapshotException { INodesInPath iip = Mockito.mock(INodesInPath.class); List snapDirs = new ArrayList<>(); FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs); - Mockito.verifyNoInteractions(iip); + Mockito.verifyZeroInteractions(iip); } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b2176ccaa90d7..9fca0fa159b87 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -84,9 +84,6 @@ 1.1 - - 4.11.0 - 2.5.0 @@ -1306,20 +1303,7 @@ org.mockito mockito-core - ${mockito.version} - test - - - org.mockito - mockito-inline - ${mockito.version} - test - - - org.mockito - mockito-core - - + 2.28.2 org.mockito diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java index 6b3da713964cc..31ca38297c856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java @@ -43,7 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; /** @@ -90,7 +90,7 @@ public void testContainerRetries() throws Exception { providerService.buildContainerRetry(mockLauncher, getConfig(), componentLaunchContext, componentInstance); - verifyNoInteractions(mockLauncher); + verifyZeroInteractions(mockLauncher); //OnFailure restart policy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index ff5b9be6ddc08..b1e8989213ba1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -51,8 +51,8 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; /** * Tests for the CGroups handler implementation. @@ -192,7 +192,7 @@ public void testMountController() throws IOException { assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock - verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); File emptyMtab = createEmptyCgroups(); try { @@ -238,7 +238,7 @@ public void testMountController() throws IOException { public void testCGroupPaths() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); @@ -281,7 +281,7 @@ public void testCGroupPaths() throws IOException { public void testCGroupOperations() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java index 07ce7b256d4c6..ba8a9309d02e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java @@ -27,7 +27,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -210,7 +210,7 @@ private void assertAllocatedGpus(int gpus, int deniedGpus, private void assertNoAllocation(GpuAllocation allocation) { assertEquals(1, allocation.getDeniedGPUs().size()); assertEquals(0, allocation.getAllowedGPUs().size()); - verifyNoInteractions(nmStateStore); + verifyZeroInteractions(nmStateStore); } private void assertAssignmentInStateStore(GpuDevice expectedGpu, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index f13d8388f9a32..ea7c213809330 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1272,7 +1272,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added here - Mockito.verifyNoMoreInteractions(command); + Mockito.verifyZeroInteractions(command); String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy + containerIdStr; @@ -1296,7 +1296,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added in either case - Mockito.verifyNoMoreInteractions(command); + Mockito.verifyZeroInteractions(command); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index a9b01bca6fac7..86ef9058f26fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anyString; @@ -394,7 +394,7 @@ public void testFindDevicesWithUdev() assertEquals("No. of devices", 1, devices.size()); Device device = devices.iterator().next(); assertSame("Device", device, testDevice); - verifyNoInteractions(mockCommandExecutor); + verifyZeroInteractions(mockCommandExecutor); verify(mockEnvProvider).apply(eq("NEC_USE_UDEV")); verifyNoMoreInteractions(mockEnvProvider); } @@ -442,6 +442,6 @@ private Device getTestDevice(int id) { private void verifyBinaryPathSet(Path expectedPath) { assertEquals("Binary path", expectedPath.toString(), plugin.getBinaryPath()); - verifyNoInteractions(udevUtil); + verifyZeroInteractions(udevUtil); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index 33cc3f0dba266..cb8cc587f68f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -23,7 +23,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.File; import java.io.IOException; @@ -666,7 +666,7 @@ public void testValidationSkippedWhenCmdLineSwitchIsDefined() FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s"); argumentHandler.parseAndConvert(args); - verifyNoInteractions(mockValidator); + verifyZeroInteractions(mockValidator); } @Test @@ -681,7 +681,7 @@ public void testValidationSkippedWhenOutputIsConsole() throws Exception { FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s", "-p"); argumentHandler.parseAndConvert(args); - verifyNoInteractions(mockValidator); + verifyZeroInteractions(mockValidator); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 530c6ddc55301..55c43666cdb5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -36,7 +36,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.ByteArrayInputStream; import java.io.File; @@ -673,7 +673,7 @@ public void testPlacementRulesConversionDisabled() throws Exception { converter.setPlacementConverter(placementConverter); converter.convert(params); - verifyNoInteractions(placementConverter); + verifyZeroInteractions(placementConverter); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java index 6871444baa03c..6599080aab59c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java @@ -21,7 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.util.List; @@ -85,7 +85,7 @@ public void testConvertUserRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -96,7 +96,7 @@ public void testConvertSpecifiedRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SPECIFIED); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -108,7 +108,7 @@ public void testConvertPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -120,7 +120,7 @@ public void testConvertSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -134,7 +134,7 @@ public void testConvertDefaultRuleWithQueueName() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.CUSTOM); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -147,7 +147,7 @@ public void testConvertDefaultRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.DEFAULT_QUEUE); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -168,7 +168,7 @@ public void testConvertRejectRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.REJECT); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -182,7 +182,7 @@ public void testConvertNestedPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP_USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -197,7 +197,7 @@ public void testConvertNestedSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP_USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -215,7 +215,7 @@ public void testConvertNestedDefaultRule() { Rule rule = description.getRules().get(0); verifyRule(description.getRules().get(0), Policy.USER); assertEquals("Parent path", "root.abc", rule.getParentQueue()); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -245,7 +245,7 @@ public void testConvertMultiplePlacementRules() { verifyRule(description.getRules().get(0), Policy.USER); verifyRule(description.getRules().get(1), Policy.PRIMARY_GROUP); verifyRule(description.getRules().get(2), Policy.SECONDARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -363,7 +363,7 @@ private void testConvertNestedRuleCreateFlagInWeightMode( any(Policy.class)); verifyNoMoreInteractions(ruleHandler); } else { - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } } diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 78f09a3971069..241e3bc237a0e 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -90,11 +90,6 @@ hadoop-yarn-applications-catalog-webapp war - - org.mockito - mockito-core - test - From 01cc6d0bc8236d87eefa646a9ad2161841984fd3 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 31 Aug 2023 19:40:04 +0530 Subject: [PATCH 002/155] HADOOP-18865. ABFS: Add "100-continue" in userAgent if enabled (#5987) Contributed by Anmol Asrani --- .../services/AppendRequestParameters.java | 10 ++++ .../fs/azurebfs/services/AbfsClient.java | 17 ++++++ .../fs/azurebfs/services/ITestAbfsClient.java | 58 +++++++++++++++---- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java index 57e559a60ec84..9da6427d65c2c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java @@ -35,6 +35,7 @@ public enum Mode { private final boolean isAppendBlob; private final String leaseId; private boolean isExpectHeaderEnabled; + private boolean isRetryDueToExpect; public AppendRequestParameters(final long position, final int offset, @@ -50,6 +51,7 @@ public AppendRequestParameters(final long position, this.isAppendBlob = isAppendBlob; this.leaseId = leaseId; this.isExpectHeaderEnabled = isExpectHeaderEnabled; + this.isRetryDueToExpect = false; } public long getPosition() { @@ -80,6 +82,14 @@ public boolean isExpectHeaderEnabled() { return isExpectHeaderEnabled; } + public boolean isRetryDueToExpect() { + return isRetryDueToExpect; + } + + public void setRetryDueToExpect(boolean retryDueToExpect) { + isRetryDueToExpect = retryDueToExpect; + } + public void setExpectHeaderEnabled(boolean expectHeaderEnabled) { isExpectHeaderEnabled = expectHeaderEnabled; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 77b8dcb2b9891..45cb538d0b007 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -87,6 +87,7 @@ */ public class AbfsClient implements Closeable { public static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); + public static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON; private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; @@ -751,6 +752,15 @@ public AbfsRestOperation append(final String path, final byte[] buffer, } } + // Check if the retry is with "Expect: 100-continue" header being present in the previous request. + if (reqParams.isRetryDueToExpect()) { + String userAgentRetry = userAgent; + // Remove the specific marker related to "Expect: 100-continue" from the User-Agent string. + userAgentRetry = userAgentRetry.replace(HUNDRED_CONTINUE_USER_AGENT, EMPTY_STRING); + requestHeaders.removeIf(header -> header.getName().equalsIgnoreCase(USER_AGENT)); + requestHeaders.add(new AbfsHttpHeader(USER_AGENT, userAgentRetry)); + } + // AbfsInputStream/AbfsOutputStream reuse SAS tokens for better performance String sasTokenForReuse = appendSASTokenToQuery(path, SASTokenProvider.WRITE_OPERATION, abfsUriQueryBuilder, cachedSasToken); @@ -780,6 +790,7 @@ public AbfsRestOperation append(final String path, final byte[] buffer, if (checkUserError(responseStatusCode) && reqParams.isExpectHeaderEnabled()) { LOG.debug("User error, retrying without 100 continue enabled for the given path {}", path); reqParams.setExpectHeaderEnabled(false); + reqParams.setRetryDueToExpect(true); return this.append(path, buffer, reqParams, cachedSasToken, tracingContext); } @@ -1371,6 +1382,12 @@ String initializeUserAgent(final AbfsConfiguration abfsConfiguration, appendIfNotEmpty(sb, ExtensionHelper.getUserAgentSuffix(tokenProvider, EMPTY_STRING), true); + if (abfsConfiguration.isExpectHeaderEnabled()) { + sb.append(SINGLE_WHITE_SPACE); + sb.append(HUNDRED_CONTINUE); + sb.append(SEMICOLON); + } + sb.append(SINGLE_WHITE_SPACE); sb.append(abfsConfiguration.getClusterName()); sb.append(FORWARD_SLASH); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java index c031e5daa6c44..18d1e3917f24e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java @@ -86,6 +86,7 @@ public final class ITestAbfsClient extends AbstractAbfsIntegrationTest { private static final String ACCOUNT_NAME = "bogusAccountName.dfs.core.windows.net"; private static final String FS_AZURE_USER_AGENT_PREFIX = "Partner Service"; + private static final String HUNDRED_CONTINUE_USER_AGENT = SINGLE_WHITE_SPACE + HUNDRED_CONTINUE + SEMICOLON; private static final String TEST_PATH = "/testfile"; public static final int REDUCED_RETRY_COUNT = 2; public static final int REDUCED_BACKOFF_INTERVAL = 100; @@ -143,15 +144,15 @@ private String getUserAgentString(AbfsConfiguration config, } @Test - public void verifybBasicInfo() throws Exception { + public void verifyBasicInfo() throws Exception { final Configuration configuration = new Configuration(); configuration.addResource(TEST_CONFIGURATION_FILE_NAME); AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, ACCOUNT_NAME); - verifybBasicInfo(getUserAgentString(abfsConfiguration, false)); + verifyBasicInfo(getUserAgentString(abfsConfiguration, false)); } - private void verifybBasicInfo(String userAgentStr) { + private void verifyBasicInfo(String userAgentStr) { Assertions.assertThat(userAgentStr) .describedAs("User-Agent string [" + userAgentStr + "] should be of the pattern: " + this.userAgentStringPattern.pattern()) @@ -180,7 +181,7 @@ public void verifyUserAgentPrefix() ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain " + FS_AZURE_USER_AGENT_PREFIX) .contains(FS_AZURE_USER_AGENT_PREFIX); @@ -190,12 +191,47 @@ public void verifyUserAgentPrefix() ACCOUNT_NAME); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain " + FS_AZURE_USER_AGENT_PREFIX) .doesNotContain(FS_AZURE_USER_AGENT_PREFIX); } + /** + * This method represents a unit test for verifying the behavior of the User-Agent header + * with respect to the "Expect: 100-continue" header setting in the Azure Blob File System (ABFS) configuration. + * + * The test ensures that the User-Agent string includes or excludes specific information based on whether the + * "Expect: 100-continue" header is enabled or disabled in the configuration. + * + */ + @Test + public void verifyUserAgentExpectHeader() + throws IOException, IllegalAccessException { + final Configuration configuration = new Configuration(); + configuration.addResource(TEST_CONFIGURATION_FILE_NAME); + configuration.set(ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY, FS_AZURE_USER_AGENT_PREFIX); + configuration.setBoolean(ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED, true); + AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, + ACCOUNT_NAME); + String userAgentStr = getUserAgentString(abfsConfiguration, false); + + verifyBasicInfo(userAgentStr); + Assertions.assertThat(userAgentStr) + .describedAs("User-Agent string should contain " + HUNDRED_CONTINUE_USER_AGENT) + .contains(HUNDRED_CONTINUE_USER_AGENT); + + configuration.setBoolean(ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED, false); + abfsConfiguration = new AbfsConfiguration(configuration, + ACCOUNT_NAME); + userAgentStr = getUserAgentString(abfsConfiguration, false); + + verifyBasicInfo(userAgentStr); + Assertions.assertThat(userAgentStr) + .describedAs("User-Agent string should not contain " + HUNDRED_CONTINUE_USER_AGENT) + .doesNotContain(HUNDRED_CONTINUE_USER_AGENT); + } + @Test public void verifyUserAgentWithoutSSLProvider() throws Exception { final Configuration configuration = new Configuration(); @@ -206,14 +242,14 @@ public void verifyUserAgentWithoutSSLProvider() throws Exception { ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, true); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain sslProvider") .contains(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName()); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain sslProvider") .doesNotContain(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName()); @@ -229,7 +265,7 @@ public void verifyUserAgentClusterName() throws Exception { ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain cluster name") .contains(clusterName); @@ -239,7 +275,7 @@ public void verifyUserAgentClusterName() throws Exception { ACCOUNT_NAME); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain cluster name") .doesNotContain(clusterName) @@ -257,7 +293,7 @@ public void verifyUserAgentClusterType() throws Exception { ACCOUNT_NAME); String userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should contain cluster type") .contains(clusterType); @@ -267,7 +303,7 @@ public void verifyUserAgentClusterType() throws Exception { ACCOUNT_NAME); userAgentStr = getUserAgentString(abfsConfiguration, false); - verifybBasicInfo(userAgentStr); + verifyBasicInfo(userAgentStr); Assertions.assertThat(userAgentStr) .describedAs("User-Agent string should not contain cluster type") .doesNotContain(clusterType) From 2f739450be88df2929c25a59ab4a48f84df5cbc2 Mon Sep 17 00:00:00 2001 From: Yang Jiandan Date: Fri, 1 Sep 2023 00:14:46 +0800 Subject: [PATCH 003/155] YARN-11552. Timeline endpoint: /clusters/{clusterid}/apps/{appid}/entity-types Error when using hdfs store (#5978) Contributed by Jiandan Yang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../storage/FileSystemTimelineReaderImpl.java | 25 +++++++++++++------ .../TestFileSystemTimelineReaderImpl.java | 24 ++++++++++-------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index 913b8360e2f26..dff21a31da952 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -164,7 +164,7 @@ private static void fillFields(TimelineEntity finalEntity, private String getFlowRunPath(String userId, String clusterId, String flowName, Long flowRunId, String appId) throws IOException { if (userId != null && flowName != null && flowRunId != null) { - return userId + File.separator + flowName + File.separator + flowRunId; + return userId + File.separator + flowName + File.separator + "*" + File.separator + flowRunId; } if (clusterId == null || appId == null) { throw new IOException("Unable to get flow info"); @@ -186,7 +186,7 @@ private String getFlowRunPath(String userId, String clusterId, continue; } return record.get(1).trim() + File.separator + record.get(2).trim() + - File.separator + record.get(3).trim(); + File.separator + "*" + File.separator + record.get(3).trim(); } parser.close(); } @@ -286,6 +286,7 @@ public int compare(Long l1, Long l2) { } } ); + dir = getNormalPath(dir); if (dir != null) { RemoteIterator fileStatuses = fs.listFiles(dir, false); @@ -394,9 +395,11 @@ public TimelineEntity getEntity(TimelineReaderContext context, Path flowRunPath = new Path(clusterIdPath, flowRunPathStr); Path appIdPath = new Path(flowRunPath, context.getAppId()); Path entityTypePath = new Path(appIdPath, context.getEntityType()); - Path entityFilePath = new Path(entityTypePath, - context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION); - + Path entityFilePath = getNormalPath(new Path(entityTypePath, + context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION)); + if (entityFilePath == null) { + return null; + } try (BufferedReader reader = new BufferedReader(new InputStreamReader( fs.open(entityFilePath), Charset.forName("UTF-8")))) { @@ -410,6 +413,14 @@ public TimelineEntity getEntity(TimelineReaderContext context, } } + private Path getNormalPath(Path globPath) throws IOException { + FileStatus[] status = fs.globStatus(globPath); + if (status == null || status.length < 1) { + LOG.info("{} do not exist.", globPath); + return null; + } + return status[0].getPath(); + } @Override public Set getEntities(TimelineReaderContext context, TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) @@ -433,13 +444,13 @@ public Set getEntities(TimelineReaderContext context, context.getClusterId(), context.getFlowName(), context.getFlowRunId(), context.getAppId()); if (context.getUserId() == null) { - context.setUserId(new Path(flowRunPathStr).getParent().getParent(). + context.setUserId(new Path(flowRunPathStr).getParent().getParent().getParent(). getName()); } Path clusterIdPath = new Path(entitiesPath, context.getClusterId()); Path flowRunPath = new Path(clusterIdPath, flowRunPathStr); Path appIdPath = new Path(flowRunPath, context.getAppId()); - FileStatus[] fileStatuses = fs.listStatus(appIdPath); + FileStatus[] fileStatuses = fs.listStatus(getNormalPath(appIdPath)); for (FileStatus fileStatus : fileStatuses) { if (fileStatus.isDirectory()) { result.add(fileStatus.getPath().getName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java index cf94749e883e8..47e5a490514c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java @@ -66,6 +66,11 @@ public class TestFileSystemTimelineReaderImpl { private static final String ROOT_DIR = new File("target", TestFileSystemTimelineReaderImpl.class.getSimpleName()).getAbsolutePath(); + private static String cluster = "cluster1"; + private static String user = "user1"; + private static String flowVersion = "v1"; + private static String flowRunId = "1"; + private FileSystemTimelineReaderImpl reader; @BeforeAll @@ -125,7 +130,7 @@ private static void writeEntityFile(TimelineEntity entity, File dir) private static void loadEntityData(String rootDir) throws Exception { File appDir = - getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", "app"); + getAppDir(rootDir, "flow1", "app1", "app"); TimelineEntity entity11 = new TimelineEntity(); entity11.setId("id_1"); entity11.setType("app"); @@ -266,8 +271,9 @@ private static void loadEntityData(String rootDir) throws Exception { entity4.addEvent(event44); writeEntityFile(entity4, appDir); - File attemptDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", - "app1", TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); + + File attemptDir = getAppDir(rootDir, "flow1", "app1", + TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); ApplicationAttemptEntity attempt1 = new ApplicationAttemptEntity(); attempt1.setId("app-attempt-1"); attempt1.setCreatedTime(1425017502003L); @@ -277,8 +283,8 @@ private static void loadEntityData(String rootDir) throws Exception { attempt2.setCreatedTime(1425017502004L); writeEntityFile(attempt2, attemptDir); - File entityDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", - "app1", TimelineEntityType.YARN_CONTAINER.toString()); + File entityDir = getAppDir(rootDir, "flow1", "app1", + TimelineEntityType.YARN_CONTAINER.toString()); ContainerEntity containerEntity1 = new ContainerEntity(); containerEntity1.setId("container_1_1"); containerEntity1.setParent(attempt1.getIdentifier()); @@ -298,8 +304,7 @@ private static void loadEntityData(String rootDir) throws Exception { writeEntityFile(containerEntity3, entityDir); File appDir2 = - getAppDir(rootDir, "cluster1", "user1", "flow1,flow", "1", "app2", - "app"); + getAppDir(rootDir, "flow1,flow", "app2", "app"); TimelineEntity entity5 = new TimelineEntity(); entity5.setId("id_5"); entity5.setType("app"); @@ -307,10 +312,9 @@ private static void loadEntityData(String rootDir) throws Exception { writeEntityFile(entity5, appDir2); } - private static File getAppDir(String rootDir, String cluster, String user, - String flowName, String flowRunId, String appId, String entityName) { + private static File getAppDir(String rootDir, String flowName, String appId, String entityName) { return new File(rootDir + File.separator + "entities" + File.separator + - cluster + File.separator + user + File.separator + flowName + + cluster + File.separator + user + File.separator + flowName + File.separator + flowVersion + File.separator + flowRunId + File.separator + appId + File.separator + entityName + File.separator); } From b6d06c89282153a0b5607e6c1236e4138669a1d9 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 1 Sep 2023 01:46:33 +0800 Subject: [PATCH 004/155] YARN-11554. Fix TestRMFailover#testWebAppProxyInStandAloneMode Failed. (#5971) --- .../test/java/org/apache/hadoop/yarn/client/TestRMFailover.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index d9f6d4e470887..a744714846bce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -212,6 +212,8 @@ public void testAutomaticFailover() public void testWebAppProxyInStandAloneMode() throws YarnException, InterruptedException, IOException { conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); + conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + WebAppProxyServer webAppProxyServer = new WebAppProxyServer(); try { conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099"); From 9857fbcd611819ca98e4ed2a4d14665445d40434 Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Sat, 2 Sep 2023 15:24:52 +0800 Subject: [PATCH 005/155] HDFS-17168. Support getTrashRoots API in WebHDFS (#5992) Contributed by Hualong Zhang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../hadoop/hdfs/DFSOpsCountStatistics.java | 1 + .../hadoop/hdfs/DistributedFileSystem.java | 2 + .../hadoop/hdfs/web/JsonUtilClient.java | 35 +++++++++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 20 ++++++ .../hdfs/web/resources/AllUsersParam.java | 49 +++++++++++++++ .../hadoop/hdfs/web/resources/GetOpParam.java | 1 + .../router/RouterWebHdfsMethods.java | 6 +- .../web/resources/NamenodeWebHdfsMethods.java | 31 ++++++++-- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 ++++++++++ .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 61 +++++++++++++++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 29 +++++++++ 11 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AllUsersParam.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 04fef2d5dcd23..05d9c562392cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -75,6 +75,7 @@ public enum OpType { GET_STORAGE_POLICIES("op_get_storage_policies"), GET_STORAGE_POLICY("op_get_storage_policy"), GET_TRASH_ROOT("op_get_trash_root"), + GET_TRASH_ROOTS("op_get_trash_roots"), GET_XATTR("op_get_xattr"), LIST_CACHE_DIRECTIVE("op_list_cache_directive"), LIST_CACHE_POOL("op_list_cache_pool"), diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 1f4c3f8d772ab..16093c8e9920c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -3612,6 +3612,8 @@ public Path getTrashRoot(Path path) { */ @Override public Collection getTrashRoots(boolean allUsers) { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOTS); Set ret = new HashSet<>(); // Get normal trash roots ret.addAll(super.getTrashRoots(allUsers)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 90dcd83ddba1e..108f74997a63e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.fs.ContentSummary; @@ -877,6 +878,40 @@ public static Map getErasureCodeCodecs(Map json) { return map; } + public static Collection getTrashRoots(Map json) { + List objs = (List) json.get("Paths"); + if (objs != null) { + FileStatus[] trashRoots = new FileStatus[objs.size()]; + for (int i = 0; i < objs.size(); i++) { + Map m = (Map) objs.get(i); + trashRoots[i] = toFileStatus(m); + } + return Arrays.asList(trashRoots); + } + return new ArrayList(0); + } + + public static FileStatus toFileStatus(Map json) { + Path path = new Path(getString(json, "path", "")); + long length = getLong(json, "length", 0); + boolean isdir = getBoolean(json, "isdir", false); + short replication = (short) getInt(json, "block_replication", -1); + long blockSize = getLong(json, "blocksize", 256); + long modificationTime = getLong(json, "modification_time", 0); + long accessTime = getLong(json, "access_time", 0); + String permString = getString(json, "permission", null); + FsPermission permission = toFsPermission(permString); + String owner = getString(json, "owner", null); + String group = getString(json, "group", null); + if (json.get("symlink") != null) { + Path symlink = new Path((String) json.get("symlink")); + return new FileStatus(length, isdir, replication, blockSize, modificationTime, + accessTime, permission, owner, group, symlink, path); + } + return new FileStatus(length, isdir, replication, blockSize, modificationTime, + accessTime, permission, owner, group, path); + } + private static List toDiffList( List objs) { if (objs == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 5210692ab324e..689a7c78d4e84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -1956,6 +1956,26 @@ String decodeResponse(Map json) throws IOException { } } + public Collection getTrashRoots(boolean allUsers) { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOTS); + + final HttpOpParam.Op op = GetOpParam.Op.GETTRASHROOTS; + try { + Collection trashRoots = + new FsPathResponseRunner>(op, null, + new AllUsersParam(allUsers)) { + @Override + Collection decodeResponse(Map json) throws IOException { + return JsonUtilClient.getTrashRoots(json); + } + }.run(); + return trashRoots; + } catch (IOException e) { + return super.getTrashRoots(allUsers); + } + } + @Override public void access(final Path path, final FsAction mode) throws IOException { final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AllUsersParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AllUsersParam.java new file mode 100644 index 0000000000000..1f7228adb6f42 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AllUsersParam.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.web.resources; + +/** AllUsers parameter. */ +public class AllUsersParam extends BooleanParam { + /** Parameter name. */ + public static final String NAME = "allusers"; + /** Default parameter value. */ + public static final String DEFAULT = FALSE; + + private static final Domain DOMAIN = new Domain(NAME); + + /** + * Constructor. + * @param value the parameter value. + */ + public AllUsersParam(final Boolean value) { + super(DOMAIN, value); + } + + /** + * Constructor. + * @param str a string representation of the parameter value. + */ + public AllUsersParam(final String str) { + this(DOMAIN.parse(str)); + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 3efe37b61f80a..1e811abc095cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -69,6 +69,7 @@ public enum Op implements HttpOpParam.Op { GETSTATUS(false, HttpURLConnection.HTTP_OK), GETECPOLICIES(false, HttpURLConnection.HTTP_OK), GETECCODECS(false, HttpURLConnection.HTTP_OK), + GETTRASHROOTS(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK); final boolean redirect; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java index 42962290daf82..953def5f1c7ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.AccessTimeParam; import org.apache.hadoop.hdfs.web.resources.AclPermissionParam; +import org.apache.hadoop.hdfs.web.resources.AllUsersParam; import org.apache.hadoop.hdfs.web.resources.BlockSizeParam; import org.apache.hadoop.hdfs.web.resources.BufferSizeParam; import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam; @@ -344,7 +345,8 @@ protected Response get( final TokenKindParam tokenKind, final TokenServiceParam tokenService, final NoRedirectParam noredirectParam, - final StartAfterParam startAfter + final StartAfterParam startAfter, + final AllUsersParam allUsers ) throws IOException, URISyntaxException { try { final Router router = getRouter(); @@ -393,7 +395,7 @@ protected Response get( offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, snapshotDiffStartPath, snapshotDiffIndex, - tokenKind, tokenService, noredirectParam, startAfter); + tokenKind, tokenService, noredirectParam, startAfter, allUsers); } default: throw new UnsupportedOperationException(op + " is not supported"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index c00ff5d8ec5e8..affab74e4f1db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -30,6 +30,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Base64; import java.util.Base64.Encoder; +import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; import java.util.List; @@ -56,6 +57,7 @@ import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.Status; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; @@ -1067,14 +1069,16 @@ public Response getRoot( @QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT) final NoRedirectParam noredirect, @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT) - final StartAfterParam startAfter + final StartAfterParam startAfter, + @QueryParam(AllUsersParam.NAME) @DefaultValue(AllUsersParam.DEFAULT) + final AllUsersParam allUsers ) throws IOException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, snapshotDiffStartPath, snapshotDiffIndex, tokenKind, tokenService, - noredirect, startAfter); + noredirect, startAfter, allUsers); } /** Handle HTTP GET request. */ @@ -1124,12 +1128,14 @@ public Response get( @QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT) final NoRedirectParam noredirect, @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT) - final StartAfterParam startAfter + final StartAfterParam startAfter, + @QueryParam(AllUsersParam.NAME) @DefaultValue(AllUsersParam.DEFAULT) + final AllUsersParam allUsers ) throws IOException, InterruptedException { init(ugi, delegation, username, doAsUser, path, op, offset, length, renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction, - snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter); + snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter, allUsers); return doAs(ugi, new PrivilegedExceptionAction() { @Override @@ -1138,7 +1144,7 @@ public Response run() throws IOException, URISyntaxException { op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, snapshotDiffStartPath, snapshotDiffIndex, - tokenKind, tokenService, noredirect, startAfter); + tokenKind, tokenService, noredirect, startAfter, allUsers); } }); } @@ -1172,7 +1178,8 @@ protected Response get( final TokenKindParam tokenKind, final TokenServiceParam tokenService, final NoRedirectParam noredirectParam, - final StartAfterParam startAfter + final StartAfterParam startAfter, + final AllUsersParam allUsers ) throws IOException, URISyntaxException { final Configuration conf = (Configuration) context .getAttribute(JspHelper.CURRENT_CONF); @@ -1322,6 +1329,12 @@ protected Response get( final String jsonStr = JsonUtil.toJsonString("Path", trashPath); return Response.ok(jsonStr).type(MediaType.APPLICATION_JSON).build(); } + case GETTRASHROOTS: { + Boolean value = allUsers.getValue(); + final Collection trashPaths = getTrashRoots(conf, value); + final String js = JsonUtil.toJsonString(trashPaths); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } case LISTSTATUS_BATCH: { byte[] start = HdfsFileStatus.EMPTY_NAME; @@ -1558,6 +1571,12 @@ public Void run() throws IOException { }; } + private Collection getTrashRoots(Configuration conf, boolean allUsers) + throws IOException { + FileSystem fs = FileSystem.get(conf != null ? conf : new Configuration()); + return fs.getTrashRoots(allUsers); + } + /** Handle HTTP DELETE request for the root. */ @DELETE diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 9b9fe94cc6dd3..5f90404ebee25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -752,6 +752,27 @@ public static Map toJsonMap(ErasureCodingPolicyInfo ecPolicyInfo return m; } + public static Map toJsonMap(FileStatus status) throws IOException { + Map m = new HashMap<>(); + m.put("path", status.getPath().toUri().getPath()); + m.put("length", status.getLen()); + m.put("isdir", status.isDirectory()); + m.put("block_replication", status.getReplication()); + m.put("blocksize", status.getBlockSize()); + m.put("modification_time", status.getModificationTime()); + m.put("access_time", status.getAccessTime()); + FsPermission perm = status.getPermission(); + if (perm != null) { + m.put("permission", toString(perm)); + } + m.put("owner", status.getOwner()); + m.put("group", status.getGroup()); + if (status.isSymlink()) { + m.put("symlink", status.getSymlink()); + } + return m; + } + public static String toJsonString(ErasureCodingPolicyInfo[] ecPolicyInfos) { final Map erasureCodingPolicies = new HashMap<>(); Object[] erasureCodingPolicyInfos = null; @@ -764,4 +785,16 @@ public static String toJsonString(ErasureCodingPolicyInfo[] ecPolicyInfos) { erasureCodingPolicies.put("ErasureCodingPolicyInfo", erasureCodingPolicyInfos); return toJsonString("ErasureCodingPolicies", erasureCodingPolicies); } + + public static String toJsonString(Collection trashPaths) throws IOException { + List list = new ArrayList<>(trashPaths); + Object[] paths = null; + if(list.size() > 0){ + paths = new Object[list.size()]; + for(int i = 0; i < list.size(); i++){ + paths[i] = toJsonMap(list.get(i)); + } + } + return toJsonString("Paths", paths); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index 969d288b4c97b..8e266077f7a0d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -63,6 +63,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop * [`GETSTATUS`](#Get_Status) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStatus) * [`GETECPOLICIES`](#Get_EC_Policies) * [`GETECCODECS`](#Get_EC_Codecs) + * [`GETTRASHROOTS`](#Get_Trash_Roots) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoots) * HTTP PUT * [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create) * [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs) @@ -1273,6 +1274,47 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStatus } } +### Get Trash Roots + +* Submit a HTTP GET request. + + curl -i "http://:/webhdfs/v1/?op=GETTRASHROOTS + &allusers=" + + The client receives a response with a [`Paths` JSON object](#Paths_JSON_Schema): + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + + { + "Paths": [{ + "blocksize": 0, + "owner": "hadoop", + "path": "/user/user0/.Trash", + "length": 0, + "permission": "755", + "modification_time": 1693050205747, + "isdir": true, + "block_replication": 0, + "access_time": 0, + "group": "supergroup" + }, { + "blocksize": 0, + "owner": "hadoop", + "path": "/user/user1/.Trash", + "length": 0, + "permission": "755", + "modification_time": 1693049382962, + "isdir": true, + "block_replication": 0, + "access_time": 0, + "group": "supergroup" + }] + } + +See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoots + Storage Policy Operations ------------------------- @@ -3277,6 +3319,25 @@ var blockLocationProperties = } ``` +### Paths JSON Schema + +```json +{ + "Paths": [{ + "blocksize": 0, + "owner": "hadoop", + "path": "/user/user0/.Trash", + "length": 0, + "permission": "755", + "modification_time": 1693050205747, + "isdir": true, + "block_replication": 0, + "access_time": 0, + "group": "supergroup" + }] +} +``` + HTTP Query Parameter Dictionary ------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 18382d201e3e8..805a06a6b8623 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -60,6 +60,7 @@ import java.util.NoSuchElementException; import java.util.Random; +import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.QuotaUsage; @@ -2342,6 +2343,34 @@ public void getAllErasureCodingCodecs() throws Exception { } } + @Test + public void testGetTrashRoots() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + cluster = new MiniDFSCluster.Builder(conf).build(); + final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + + // Create user trash + Path currUserHome = webFS.getHomeDirectory(); + Path currUserTrash = new Path(currUserHome, FileSystem.TRASH_PREFIX); + webFS.mkdirs(currUserTrash); + + Collection webTrashRoots = webFS.getTrashRoots(true); + assertEquals(1, webTrashRoots.size()); + + // Create trash root for user0 + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user0"); + String user0HomeStr = DFSUtilClient.getHomeDirectory(conf, ugi); + Path user0Trash = new Path(user0HomeStr, FileSystem.TRASH_PREFIX); + webFS.mkdirs(user0Trash); + + webTrashRoots = webFS.getTrashRoots(true); + assertEquals(2, webTrashRoots.size()); + + webTrashRoots = webFS.getTrashRoots(false); + assertEquals(1, webTrashRoots.size()); + } + /** * Get FileStatus JSONObject from ListStatus response. */ From d5334fa76170b99f1ddd6b307482d226da12f1a9 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sun, 3 Sep 2023 15:38:21 +0800 Subject: [PATCH 006/155] YARN-6537. Running RM tests against the Router. (#5957) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../yarn/server/nodemanager/NodeManager.java | 5 + .../hadoop-yarn-server-router/pom.xml | 18 ++ .../subcluster/TestFederationSubCluster.java | 210 ++++++++++++++++++ .../router/subcluster/TestMockRouter.java | 93 ++++++++ .../router/subcluster/TestMockSubCluster.java | 100 +++++++++ ...stYarnFederationWithCapacityScheduler.java | 76 +++++++ .../TestYarnFederationWithFairScheduler.java | 75 +++++++ .../server/router/webapp/JavaProcess.java | 24 +- .../webapp/TestRouterWebServicesREST.java | 151 ++++++------- .../src/test/resources/fair-scheduler.xml | 43 ++++ .../hadoop/yarn/server/MiniYARNCluster.java | 19 ++ 12 files changed, 729 insertions(+), 87 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestFederationSubCluster.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockRouter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockSubCluster.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/capacity/TestYarnFederationWithCapacityScheduler.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/fair/TestYarnFederationWithFairScheduler.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/fair-scheduler.xml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e252590ea3188..874ee9d08d9ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -517,7 +517,7 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION = false; /** The address of the RM admin interface.*/ - public static final String RM_ADMIN_ADDRESS = + public static final String RM_ADMIN_ADDRESS = RM_PREFIX + "admin.address"; public static final int DEFAULT_RM_ADMIN_PORT = 8033; public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 438a39b0973e1..9092425042781 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -1074,4 +1074,9 @@ private NMLogAggregationStatusTracker createNMLogAggregationStatusTracker( public AsyncDispatcher getDispatcher() { return dispatcher; } + + @VisibleForTesting + public void disableWebServer() { + removeService(((NMContext) context).webServer); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml index ec4fe86e7c295..b171876471772 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml @@ -150,6 +150,24 @@ test-jar + + org.apache.curator + curator-client + + + + org.apache.curator + curator-test + test + + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestFederationSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestFederationSubCluster.java new file mode 100644 index 0000000000000..f9cd707821814 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestFederationSubCluster.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.subcluster; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.commons.collections.CollectionUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.retry.RetryNTimes; +import org.apache.curator.test.TestingServer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.retry.RetryPolicy; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.router.webapp.JavaProcess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.TimeoutException; + +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static javax.ws.rs.core.MediaType.APPLICATION_XML; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH; +import static org.apache.hadoop.yarn.server.router.webapp.TestRouterWebServicesREST.waitWebAppRunning; +import static org.junit.Assert.assertEquals; + +public class TestFederationSubCluster { + + private static final Logger LOG = LoggerFactory.getLogger(TestFederationSubCluster.class); + private static TestingServer curatorTestingServer; + private static CuratorFramework curatorFramework; + private static JavaProcess subCluster1; + private static JavaProcess subCluster2; + private static JavaProcess router; + public static final String ZK_FEDERATION_STATESTORE = + "org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore"; + private static String userName = "test"; + + public void startFederationSubCluster(int zkPort, String sc1Param, + String sc2Param, String routerParam) throws IOException, InterruptedException, + YarnException, TimeoutException { + + // Step1. Initialize ZK's service. + try { + curatorTestingServer = new TestingServer(zkPort); + curatorTestingServer.start(); + String connectString = curatorTestingServer.getConnectString(); + curatorFramework = CuratorFrameworkFactory.builder() + .connectString(connectString) + .retryPolicy(new RetryNTimes(100, 100)) + .build(); + curatorFramework.start(); + curatorFramework.getConnectionStateListenable().addListener((client, newState) -> { + if (newState == ConnectionState.CONNECTED) { + System.out.println("Connected to the ZooKeeper server!"); + } + }); + } catch (Exception e) { + LOG.error("Cannot initialize ZooKeeper store.", e); + throw new IOException(e); + } + + // Step2. Create a temporary directory output log. + File baseDir = GenericTestUtils.getTestDir("processes"); + baseDir.mkdirs(); + String baseName = TestFederationSubCluster.class.getSimpleName(); + + // Step3. Initialize subCluster SC-1 + String sc1WebAddress = getSCWebAddress(sc1Param); + File rmOutput = new File(baseDir, baseName + "-" + Time.now() + "-rm.log"); + rmOutput.createNewFile(); + List addClasspath = new LinkedList<>(); + addClasspath.add("../hadoop-yarn-server-timelineservice/target/classes"); + subCluster1 = new JavaProcess(TestMockSubCluster.class, addClasspath, rmOutput, sc1Param); + waitWebAppRunning(sc1WebAddress, RM_WEB_SERVICE_PATH); + + // Step4. Initialize subCluster SC-2 + String sc2WebAddress = getSCWebAddress(sc2Param); + File rmOutput2 = new File(baseDir, baseName + "-" + Time.now() + "-rm.log"); + rmOutput2.createNewFile(); + List addClasspath2 = new LinkedList<>(); + addClasspath2.add("../hadoop-yarn-server-timelineservice/target/classes"); + subCluster2 = new JavaProcess(TestMockSubCluster.class, addClasspath2, rmOutput2, sc2Param); + waitWebAppRunning(sc2WebAddress, RM_WEB_SERVICE_PATH); + + // Step5. Confirm that subClusters have been registered to ZK. + String zkAddress = getZkAddress(zkPort); + verifyRegistration(zkAddress); + + // Step6. Initialize router + String routerWebAddress = getRouterWebAddress(routerParam); + File routerOutput = new File(baseDir, baseName + "-" + Time.now() + "-router.log"); + routerOutput.createNewFile(); + router = new JavaProcess(TestMockRouter.class, null, routerOutput, routerParam); + waitWebAppRunning(routerWebAddress, RM_WEB_SERVICE_PATH); + } + + private String getSCWebAddress(String scParam) { + String[] scParams = scParam.split(","); + return "http://localhost:" + scParams[3]; + } + + private String getRouterWebAddress(String routerParam) { + String[] routerParams = routerParam.split(","); + return "http://localhost:" + routerParams[2]; + } + + private String getZkAddress(int port) { + return "localhost:" + port; + } + + public void stop() throws Exception { + if (subCluster1 != null) { + subCluster1.stop(); + } + if (subCluster2 != null) { + subCluster2.stop(); + } + if (router != null) { + router.stop(); + } + if (curatorTestingServer != null) { + curatorTestingServer.stop(); + } + } + + private void verifyRegistration(String zkAddress) + throws YarnException, InterruptedException, TimeoutException { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.set(YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS, ZK_FEDERATION_STATESTORE); + conf.set(CommonConfigurationKeys.ZK_ADDRESS, zkAddress); + RetryPolicy retryPolicy = FederationStateStoreFacade.createRetryPolicy(conf); + FederationStateStore stateStore = (FederationStateStore) + FederationStateStoreFacade.createRetryInstance(conf, + YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS, + YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS, + FederationStateStore.class, retryPolicy); + stateStore.init(conf); + FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + GetSubClustersInfoRequest request = GetSubClustersInfoRequest.newInstance(true); + + GenericTestUtils.waitFor(() -> { + try { + GetSubClustersInfoResponse response = stateStore.getSubClusters(request); + List subClusters = response.getSubClusters(); + if (CollectionUtils.isNotEmpty(subClusters)) { + return true; + } + } catch (Exception e) { + } + return false; + }, 5000, 50 * 1000); + } + + public static T performGetCalls(final String routerAddress, final String path, + final Class returnType, final String queryName, + final String queryValue) throws IOException, InterruptedException { + + Client clientToRouter = Client.create(); + WebResource toRouter = clientToRouter.resource(routerAddress).path(path); + + final WebResource.Builder toRouterBuilder; + + if (queryValue != null && queryName != null) { + toRouterBuilder = toRouter.queryParam(queryName, queryValue).accept(APPLICATION_XML); + } else { + toRouterBuilder = toRouter.accept(APPLICATION_XML); + } + + return UserGroupInformation.createRemoteUser(userName).doAs( + (PrivilegedExceptionAction) () -> { + ClientResponse response = toRouterBuilder.get(ClientResponse.class); + assertEquals(SC_OK, response.getStatus()); + return response.getEntity(returnType); + }); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockRouter.java new file mode 100644 index 0000000000000..751a3fed6487d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockRouter.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.subcluster; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.retry.RetryPolicy; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.router.Router; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests {@link Router}. + */ +public class TestMockRouter { + + private static final Logger LOG = LoggerFactory.getLogger(TestMockRouter.class); + + public TestMockRouter() { + } + + public static void main(String[] args) throws YarnException { + if (ArrayUtils.isEmpty(args)) { + return; + } + + // Step1. Parse the parameters. + String[] params = args[0].split(","); + int pRouterClientRMPort = Integer.parseInt(params[0]); + int pRouterAdminAddressPort = Integer.parseInt(params[1]); + int pRouterWebAddressPort = Integer.parseInt(params[2]); + String zkAddress = params[3]; + + LOG.info("routerClientRMPort={}, routerAdminAddressPort={}, routerWebAddressPort={}, " + + "zkAddress = {}.", pRouterClientRMPort, pRouterAdminAddressPort, + pRouterWebAddressPort, zkAddress); + + YarnConfiguration conf = new YarnConfiguration(); + Router router = new Router(); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.set(YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS, + "org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore"); + conf.set(YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, + "org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST"); + conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, + "org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor"); + conf.set(CommonConfigurationKeys.ZK_ADDRESS, zkAddress); + conf.set(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, + "org.apache.hadoop.yarn.server.router.rmadmin.FederationRMAdminInterceptor"); + conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, -1); + conf.set(YarnConfiguration.ROUTER_CLIENTRM_ADDRESS, getHostNameAndPort(pRouterClientRMPort)); + conf.set(YarnConfiguration.ROUTER_RMADMIN_ADDRESS, getHostNameAndPort(pRouterAdminAddressPort)); + conf.set(YarnConfiguration.ROUTER_WEBAPP_ADDRESS, + getHostNameAndPort(pRouterWebAddressPort)); + + RetryPolicy retryPolicy = FederationStateStoreFacade.createRetryPolicy(conf); + + router.init(conf); + router.start(); + + FederationStateStore stateStore = (FederationStateStore) + FederationStateStoreFacade.createRetryInstance(conf, + YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS, + YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS, + FederationStateStore.class, retryPolicy); + stateStore.init(conf); + FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + } + + private static String getHostNameAndPort(int port) { + return MiniYARNCluster.getHostname() + ":" + port; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockSubCluster.java new file mode 100644 index 0000000000000..00227ae9ffc56 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestMockSubCluster.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.subcluster; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster.ZK_FEDERATION_STATESTORE; + +public class TestMockSubCluster { + + private static final Logger LOG = LoggerFactory.getLogger(TestMockSubCluster.class); + private Configuration conf; + private String subClusterId; + + public TestMockSubCluster() { + } + + public TestMockSubCluster(String pSubClusterId, Configuration pConf) { + this.conf = pConf; + this.subClusterId = pSubClusterId; + } + + private static String getHostNameAndPort(int port) { + return MiniYARNCluster.getHostname() + ":" + port; + } + + public void startYarnSubCluster() { + MiniYARNCluster yrCluster = new MiniYARNCluster(subClusterId, 3, 1, 1, false); + yrCluster.init(conf); + yrCluster.start(); + } + + public static void main(String[] args) { + if (ArrayUtils.isEmpty(args)) { + return; + } + + // Step1. Parse the parameters. + String[] params = args[0].split(","); + int pRmAddressPort = Integer.parseInt(params[0]); + int pRmSchedulerAddressPort = Integer.parseInt(params[1]); + int pRmTrackerAddressPort = Integer.parseInt(params[2]); + int pRmWebAddressPort = Integer.parseInt(params[3]); + int pRmAdminAddressPort = Integer.parseInt(params[4]); + String pSubClusterId = params[5]; + String pZkAddress = params[6]; + String schedulerType = params[7]; + + // Step 2. Print the parameters. + LOG.info("subClusterId = {}, rmAddressPort = {}, rmSchedulerAddressPort = {}, " + + "rmTrackerAddressPort = {}, rmWebAddressPort = {}, rmAdminAddressPort = {}", + pSubClusterId, pRmAddressPort, pRmSchedulerAddressPort, pRmTrackerAddressPort, + pRmWebAddressPort, pRmAdminAddressPort); + + // Step 3. determine which scheduler to use. + Configuration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_ADDRESS, getHostNameAndPort(pRmAddressPort)); + conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, getHostNameAndPort(pRmAdminAddressPort)); + conf.set(YarnConfiguration.RM_HOSTNAME, MiniYARNCluster.getHostname()); + conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, getHostNameAndPort(pRmSchedulerAddressPort)); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + getHostNameAndPort(pRmTrackerAddressPort)); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, getHostNameAndPort(pRmWebAddressPort)); + conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.set(YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS, ZK_FEDERATION_STATESTORE); + conf.set(CommonConfigurationKeys.ZK_ADDRESS, pZkAddress); + conf.set(YarnConfiguration.RM_CLUSTER_ID, pSubClusterId); + if (schedulerType.equals("fair-scheduler")) { + conf.set("yarn.resourcemanager.scheduler.class", + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + conf.set("yarn.scheduler.fair.allocation.file", "fair-scheduler.xml"); + } + + // Step 4, start the mockSubCluster cluster. + TestMockSubCluster sc = new TestMockSubCluster(pSubClusterId, conf); + sc.startYarnSubCluster(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/capacity/TestYarnFederationWithCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/capacity/TestYarnFederationWithCapacityScheduler.java new file mode 100644 index 0000000000000..f37e1245bdcdd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/capacity/TestYarnFederationWithCapacityScheduler.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.subcluster.capacity; + +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; +import org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster; +import org.apache.hadoop.yarn.server.router.webapp.dao.FederationClusterInfo; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestYarnFederationWithCapacityScheduler { + + private static TestFederationSubCluster testFederationSubCluster; + private static Set subClusters; + private static final String ROUTER_WEB_ADDRESS = "http://localhost:18089"; + + @BeforeClass + public static void setUp() + throws IOException, InterruptedException, YarnException, TimeoutException { + testFederationSubCluster = new TestFederationSubCluster(); + testFederationSubCluster.startFederationSubCluster(2181, + "18032,18030,18031,18088,18033,SC-1,127.0.0.1:2181,capacity-scheduler", + "28032,28030,28031,28088,28033,SC-2,127.0.0.1:2181,capacity-scheduler", + "18050,18052,18089,127.0.0.1:2181"); + subClusters = Sets.newHashSet(); + subClusters.add("SC-1"); + subClusters.add("SC-2"); + } + + @AfterClass + public static void shutDown() throws Exception { + testFederationSubCluster.stop(); + } + + @Test + public void testGetClusterInfo() throws InterruptedException, IOException { + FederationClusterInfo federationClusterInfo = + TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS, RM_WEB_SERVICE_PATH, + FederationClusterInfo.class, null, null); + List clusterInfos = federationClusterInfo.getList(); + assertNotNull(clusterInfos); + assertEquals(2, clusterInfos.size()); + for (ClusterInfo clusterInfo : clusterInfos) { + assertNotNull(clusterInfo); + assertTrue(subClusters.contains(clusterInfo.getSubClusterId())); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/fair/TestYarnFederationWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/fair/TestYarnFederationWithFairScheduler.java new file mode 100644 index 0000000000000..ce27d5a3fc7e6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/fair/TestYarnFederationWithFairScheduler.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.subcluster.fair; + +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; +import org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster; +import org.apache.hadoop.yarn.server.router.webapp.dao.FederationClusterInfo; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestYarnFederationWithFairScheduler { + private static TestFederationSubCluster testFederationSubCluster; + private static Set subClusters; + private static final String ROUTER_WEB_ADDRESS = "http://localhost:28089"; + + @BeforeClass + public static void setUp() + throws IOException, InterruptedException, YarnException, TimeoutException { + testFederationSubCluster = new TestFederationSubCluster(); + testFederationSubCluster.startFederationSubCluster(2182, + "38032,38030,38031,38088,38033,SC-1,127.0.0.1:2182,fair-scheduler", + "48032,48030,48031,48088,48033,SC-2,127.0.0.1:2182,fair-scheduler", + "28050,28052,28089,127.0.0.1:2182"); + subClusters = Sets.newHashSet(); + subClusters.add("SC-1"); + subClusters.add("SC-2"); + } + + @AfterClass + public static void shutDown() throws Exception { + testFederationSubCluster.stop(); + } + + @Test + public void testGetClusterInfo() throws InterruptedException, IOException { + FederationClusterInfo federationClusterInfo = + TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS, RM_WEB_SERVICE_PATH, + FederationClusterInfo.class, null, null); + List clusterInfos = federationClusterInfo.getList(); + assertNotNull(clusterInfos); + assertEquals(2, clusterInfos.size()); + for (ClusterInfo clusterInfo : clusterInfos) { + assertNotNull(clusterInfo); + assertTrue(subClusters.contains(clusterInfo.getSubClusterId())); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java index 180cb5536b6f9..3760b96c241d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java @@ -27,7 +27,7 @@ */ public class JavaProcess { - private Process process = null; + private Process process; public JavaProcess(Class clazz, File output) throws IOException, InterruptedException { @@ -35,7 +35,7 @@ public JavaProcess(Class clazz, File output) } public JavaProcess(Class clazz, List addClassPaths, File output) - throws IOException, InterruptedException { + throws IOException { String javaHome = System.getProperty("java.home"); String javaBin = javaHome + File.separator + "bin" + File.separator + "java"; @@ -55,6 +55,26 @@ public JavaProcess(Class clazz, List addClassPaths, File output) process = builder.start(); } + public JavaProcess(Class clazz, List addClassPaths, File output, String param) + throws IOException { + String javaHome = System.getProperty("java.home"); + String javaBin = javaHome + File.separator + "bin" + File.separator + "java"; + String classpath = System.getProperty("java.class.path"); + classpath = classpath.concat("./src/test/resources"); + if (addClassPaths != null) { + for (String addClasspath : addClassPaths) { + classpath = classpath.concat(File.pathSeparatorChar + addClasspath); + } + } + String className = clazz.getCanonicalName(); + ProcessBuilder builder = + new ProcessBuilder(javaBin, "-cp", classpath, className, param); + builder.redirectInput(ProcessBuilder.Redirect.INHERIT); + builder.redirectOutput(output); + builder.redirectError(output); + process = builder.start(); + } + public void stop() throws InterruptedException { if (process != null) { process.destroy(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java index 7eb17ef6cd2eb..40c9c76fc1cb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java @@ -79,7 +79,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; @@ -133,7 +132,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.util.function.Supplier; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientHandlerException; @@ -178,29 +176,26 @@ public class TestRouterWebServicesREST { /** * Wait until the webservice is up and running. */ - private static void waitWebAppRunning( + public static void waitWebAppRunning( final String address, final String path) { try { final Client clientToRouter = Client.create(); final WebResource toRouter = clientToRouter .resource(address) .path(path); - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - try { - ClientResponse response = toRouter - .accept(APPLICATION_JSON) - .get(ClientResponse.class); - if (response.getStatus() == SC_OK) { - // process is up and running - return true; - } - } catch (ClientHandlerException e) { - // process is not up and running + GenericTestUtils.waitFor(() -> { + try { + ClientResponse response = toRouter + .accept(APPLICATION_JSON) + .get(ClientResponse.class); + if (response.getStatus() == SC_OK) { + // process is up and running + return true; } - return false; + } catch (ClientHandlerException e) { + // process is not up and running } + return false; }, 1000, 20 * 1000); } catch (Exception e) { fail("Web app not running"); @@ -278,19 +273,16 @@ private static List performGetCalls(final String path, } return UserGroupInformation.createRemoteUser(userName) - .doAs(new PrivilegedExceptionAction>() { - @Override - public List run() throws Exception { - ClientResponse response = - toRouterBuilder.get(ClientResponse.class); - ClientResponse response2 = toRMBuilder.get(ClientResponse.class); - assertEquals(SC_OK, response.getStatus()); - assertEquals(SC_OK, response2.getStatus()); - List responses = new ArrayList<>(); - responses.add(response.getEntity(returnType)); - responses.add(response2.getEntity(returnType)); - return responses; - } + .doAs((PrivilegedExceptionAction>) () -> { + ClientResponse response = + toRouterBuilder.get(ClientResponse.class); + ClientResponse response2 = toRMBuilder.get(ClientResponse.class); + assertEquals(SC_OK, response.getStatus()); + assertEquals(SC_OK, response2.getStatus()); + List responses = new ArrayList<>(); + responses.add(response.getEntity(returnType)); + responses.add(response2.getEntity(returnType)); + return responses; }); } @@ -302,45 +294,42 @@ private static ClientResponse performCall(final String webAddress, final HTTPMethods method) throws IOException, InterruptedException { return UserGroupInformation.createRemoteUser(userName) - .doAs(new PrivilegedExceptionAction() { - @Override - public ClientResponse run() throws Exception { - Client clientToRouter = Client.create(); - WebResource toRouter = clientToRouter - .resource(routerAddress) - .path(webAddress); - - WebResource toRouterWR = toRouter; - if (queryKey != null && queryValue != null) { - toRouterWR = toRouterWR.queryParam(queryKey, queryValue); - } - - Builder builder = null; - if (context != null) { - builder = toRouterWR.entity(context, APPLICATION_JSON); - builder = builder.accept(APPLICATION_JSON); - } else { - builder = toRouter.accept(APPLICATION_JSON); - } - - ClientResponse response = null; - - switch (method) { - case DELETE: - response = builder.delete(ClientResponse.class); - break; - case POST: - response = builder.post(ClientResponse.class); - break; - case PUT: - response = builder.put(ClientResponse.class); - break; - default: - break; - } - - return response; + .doAs((PrivilegedExceptionAction) () -> { + Client clientToRouter = Client.create(); + WebResource toRouter = clientToRouter + .resource(routerAddress) + .path(webAddress); + + WebResource toRouterWR = toRouter; + if (queryKey != null && queryValue != null) { + toRouterWR = toRouterWR.queryParam(queryKey, queryValue); + } + + Builder builder; + if (context != null) { + builder = toRouterWR.entity(context, APPLICATION_JSON); + builder = builder.accept(APPLICATION_JSON); + } else { + builder = toRouter.accept(APPLICATION_JSON); + } + + ClientResponse response = null; + + switch (method) { + case DELETE: + response = builder.delete(ClientResponse.class); + break; + case POST: + response = builder.post(ClientResponse.class); + break; + case PUT: + response = builder.put(ClientResponse.class); + break; + default: + break; } + + return response; }); } @@ -1353,17 +1342,14 @@ public void testGetAppsMultiThread() throws Exception { testAppsXML(); // Wait at most 10 seconds until we see all the applications - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - try { - // Check if we have the 2 apps we submitted - return getNumApps() == iniNumApps + 2; - } catch (Exception e) { - fail(); - } - return false; + GenericTestUtils.waitFor(() -> { + try { + // Check if we have the 2 apps we submitted + return getNumApps() == iniNumApps + 2; + } catch (Exception e) { + fail(); } + return false; }, 100, 10 * 1000); // Multithreaded getApps() @@ -1375,12 +1361,9 @@ public Boolean get() { try { // Submit a bunch of operations concurrently for (int i = 0; i < NUM_THREADS_TESTS; i++) { - svc.submit(new Callable() { - @Override - public Void call() throws Exception { - assertEquals(iniNumApps + 2, getNumApps()); - return null; - } + svc.submit(() -> { + assertEquals(iniNumApps + 2, getNumApps()); + return null; }); } } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/fair-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/fair-scheduler.xml new file mode 100644 index 0000000000000..cc00c45c97d39 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/fair-scheduler.xml @@ -0,0 +1,43 @@ + + + + + + + + 1.0 + + 0.33 + 8192 mb, 4 vcores + 16384 mb, 8 vcores + + + 0.33 + 8192 mb, 4 vcores + 16384 mb, 8 vcores + + + 0.34 + 8192 mb, 4 vcores + 16384 mb, 8 vcores + + + 5 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 026495fa202c4..6472a21f961a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -145,6 +145,7 @@ public class MiniYARNCluster extends CompositeService { private boolean useFixedPorts; private boolean useRpc = false; private int failoverTimeout; + private boolean isNMWebEnabled = true; private ConcurrentMap appMasters = new ConcurrentHashMap(16, 0.75f, 2); @@ -251,6 +252,21 @@ public MiniYARNCluster(String testName, int numNodeManagers, this(testName, 1, numNodeManagers, numLocalDirs, numLogDirs); } + /** + * Constructor of MiniYARNCluster. + * + * @param testName name of the test + * @param numNodeManagers the number of node managers in the cluster + * @param numLocalDirs the number of nm-local-dirs per nodemanager + * @param numLogDirs the number of nm-log-dirs per nodemanager + * @param nMWebEnabled Whether to enable the WebNM page + */ + public MiniYARNCluster(String testName, int numNodeManagers, + int numLocalDirs, int numLogDirs, boolean nMWebEnabled) { + this(testName, 1, numNodeManagers, numLocalDirs, numLogDirs); + isNMWebEnabled = nMWebEnabled; + } + @Override public void serviceInit(Configuration conf) throws Exception { useFixedPorts = conf.getBoolean( @@ -619,6 +635,9 @@ private String prepareDirs(String dirType, int numDirs) { } protected synchronized void serviceStart() throws Exception { + if (!isNMWebEnabled) { + nodeManagers[index].disableWebServer(); + } nodeManagers[index].start(); if (nodeManagers[index].getServiceState() != STATE.STARTED) { // NM could have failed. From a6c2526c6c7ccfc4483f58695aedf0cb892b4c4d Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sun, 3 Sep 2023 20:08:40 +0800 Subject: [PATCH 007/155] YARN-11435. [Router] FederationStateStoreFacade is not reinitialized with Router conf. (#5967) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- ...TestFederationRMFailoverProxyProvider.java | 4 +- .../FederationRMFailoverProxyProvider.java | 2 +- .../utils/FederationStateStoreFacade.java | 52 +++++++++++++++++-- .../federation/cache/TestFederationCache.java | 3 +- .../policies/BaseFederationPoliciesTest.java | 9 ++-- ...nPolicyInitializationContextValidator.java | 4 +- .../policies/TestRouterPolicyFacade.java | 3 +- .../manager/BasePolicyManagerTest.java | 4 +- .../utils/FederationPoliciesTestUtil.java | 21 ++++++-- .../utils/TestFederationStateStoreFacade.java | 5 +- .../GlobalPolicyGenerator.java | 2 +- .../TestGPGPolicyFacade.java | 4 +- .../policygenerator/TestPolicyGenerator.java | 6 +-- .../AbstractGlobalPolicyGeneratorTest.java | 2 +- .../TestSubClusterCleaner.java | 2 +- .../amrmproxy/AMRMProxyService.java | 2 +- .../amrmproxy/FederationInterceptor.java | 2 +- .../amrmproxy/TestFederationInterceptor.java | 2 +- .../TestFederationInterceptorSecure.java | 2 +- .../router/cleaner/SubClusterCleaner.java | 2 +- .../clientrm/FederationClientInterceptor.java | 2 +- .../clientrm/RouterClientRMService.java | 2 +- .../rmadmin/FederationRMAdminInterceptor.java | 2 +- .../router/rmadmin/RMAdminProtocolMethod.java | 2 +- .../RouterDelegationTokenSecretManager.java | 6 ++- .../yarn/server/router/webapp/AboutBlock.java | 2 +- .../yarn/server/router/webapp/AppsBlock.java | 2 +- .../webapp/FederationInterceptorREST.java | 2 +- .../server/router/webapp/NodeLabelsBlock.java | 3 +- .../yarn/server/router/webapp/NodesBlock.java | 3 +- .../server/router/webapp/RouterBlock.java | 2 +- .../router/cleaner/TestSubClusterCleaner.java | 2 +- .../TestFederationClientInterceptor.java | 2 +- .../TestFederationClientInterceptorRetry.java | 2 +- .../TestableFederationClientInterceptor.java | 2 +- .../TestFederationRMAdminInterceptor.java | 4 +- .../secure/AbstractSecureRouterTest.java | 2 +- .../yarn/server/router/webapp/MockRouter.java | 2 +- .../webapp/TestFederationInterceptorREST.java | 5 +- .../TestFederationInterceptorRESTRetry.java | 2 +- .../router/webapp/TestRouterWebAppProxy.java | 2 +- .../server/webproxy/FedAppReportFetcher.java | 2 +- .../webproxy/TestFedAppReportFetcher.java | 2 +- .../webproxy/TestWebAppProxyServletFed.java | 2 +- 44 files changed, 129 insertions(+), 63 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java index 74cf5152d5a88..11a93076dea35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java @@ -77,7 +77,7 @@ public void setUp() throws IOException, YarnException { stateStore = spy(new MemoryFederationStateStore()); stateStore.init(conf); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + FederationStateStoreFacade.getInstance(conf).reinitialize(stateStore, conf); verify(stateStore, times(0)) .getSubClusters(any(GetSubClustersInfoRequest.class)); } @@ -180,7 +180,7 @@ public void run() { .getSubClusters(any(GetSubClustersInfoRequest.class)); // Force flush cache, so that it will pick up the new RM address - FederationStateStoreFacade.getInstance().getSubCluster(subClusterId, + FederationStateStoreFacade.getInstance(conf).getSubCluster(subClusterId, true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java index e0912c8c811b8..b210f9882a434 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java @@ -76,7 +76,7 @@ public void init(Configuration configuration, RMProxy proxy, String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID); Preconditions.checkNotNull(clusterId, "Missing RM ClusterId"); this.subClusterId = SubClusterId.newInstance(clusterId); - this.facade = FederationStateStoreFacade.getInstance(); + this.facade = FederationStateStoreFacade.getInstance(configuration); if (configuration instanceof YarnConfiguration) { this.conf = (YarnConfiguration) configuration; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index d1b08ecaed5bc..26136b11de6f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -105,8 +105,7 @@ public final class FederationStateStoreFacade { private static final Logger LOG = LoggerFactory.getLogger(FederationStateStoreFacade.class); - private static final FederationStateStoreFacade FACADE = - new FederationStateStoreFacade(); + private static volatile FederationStateStoreFacade facade; private static Random rand = new Random(System.currentTimeMillis()); @@ -115,8 +114,8 @@ public final class FederationStateStoreFacade { private SubClusterResolver subclusterResolver; private FederationCache federationCache; - private FederationStateStoreFacade() { - initializeFacadeInternal(new Configuration()); + private FederationStateStoreFacade(Configuration conf) { + initializeFacadeInternal(conf); } private void initializeFacadeInternal(Configuration config) { @@ -199,7 +198,50 @@ public static RetryPolicy createRetryPolicy(Configuration conf) { * @return the singleton {@link FederationStateStoreFacade} instance */ public static FederationStateStoreFacade getInstance() { - return FACADE; + return getInstanceInternal(new Configuration()); + } + + /** + * Returns the singleton instance of the FederationStateStoreFacade object. + * + * @param conf configuration. + * @return the singleton {@link FederationStateStoreFacade} instance + */ + public static FederationStateStoreFacade getInstance(Configuration conf) { + return getInstanceInternal(conf); + } + + /** + * Returns the singleton instance of the FederationStateStoreFacade object. + * + * @param conf configuration. + * @return the singleton {@link FederationStateStoreFacade} instance + */ + private static FederationStateStoreFacade getInstanceInternal(Configuration conf){ + if (facade != null) { + return facade; + } + generateStateStoreFacade(conf); + return facade; + } + + /** + * Generate the singleton instance of the FederationStateStoreFacade object. + * + * @param conf configuration. + */ + private static void generateStateStoreFacade(Configuration conf){ + if (facade == null) { + synchronized (FederationStateStoreFacade.class) { + if (facade == null) { + Configuration yarnConf = new Configuration(); + if (conf != null) { + yarnConf = conf; + } + facade = new FederationStateStoreFacade(yarnConf); + } + } + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java index 0e77bcfc84504..8e0f15802bcc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/cache/TestFederationCache.java @@ -60,13 +60,14 @@ public static Collection getParameters() { private Configuration conf; private FederationStateStore stateStore; private FederationStateStoreTestUtil stateStoreTestUtil; - private FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + private FederationStateStoreFacade facade; public TestFederationCache(Class cacheClassName) { conf = new Configuration(); conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 1); conf.setClass(YarnConfiguration.FEDERATION_FACADE_CACHE_CLASS, cacheClassName, FederationCache.class); + facade = FederationStateStoreFacade.getInstance(conf); } @Before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java index d9ebd2f1c5e92..37aa370523267 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Random; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -75,7 +76,8 @@ public void testReinitilialize() throws YarnException { .newInstance("queue1", getPolicy().getClass().getCanonicalName(), buf)); fpc.setFederationSubclusterResolver( FederationPoliciesTestUtil.initResolver()); - fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade()); + Configuration conf = new Configuration(); + fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade(conf)); getPolicy().reinitialize(fpc); } @@ -100,7 +102,8 @@ public void testReinitilializeBad3() throws YarnException { .newInstance("queue1", "WrongPolicyName", buf)); fpc.setFederationSubclusterResolver( FederationPoliciesTestUtil.initResolver()); - fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade()); + Configuration conf = new Configuration(); + fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade(conf)); getPolicy().reinitialize(fpc); } @@ -212,9 +215,9 @@ public String generateClusterMetricsInfo(int id) { public FederationStateStoreFacade getMemoryFacade() throws YarnException { // setting up a store and its facade (with caching off) - FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(); YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); + FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(conf); FederationStateStore store = new MemoryFederationStateStore(); store.init(conf); fedFacade.reinitialize(store, conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java index 611a48611eaeb..91b025f75f35f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager; @@ -45,7 +46,8 @@ public class TestFederationPolicyInitializationContextValidator { @Before public void setUp() throws Exception { - goodFacade = FederationPoliciesTestUtil.initFacade(); + Configuration conf = new Configuration(); + goodFacade = FederationPoliciesTestUtil.initFacade(conf); goodConfig = new MockPolicyManager().serializeConf(); goodSR = FederationPoliciesTestUtil.initResolver(); goodHome = SubClusterId.newInstance("homesubcluster"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java index d0e2decb2de2f..528a27c8b1cfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java @@ -62,10 +62,9 @@ public class TestRouterPolicyFacade { public void setup() throws YarnException { // setting up a store and its facade (with caching off) - FederationStateStoreFacade fedFacade = - FederationStateStoreFacade.getInstance(); YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, "0"); + FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(conf); store = new MemoryFederationStateStore(); store.init(conf); fedFacade.reinitialize(store, conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java index 55be316e8bfe3..6cc585c1bdd09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; @@ -72,13 +73,14 @@ protected static void serializeAndDeserializePolicyManager( Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception { // serializeConf it in a context + Configuration conf = new Configuration(); SubClusterPolicyConfiguration fpc = wfp.serializeConf(); fpc.setType(policyManagerType.getCanonicalName()); FederationPolicyInitializationContext context = new FederationPolicyInitializationContext(); context.setSubClusterPolicyConfiguration(fpc); context - .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade()); + .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade(conf)); context.setFederationSubclusterResolver( FederationPoliciesTestUtil.initResolver()); context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java index ad4daf945fc6e..0fd6e5cfd1bc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java @@ -123,7 +123,7 @@ public static void initializePolicyContext( fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration .newInstance("queue1", policy.getClass().getCanonicalName(), buf)); FederationStateStoreFacade facade = FederationStateStoreFacade - .getInstance(); + .getInstance(conf); FederationStateStore fss = mock(FederationStateStore.class); if (activeSubclusters == null) { @@ -242,9 +242,8 @@ public static SubClusterResolver initResolver() { public static FederationStateStoreFacade initFacade( List subClusterInfos, SubClusterPolicyConfiguration - policyConfiguration) throws YarnException { - FederationStateStoreFacade goodFacade = FederationStateStoreFacade - .getInstance(); + policyConfiguration, Configuration conf) throws YarnException { + FederationStateStoreFacade goodFacade = FederationStateStoreFacade.getInstance(conf); FederationStateStore fss = mock(FederationStateStore.class); GetSubClustersInfoResponse response = GetSubClustersInfoResponse .newInstance(subClusterInfos); @@ -276,8 +275,20 @@ public static FederationStateStoreFacade initFacade( * @throws YarnException in case the initialization is not successful. */ public static FederationStateStoreFacade initFacade() throws YarnException { + return initFacade(new Configuration()); + } + + /** + * Initialiaze a main-memory {@link FederationStateStoreFacade} used for + * testing, wiht a mock resolver. + * + * @param conf Configuration. + * @return the facade. + * @throws YarnException in case the initialization is not successful. + */ + public static FederationStateStoreFacade initFacade(Configuration conf) throws YarnException { SubClusterPolicyConfiguration policyConfiguration = SubClusterPolicyConfiguration.newInstance(null, null, null); - return initFacade(new ArrayList<>(), policyConfiguration); + return initFacade(new ArrayList<>(), policyConfiguration, conf); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index aea2d787c0568..e2192e8ae7f2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -80,8 +80,8 @@ public static Collection getParameters() { private Configuration conf; private FederationStateStore stateStore; private FederationStateStoreTestUtil stateStoreTestUtil; - private FederationStateStoreFacade facade = - FederationStateStoreFacade.getInstance(); + private FederationStateStoreFacade facade; + private Boolean isCachingEnabled; public TestFederationStateStoreFacade(Boolean isCachingEnabled) { @@ -90,6 +90,7 @@ public TestFederationStateStoreFacade(Boolean isCachingEnabled) { conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); } this.isCachingEnabled = isCachingEnabled; + facade = FederationStateStoreFacade.getInstance(conf); } @Before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index 5f89052fd89eb..80434ecd8f22e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -112,7 +112,7 @@ protected void initAndStart(Configuration conf, boolean hasToReboot) { @Override protected void serviceInit(Configuration conf) throws Exception { // Set up the context - this.gpgContext.setStateStoreFacade(FederationStateStoreFacade.getInstance()); + this.gpgContext.setStateStoreFacade(FederationStateStoreFacade.getInstance(conf)); GPGPolicyFacade gpgPolicyFacade = new GPGPolicyFacade(this.gpgContext.getStateStoreFacade(), conf); this.gpgContext.setPolicyFacade(gpgPolicyFacade); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java index d78c11fa1db86..d20d6c0485f19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java @@ -53,8 +53,7 @@ public class TestGPGPolicyFacade { private Configuration conf; private FederationStateStore stateStore; - private FederationStateStoreFacade facade = - FederationStateStoreFacade.getInstance(); + private FederationStateStoreFacade facade; private GPGPolicyFacade policyFacade; private Set subClusterIds; @@ -70,6 +69,7 @@ public TestGPGPolicyFacade() { subClusterIds.add(SubClusterId.newInstance("sc0")); subClusterIds.add(SubClusterId.newInstance("sc1")); subClusterIds.add(SubClusterId.newInstance("sc2")); + facade = FederationStateStoreFacade.getInstance(conf); } @Before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java index 5fcbea0760f2c..72e97f8a75087 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java @@ -87,8 +87,7 @@ public class TestPolicyGenerator { private Configuration conf; private FederationStateStore stateStore; - private FederationStateStoreFacade facade = - FederationStateStoreFacade.getInstance(); + private FederationStateStoreFacade facade; private List subClusterIds; private Map subClusterInfos; @@ -102,10 +101,11 @@ public class TestPolicyGenerator { public TestPolicyGenerator() { conf = new Configuration(); conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); - + facade = FederationStateStoreFacade.getInstance(conf); gpgContext = new GPGContextImpl(); gpgContext.setPolicyFacade(new GPGPolicyFacade(facade, conf)); gpgContext.setStateStoreFacade(facade); + } @Before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/AbstractGlobalPolicyGeneratorTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/AbstractGlobalPolicyGeneratorTest.java index 04286c80d63a4..03a8e8f22c3b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/AbstractGlobalPolicyGeneratorTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/secure/AbstractGlobalPolicyGeneratorTest.java @@ -136,7 +136,7 @@ public synchronized void startSecureGPG() { assertNull("GPG is already running", gpg); MemoryFederationStateStore stateStore = new MemoryFederationStateStore(); stateStore.init(conf); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + FederationStateStoreFacade.getInstance(conf).reinitialize(stateStore, conf); UserGroupInformation.setConfiguration(conf); gpg = new GlobalPolicyGenerator(); gpg.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java index 202733940d294..996aea820a8c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java @@ -63,7 +63,7 @@ public void setup() throws YarnException { stateStore = new MemoryFederationStateStore(); stateStore.init(conf); - facade = FederationStateStoreFacade.getInstance(); + facade = FederationStateStoreFacade.getInstance(conf); facade.reinitialize(stateStore, conf); gpgContext = new GPGContextImpl(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java index c0253b334d92f..efdd8d66e2821 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java @@ -145,7 +145,7 @@ protected void serviceInit(Configuration conf) throws Exception { RegistryOperations.class); addService(this.registry); } - this.federationFacade = FederationStateStoreFacade.getInstance(); + this.federationFacade = FederationStateStoreFacade.getInstance(conf); this.federationEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java index 251e5fac64b7b..e7fcb1c3785c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java @@ -335,7 +335,7 @@ public void init(AMRMProxyApplicationContext appContext) { this.lastAllocateResponse .setResponseId(AMRMClientUtils.PRE_REGISTER_RESPONSE_ID); - this.federationFacade = FederationStateStoreFacade.getInstance(); + this.federationFacade = FederationStateStoreFacade.getInstance(conf); this.subClusterResolver = this.federationFacade.getSubClusterResolver(); // AMRMProxyPolicy will be initialized in registerApplicationMaster diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java index b540e7650ebe2..c60fda5f72bfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java @@ -118,7 +118,7 @@ public void setUp() throws IOException { stateStore = new MemoryFederationStateStore(); stateStore.init(getConf()); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, + FederationStateStoreFacade.getInstance(getConf()).reinitialize(stateStore, getConf()); nmStateStore = new NMMemoryStateStoreService(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java index 0241f4e1aaee5..5b3f94dcb5989 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java @@ -144,7 +144,7 @@ public void setUp() throws IOException { stateStore = new MemoryFederationStateStore(); stateStore.init(conf); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + FederationStateStoreFacade.getInstance(conf).reinitialize(stateStore, conf); nmStateStore = new NMMemoryStateStoreService(); nmStateStore.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/cleaner/SubClusterCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/cleaner/SubClusterCleaner.java index 19d5adc2758ea..35e4d7a998eb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/cleaner/SubClusterCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/cleaner/SubClusterCleaner.java @@ -44,7 +44,7 @@ public class SubClusterCleaner implements Runnable { private long heartbeatExpirationMillis; public SubClusterCleaner(Configuration conf) { - federationFacade = FederationStateStoreFacade.getInstance(); + federationFacade = FederationStateStoreFacade.getInstance(conf); this.heartbeatExpirationMillis = conf.getTimeDuration(YarnConfiguration.ROUTER_SUBCLUSTER_EXPIRATION_TIME, YarnConfiguration.DEFAULT_ROUTER_SUBCLUSTER_EXPIRATION_TIME, TimeUnit.MILLISECONDS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 337b8fd8e45f6..63a57394174ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -212,7 +212,7 @@ public class FederationClientInterceptor public void init(String userName) { super.init(userName); - federationFacade = FederationStateStoreFacade.getInstance(); + federationFacade = FederationStateStoreFacade.getInstance(getConf()); rand = new Random(System.currentTimeMillis()); int numMinThreads = getNumMinThreads(getConf()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index dd87bee0d3116..3d45ad207cf40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -629,7 +629,7 @@ protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecret TimeUnit.MILLISECONDS); return new RouterDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, removeScanInterval); + tokenMaxLifetime, tokenRenewInterval, removeScanInterval, conf); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index 5c8726083421c..a80e31f67d509 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -130,7 +130,7 @@ public void init(String userName) { this.executorService = new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.MILLISECONDS, workQueue, threadFactory); - federationFacade = FederationStateStoreFacade.getInstance(); + federationFacade = FederationStateStoreFacade.getInstance(this.getConf()); this.conf = this.getConf(); this.adminRMProxies = new ConcurrentHashMap<>(); routerMetrics = RouterMetrics.getMetrics(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java index 1a5b038f19cb2..665545ce6eab4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java @@ -63,7 +63,7 @@ public RMAdminProtocolMethod(Class[] pTypes, Object... pParams) public Collection invokeConcurrent(FederationRMAdminInterceptor interceptor, Class clazz, String subClusterId) throws YarnException { this.rmAdminInterceptor = interceptor; - this.federationFacade = FederationStateStoreFacade.getInstance(); + this.federationFacade = FederationStateStoreFacade.getInstance(interceptor.getConf()); this.configuration = interceptor.getConf(); if (StringUtils.isNotBlank(subClusterId)) { return invoke(clazz, subClusterId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 57d2aaa4bf147..16a00808a7600 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -19,6 +19,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.security.token.delegation.RouterDelegationTokenSupport; @@ -66,13 +67,14 @@ public class RouterDelegationTokenSecretManager * @param delegationTokenRenewInterval how often the tokens must be renewed * in milliseconds * @param delegationTokenRemoverScanInterval how often the tokens are scanned + * @param conf Configuration. */ public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenMaxLifetime, long delegationTokenRenewInterval, - long delegationTokenRemoverScanInterval) { + long delegationTokenRemoverScanInterval, Configuration conf) { super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval, delegationTokenRemoverScanInterval); - this.federationFacade = FederationStateStoreFacade.getInstance(); + this.federationFacade = FederationStateStoreFacade.getInstance(conf); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java index 878ac75d1c77b..4ae43f01f24a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java @@ -63,7 +63,7 @@ protected void render(Block html) { * @param isEnabled true, federation is enabled; false, federation is not enabled. */ private void initYarnRouterBasicInformation(boolean isEnabled) { - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(router.getConfig()); RouterInfo routerInfo = new RouterInfo(router); String lastStartTime = DateFormatUtils.format(routerInfo.getStartedOn(), DATE_PATTERN); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java index 7f277ae3ae7a7..49ad710bb11de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java @@ -104,7 +104,7 @@ private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) { private AppsInfo getSubClusterAppsInfo(String subCluster, String states) { try { SubClusterId subClusterId = SubClusterId.newInstance(subCluster); - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(this.conf); SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId); if (subClusterInfo != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index f11ca9bc89c03..71268ac672d98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -188,7 +188,7 @@ public void init(String user) { super.init(user); - federationFacade = FederationStateStoreFacade.getInstance(); + federationFacade = FederationStateStoreFacade.getInstance(getConf()); final Configuration conf = this.getConf(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java index 62e2b5d8537e7..4b77164bec7f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java @@ -73,7 +73,8 @@ protected void render(Block html) { private NodeLabelsInfo getSubClusterNodeLabelsInfo(String subCluster) { try { SubClusterId subClusterId = SubClusterId.newInstance(subCluster); - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade facade = + FederationStateStoreFacade.getInstance(router.getConfig()); SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId); if (subClusterInfo != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index 7e92506e0d68b..8d0fa5322b916 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -90,7 +90,8 @@ private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) { private NodesInfo getSubClusterNodesInfo(String subCluster) { try { SubClusterId subClusterId = SubClusterId.newInstance(subCluster); - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade facade = + FederationStateStoreFacade.getInstance(this.router.getConfig()); SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId); if (subClusterInfo != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java index 31ab83daaaf10..7fd8d246a337d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java @@ -51,7 +51,7 @@ public RouterBlock(Router router, ViewContext ctx) { super(ctx); this.ctx = ctx; this.router = router; - this.facade = FederationStateStoreFacade.getInstance(); + this.facade = FederationStateStoreFacade.getInstance(router.getConfig()); this.conf = this.router.getConfig(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/cleaner/TestSubClusterCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/cleaner/TestSubClusterCleaner.java index 57d427581a36a..d24ac9827da04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/cleaner/TestSubClusterCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/cleaner/TestSubClusterCleaner.java @@ -57,7 +57,7 @@ public void setup() throws YarnException { stateStore = new MemoryFederationStateStore(); stateStore.init(conf); - facade = FederationStateStoreFacade.getInstance(); + facade = FederationStateStoreFacade.getInstance(conf); facade.reinitialize(stateStore, conf); cleaner = new SubClusterCleaner(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java index 6f7248a0866b4..82eb1da9ae489 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java @@ -191,7 +191,7 @@ public void setUp() throws IOException { stateStore = new MemoryFederationStateStore(); stateStore.init(this.getConf()); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf()); + FederationStateStoreFacade.getInstance(getConf()).reinitialize(stateStore, getConf()); stateStoreUtil = new FederationStateStoreTestUtil(stateStore); interceptor.setConf(this.getConf()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java index 2d0bc6b3507db..25ca03353e1c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java @@ -110,7 +110,7 @@ public void setUp() throws IOException { stateStore = new MemoryFederationStateStore(); stateStore.init(this.getConf()); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, + FederationStateStoreFacade.getInstance(getConf()).reinitialize(stateStore, getConf()); stateStoreUtil = new FederationStateStoreTestUtil(stateStore); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java index fa25bc4d0a5e0..8e80bf2c0adb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java @@ -245,6 +245,6 @@ public RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecretMan TimeUnit.MILLISECONDS); return new RouterDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, removeScanInterval); + tokenMaxLifetime, tokenRenewInterval, removeScanInterval, conf); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java index 6bdf6009a2e89..20266d5b54b6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java @@ -120,8 +120,8 @@ public void setUp() { // Initialize facade & stateSore stateStore = new MemoryFederationStateStore(); stateStore.init(this.getConf()); - facade = FederationStateStoreFacade.getInstance(); - facade.reinitialize(stateStore, getConf()); + facade = FederationStateStoreFacade.getInstance(this.getConf()); + facade.reinitialize(stateStore, this.getConf()); stateStoreUtil = new FederationStateStoreTestUtil(stateStore); // Initialize interceptor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java index 062d732e8738b..a074689cccdbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java @@ -183,7 +183,7 @@ public synchronized void startSecureRouter() { assertNull("Router is already running", router); MemoryFederationStateStore stateStore = new MemoryFederationStateStore(); stateStore.init(getConf()); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf()); + FederationStateStoreFacade.getInstance(getConf()).reinitialize(stateStore, getConf()); UserGroupInformation.setConfiguration(conf); router = new Router(); router.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java index e3790108c0ae5..95b9e13936fd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java @@ -50,7 +50,7 @@ public MockRouter(Configuration configuration) YarnConfiguration.DEFAULT_FEDERATION_ENABLED); if (isEnabled) { - facade = FederationStateStoreFacade.getInstance(); + facade = FederationStateStoreFacade.getInstance(configuration); initTestFederationSubCluster(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java index 5279902b58ab6..9bde2a3dcd14c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java @@ -188,7 +188,7 @@ public void setUp() throws YarnException, IOException { stateStore = new MemoryFederationStateStore(); stateStore.init(this.getConf()); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, + FederationStateStoreFacade.getInstance(this.getConf()).reinitialize(stateStore, this.getConf()); stateStoreUtil = new FederationStateStoreTestUtil(stateStore); @@ -215,7 +215,8 @@ public void setUp() throws YarnException, IOException { RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY, RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); RouterDelegationTokenSecretManager tokenSecretManager = new RouterDelegationTokenSecretManager( - secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, removeScanInterval); + secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, removeScanInterval, + this.getConf()); tokenSecretManager.startThreads(); routerClientRMService.setRouterDTSecretManager(tokenSecretManager); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java index 790cf410bed75..762a441798027 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java @@ -93,7 +93,7 @@ public void setUp() { stateStore = new MemoryFederationStateStore(); stateStore.init(conf); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, + FederationStateStoreFacade.getInstance(conf).reinitialize(stateStore, getConf()); stateStoreUtil = new FederationStateStoreTestUtil(stateStore); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java index 244ed132410db..205b6e9236a1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebAppProxy.java @@ -153,7 +153,7 @@ public void testRouterWebAppProxyFed() throws Exception { newApplicationReport(appId4, YarnApplicationState.FINISHED, proxyAppUrl4, null))); // Initial federation store. - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(conf); facade.getStateStore() .registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo1)); facade.getStateStore() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java index 24e675160829e..6697ff01842db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/FedAppReportFetcher.java @@ -50,7 +50,7 @@ public class FedAppReportFetcher extends AppReportFetcher { public FedAppReportFetcher(Configuration conf) { super(conf); subClusters = new ConcurrentHashMap<>(); - federationFacade = FederationStateStoreFacade.getInstance(); + federationFacade = FederationStateStoreFacade.getInstance(conf); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java index e1aaf7ae693ec..d7fb7feb7979c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestFedAppReportFetcher.java @@ -77,7 +77,7 @@ private void testHelper(boolean isAHSEnabled) conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, isAHSEnabled); - FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(this.conf); FederationStateStore fss = new MemoryFederationStateStore(); fss.init(conf); fedFacade.reinitialize(fss, conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java index 8931bc577a224..6d87e46ac8dc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServletFed.java @@ -192,7 +192,7 @@ public void testWebAppProxyServletFed() throws Exception { .newInstance(newApplicationReport(appId4, YarnApplicationState.FINISHED, null))); // Initial federation store. - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(conf); facade.getStateStore() .registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo1)); facade.getStateStore() From 2831c7ce266b2132ab5a137afd624f689d070677 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Tue, 5 Sep 2023 17:34:05 +0800 Subject: [PATCH 008/155] HADOOP-18880. Add some rpc related metrics to Metrics.md (#6015) Contributed by Yanghai Hu. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../apache/hadoop/ipc/metrics/RpcMetrics.java | 2 +- .../src/site/markdown/Metrics.md | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index 5e0ea6228c27c..b9be973204d21 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -141,7 +141,7 @@ public static RpcMetrics create(Server server, Configuration conf) { MutableCounterLong rpcAuthorizationSuccesses; @Metric("Number of client backoff requests") MutableCounterLong rpcClientBackoff; - @Metric("Number of Slow RPC calls") + @Metric("Number of slow RPC calls") MutableCounterLong rpcSlowCalls; @Metric("Number of requeue calls") MutableCounterLong rpcRequeueCalls; diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index e8d3cc4552b4d..01d89b81356e4 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -74,16 +74,21 @@ The default timeunit used for RPC metrics is milliseconds (as per the below desc | `SentBytes` | Total number of sent bytes | | `RpcQueueTimeNumOps` | Total number of RPC calls | | `RpcQueueTimeAvgTime` | Average queue time in milliseconds | -| `RpcLockWaitTimeNumOps` | Total number of RPC call (same as RpcQueueTimeNumOps) | +| `RpcLockWaitTimeNumOps` | Total number of RPC calls (same as RpcQueueTimeNumOps) | | `RpcLockWaitTimeAvgTime` | Average time waiting for lock acquisition in milliseconds | | `RpcProcessingTimeNumOps` | Total number of RPC calls (same to RpcQueueTimeNumOps) | | `RpcProcessingAvgTime` | Average Processing time in milliseconds | +| `DeferredRpcProcessingTimeNumOps` | Total number of Deferred RPC calls | +| `DeferredRpcProcessingAvgTime` | Average Deferred Processing time in milliseconds | +| `RpcResponseTimeNumOps` | Total number of RPC calls (same to RpcQueueTimeNumOps) | +| `RpcResponseAvgTime` | Average Response time in milliseconds | | `RpcAuthenticationFailures` | Total number of authentication failures | | `RpcAuthenticationSuccesses` | Total number of authentication successes | | `RpcAuthorizationFailures` | Total number of authorization failures | | `RpcAuthorizationSuccesses` | Total number of authorization successes | | `RpcClientBackoff` | Total number of client backoff requests | | `RpcSlowCalls` | Total number of slow RPC calls | +| `RpcRequeueCalls` | Total number of requeue RPC calls | | `RpcCallsSuccesses` | Total number of RPC calls that are successfully processed | | `NumOpenConnections` | Current number of open connections | | `NumInProcessHandler` | Current number of handlers on working | @@ -107,6 +112,18 @@ The default timeunit used for RPC metrics is milliseconds (as per the below desc | `rpcLockWaitTime`*num*`s90thPercentileLatency` | Shows the 90th percentile of RPC lock wait time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | | `rpcLockWaitTime`*num*`s95thPercentileLatency` | Shows the 95th percentile of RPC lock wait time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | | `rpcLockWaitTime`*num*`s99thPercentileLatency` | Shows the 99th percentile of RPC lock wait time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `rpcResponseTime`*num*`sNumOps` | Shows total number of RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `rpcResponseTime`*num*`s50thPercentileLatency` | Shows the 50th percentile of RPC response time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `rpcResponseTime`*num*`s75thPercentileLatency` | Shows the 75th percentile of RPC response time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `rpcResponseTime`*num*`s90thPercentileLatency` | Shows the 90th percentile of RPC response time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `rpcResponseTime`*num*`s95thPercentileLatency` | Shows the 95th percentile of RPC response time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `rpcResponseTime`*num*`s99thPercentileLatency` | Shows the 99th percentile of RPC response time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `deferredRpcProcessingTime`*num*`sNumOps` | Shows total number of Deferred RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `deferredRpcProcessingTime`*num*`s50thPercentileLatency` | Shows the 50th percentile of Deferred RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `deferredRpcProcessingTime`*num*`s75thPercentileLatency` | Shows the 75th percentile of Deferred RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `deferredRpcProcessingTime`*num*`s90thPercentileLatency` | Shows the 90th percentile of Deferred RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `deferredRpcProcessingTime`*num*`s95thPercentileLatency` | Shows the 95th percentile of Deferred RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | +| `deferredRpcProcessingTime`*num*`s99thPercentileLatency` | Shows the 99th percentile of Deferred RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | | `TotalRequests` | Total num of requests served by the RPC server. | | `TotalRequestsPerSeconds` | Total num of requests per second served by the RPC server. | From c40a6bd46a5c2438f8dc53c9076438ef1cf1b098 Mon Sep 17 00:00:00 2001 From: Jian Zhang <38941777+KeeProMise@users.noreply.github.com> Date: Wed, 6 Sep 2023 08:48:27 +0800 Subject: [PATCH 009/155] HDFS-17166. RBF: Throwing NoNamenodesAvailableException for a long time, when failover (#5990) --- .../resolver/ActiveNamenodeResolver.java | 10 +++ .../resolver/MembershipNamenodeResolver.java | 41 ++++++++++ .../federation/router/RouterRpcClient.java | 2 + .../federation/MiniRouterDFSCluster.java | 9 +++ .../hdfs/server/federation/MockResolver.java | 5 ++ .../TestRouterClientRejectOverload.java | 81 +++++++++++++++++++ 6 files changed, 148 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java index cae1f478604d6..de89a152c2bbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java @@ -146,4 +146,14 @@ List getNamenodesForNameserviceId( * @param routerId Unique string identifier for the router. */ void setRouterId(String routerId); + + /** + * Rotate cache, make the current namenode have the lowest priority, + * to ensure that the current namenode will not be accessed first next time. + * + * @param nsId name service id + * @param namenode namenode contexts + * @param listObserversFirst Observer read case, observer NN will be ranked first + */ + void rotateCache(String nsId, FederationNamenodeContext namenode, boolean listObserversFirst); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java index db1dcdf1818f3..c0e800e0430d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java @@ -478,4 +478,45 @@ private List getRecentRegistrationForQuery( public void setRouterId(String router) { this.routerId = router; } + + /** + * Rotate cache, make the current namenode have the lowest priority, + * to ensure that the current namenode will not be accessed first next time. + * + * @param nsId name service id + * @param namenode namenode contexts + * @param listObserversFirst Observer read case, observer NN will be ranked first + */ + @Override + public void rotateCache( + String nsId, FederationNamenodeContext namenode, boolean listObserversFirst) { + cacheNS.compute(Pair.of(nsId, listObserversFirst), (ns, namenodeContexts) -> { + if (namenodeContexts == null || namenodeContexts.size() <= 1) { + return namenodeContexts; + } + FederationNamenodeContext firstNamenodeContext = namenodeContexts.get(0); + /* + * If the first nn in the cache is active, the active nn priority cannot be lowered. + * This happens when other threads have already updated the cache. + */ + if (firstNamenodeContext.getState().equals(ACTIVE)) { + return namenodeContexts; + } + /* + * If the first nn in the cache at this time is not the nn + * that needs to be lowered in priority, there is no need to rotate. + * This happens when other threads have already rotated the cache. + */ + if (firstNamenodeContext.getRpcAddress().equals(namenode.getRpcAddress())) { + List rotatedNnContexts = new ArrayList<>(namenodeContexts); + Collections.rotate(rotatedNnContexts, -1); + String firstNamenodeId = namenodeContexts.get(0).getNamenodeId(); + LOG.info("Rotate cache of pair , put namenode: {} in the " + + "first position of the cache and namenode: {} in the last position of the cache", + nsId, listObserversFirst, firstNamenodeId, namenode.getNamenodeId()); + return rotatedNnContexts; + } + return namenodeContexts; + }); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index 321d97e5dac4b..b38900c3bc264 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -599,6 +599,8 @@ public Object invokeMethod( } LOG.error("Cannot get available namenode for {} {} error: {}", nsId, rpcAddress, ioe.getMessage()); + // Rotate cache so that client can retry the next namenode in the cache + this.namenodeResolver.rotateCache(nsId, namenode, shouldUseObserver); // Throw RetriableException so that client can retry throw new RetriableException(ioe); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index 2c70395870496..bf22cf01148a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -1202,4 +1202,13 @@ public void waitClusterUp() throws IOException { throw new IOException("Cannot wait for the namenodes", e); } } + + /** + * Get cache flush interval in milliseconds. + * + * @return Cache flush interval in milliseconds. + */ + public long getCacheFlushInterval() { + return cacheFlushInterval; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java index 4aaa8e7569e88..554879856ac1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java @@ -397,6 +397,11 @@ public List getMountPoints(String path) throws IOException { public void setRouterId(String router) { } + @Override + public void rotateCache( + String nsId, FederationNamenodeContext namenode, boolean listObserversFirst) { + } + /** * Mocks the availability of default namespace. * @param b if true default namespace is unset. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java index 8d776546801ba..176ac4b078250 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.ha.HAServiceProtocol.HAServiceState.ACTIVE; +import static org.apache.hadoop.ha.HAServiceProtocol.HAServiceState.STANDBY; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.simulateSlowNamenode; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.simulateThrowExceptionRouterRpcServer; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.transitionClusterNSToStandby; @@ -42,16 +44,19 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.test.GenericTestUtils; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -357,6 +362,82 @@ public void testNoNamenodesAvailable() throws Exception{ assertEquals(originalRouter0Failures, rpcMetrics0.getProxyOpNoNamenodes()); } + /** + * When failover occurs, the router may record that the ns has no active namenode. + * Only when the router updates the cache next time can the memory status be updated, + * causing the router to report NoNamenodesAvailableException for a long time. + */ + @Test + public void testNoNamenodesAvailableLongTimeWhenNsFailover() throws Exception { + setupCluster(false, true); + transitionClusterNSToStandby(cluster); + for (RouterContext routerContext : cluster.getRouters()) { + // Manually trigger the heartbeat + Collection heartbeatServices = routerContext + .getRouter().getNamenodeHeartbeatServices(); + for (NamenodeHeartbeatService service : heartbeatServices) { + service.periodicInvoke(); + } + // Update service cache + routerContext.getRouter().getStateStore().refreshCaches(true); + } + // Record the time after the router first updated the cache + long firstLoadTime = Time.now(); + List namenodes = cluster.getNamenodes(); + + // Make sure all namenodes are in standby state + for (MiniRouterDFSCluster.NamenodeContext namenodeContext : namenodes) { + assertEquals(STANDBY.ordinal(), namenodeContext.getNamenode().getNameNodeState()); + } + + Configuration conf = cluster.getRouterClientConf(); + // Set dfs.client.failover.random.order false, to pick 1st router at first + conf.setBoolean("dfs.client.failover.random.order", false); + + DFSClient routerClient = new DFSClient(new URI("hdfs://fed"), conf); + + for (RouterContext routerContext : cluster.getRouters()) { + // Get the second namenode in the router cache and make it active + List ns0 = routerContext.getRouter() + .getNamenodeResolver() + .getNamenodesForNameserviceId("ns0", false); + + String nsId = ns0.get(1).getNamenodeId(); + cluster.switchToActive("ns0", nsId); + // Manually trigger the heartbeat, but the router does not manually load the cache + Collection heartbeatServices = routerContext + .getRouter().getNamenodeHeartbeatServices(); + for (NamenodeHeartbeatService service : heartbeatServices) { + service.periodicInvoke(); + } + assertEquals(ACTIVE.ordinal(), + cluster.getNamenode("ns0", nsId).getNamenode().getNameNodeState()); + } + + // Get router0 metrics + FederationRPCMetrics rpcMetrics0 = cluster.getRouters().get(0) + .getRouter().getRpcServer().getRPCMetrics(); + // Original failures + long originalRouter0Failures = rpcMetrics0.getProxyOpNoNamenodes(); + + /* + * At this time, the router has recorded 2 standby namenodes in memory, + * and the first accessed namenode is indeed standby, + * then an NoNamenodesAvailableException will be reported for the first access, + * and the next access will be successful. + */ + routerClient.getFileInfo("/"); + long successReadTime = Time.now(); + assertEquals(originalRouter0Failures + 1, rpcMetrics0.getProxyOpNoNamenodes()); + + /* + * access the active namenode without waiting for the router to update the cache, + * even if there are 2 standby states recorded in the router memory. + */ + assertTrue(successReadTime - firstLoadTime < cluster.getCacheFlushInterval()); + } + + @Test public void testAsyncCallerPoolMetrics() throws Exception { setupCluster(true, false); From 7c941e00b45ce1ccc02d7a5d7ec575a04c8bfb1e Mon Sep 17 00:00:00 2001 From: ZanderXu Date: Wed, 6 Sep 2023 10:27:52 +0800 Subject: [PATCH 010/155] HDFS-16933. A race in SerialNumberMap will cause wrong owner, group and XATTR. (#5430). Contributed by ZanderXu. Signed-off-by: He Xiaoqiao --- .../hdfs/server/namenode/SerialNumberMap.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java index ff116b511889e..453844ec95165 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java @@ -61,17 +61,22 @@ public int get(T t) { } Integer sn = t2i.get(t); if (sn == null) { - sn = current.getAndIncrement(); - if (sn > max) { - current.getAndDecrement(); - throw new IllegalStateException(name + ": serial number map is full"); + synchronized (this) { + sn = t2i.get(t); + if (sn == null) { + sn = current.getAndIncrement(); + if (sn > max) { + current.getAndDecrement(); + throw new IllegalStateException(name + ": serial number map is full"); + } + Integer old = t2i.putIfAbsent(t, sn); + if (old != null) { + current.getAndDecrement(); + return old; + } + i2t.put(sn, t); + } } - Integer old = t2i.putIfAbsent(t, sn); - if (old != null) { - current.getAndDecrement(); - return old; - } - i2t.put(sn, t); } return sn; } From c2c6972f251594d3ec1e92894afcb1dac6364efb Mon Sep 17 00:00:00 2001 From: Liangjun He <2005hithlj@163.com> Date: Wed, 6 Sep 2023 20:19:04 +0800 Subject: [PATCH 011/155] HDFS-17140. Revisit the BPOfferService.reportBadBlocks() method. (#5924). Contributed by Liangjun He. Reviewed-by: Shilun Fan Signed-off-by: He Xiaoqiao --- .../org/apache/hadoop/hdfs/server/datanode/BPOfferService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index fdd66cb05d378..b228fb2d5716a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -291,9 +291,8 @@ public String toString() { void reportBadBlocks(ExtendedBlock block, String storageUuid, StorageType storageType) { checkBlock(block); + ReportBadBlockAction rbbAction = new ReportBadBlockAction(block, storageUuid, storageType); for (BPServiceActor actor : bpServices) { - ReportBadBlockAction rbbAction = new ReportBadBlockAction - (block, storageUuid, storageType); actor.bpThreadEnqueue(rbbAction); } } From e1dde3bc23120160c407b4c65d5b61c8f3a593a5 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 6 Sep 2023 22:08:59 +0800 Subject: [PATCH 012/155] YARN-11537. [Federation] Router CLI Supports List SubClusterPolicyConfiguration Of Queues. (#5944) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- ...ResourceManagerAdministrationProtocol.java | 17 +- .../QueryFederationQueuePoliciesRequest.java | 132 +++++++++++ .../QueryFederationQueuePoliciesResponse.java | 149 ++++++++++++ ...ourcemanager_administration_protocol.proto | 1 + ...erver_resourcemanager_service_protos.proto | 15 ++ .../hadoop/yarn/client/cli/RouterCLI.java | 103 ++++++++- .../hadoop/yarn/client/cli/TestRouterCLI.java | 26 +++ ...gerAdministrationProtocolPBClientImpl.java | 19 ++ ...erAdministrationProtocolPBServiceImpl.java | 23 ++ ...yFederationQueuePoliciesRequestPBImpl.java | 196 ++++++++++++++++ ...FederationQueuePoliciesResponsePBImpl.java | 212 ++++++++++++++++++ .../server/MockResourceManagerFacade.java | 8 + .../server/resourcemanager/AdminService.java | 10 + .../yarn/server/router/RouterMetrics.java | 32 +++ .../DefaultRMAdminRequestInterceptor.java | 8 + .../rmadmin/FederationRMAdminInterceptor.java | 211 +++++++++++++++++ .../router/rmadmin/RouterRMAdminService.java | 9 + .../yarn/server/router/TestRouterMetrics.java | 34 +++ .../PassThroughRMAdminRequestInterceptor.java | 8 + .../TestFederationRMAdminInterceptor.java | 126 ++++++++++- 20 files changed, 1333 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesResponsePBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 1ad77e0b30ecc..916dcd50aa0d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -62,6 +62,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; @Private public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol { @@ -194,7 +196,7 @@ SaveFederationQueuePolicyResponse saveFederationQueuePolicy( /** * In YARN-Federation mode, this method provides a way to save queue policies in batches. * - * @param request BatchSaveFederationQueuePolicies Request + * @param request BatchSaveFederationQueuePolicies Request. * @return Response from batchSaveFederationQueuePolicies. * @throws YarnException exceptions from yarn servers. * @throws IOException if an IO error occurred. @@ -203,4 +205,17 @@ SaveFederationQueuePolicyResponse saveFederationQueuePolicy( @Idempotent BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies( BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException; + + /** + * In YARN-Federation mode, this method provides a way to list policies. + * + * @param request QueryFederationQueuePoliciesRequest Request. + * @return Response from listFederationQueuePolicies. + * @throws YarnException exceptions from yarn servers. + * @throws IOException if an IO error occurred. + */ + @Private + @Idempotent + QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesRequest.java new file mode 100644 index 0000000000000..7572ff495a54e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesRequest.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * Request for querying Federation Queue Policies. + * It includes several query conditions such as queue, queues, pageSize, and currentPage. + * + * queue: The specific queue name or identifier to query the Federation Queue Policy for. + * queues: A list of queue names or identifiers for which to query the Federation Queue Policies. + * pageSize: The number of policies to display per page. + * currentPage: The current page number. + */ +@Private +@Unstable +public abstract class QueryFederationQueuePoliciesRequest { + + @Private + @Unstable + public static QueryFederationQueuePoliciesRequest newInstance( + int pageSize, int currentPage, String queue, List queues) { + QueryFederationQueuePoliciesRequest request = + Records.newRecord(QueryFederationQueuePoliciesRequest.class); + request.setPageSize(pageSize); + request.setCurrentPage(currentPage); + request.setQueue(queue); + request.setQueues(queues); + return request; + } + + /** + * Sets the page size for FederationQueuePolicies pagination. + * + * @param pageSize The number of policies to display per page. + */ + @Private + @Unstable + public abstract void setPageSize(int pageSize); + + /** + * Retrieves the page size. + * + * @return The number of policies to display per page. + */ + @Public + @Unstable + public abstract int getPageSize(); + + /** + * Sets the current page in the FederationQueuePolicies pagination. + * + * @param currentPage The current page number. + */ + @Private + @Unstable + public abstract void setCurrentPage(int currentPage); + + /** + * Returns the current page number in the FederationQueuePolicies pagination. + * + * @return The current page number. + */ + @Public + @Unstable + public abstract int getCurrentPage(); + + /** + * Retrieves the queue. + * + * @return The name or identifier of the current queue. + */ + @Public + @Unstable + public abstract String getQueue(); + + /** + * Sets the queue to the specified value. + * + * We will use the fully qualified name matching for queues. + * For example, if the user inputs 'a', we will match + * queues that contain 'a' in their fully qualified names, + * such as 'root.a', 'root.b.a', and so on. + * + * @param queue queue name. + */ + @Private + @Unstable + public abstract void setQueue(String queue); + + /** + * Retrieves a list of queues. + * + * This part contains exact matches, + * which will match the queues contained in the list. + * + * @return A list of queue names or identifiers. + */ + @Public + @Unstable + public abstract List getQueues(); + + /** + * Sets the list of queues to the specified values. + * + * @param queues A list of queue names or identifiers to set. + */ + @Private + @Unstable + public abstract void setQueues(List queues); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesResponse.java new file mode 100644 index 0000000000000..b42356230b1da --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesResponse.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * This is the QueryFederationQueuePoliciesResponse, which contains the following information: + * 1. Number of policy information included, + * 2. total Page number, + * 3. pageSize Conditions passed by the user, + * 4. Result of queue weight information returned. + */ +@Private +@Unstable +public abstract class QueryFederationQueuePoliciesResponse { + + @Private + @Unstable + public static QueryFederationQueuePoliciesResponse newInstance( + int totalSize, int totalPage, int currentPage, int pageSize, + List federationQueueWeights) { + QueryFederationQueuePoliciesResponse response = + Records.newRecord(QueryFederationQueuePoliciesResponse.class); + response.setTotalSize(totalSize); + response.setTotalPage(totalPage); + response.setCurrentPage(currentPage); + response.setPageSize(pageSize); + response.setFederationQueueWeights(federationQueueWeights); + return response; + } + + @Private + @Unstable + public static QueryFederationQueuePoliciesResponse newInstance() { + QueryFederationQueuePoliciesResponse response = + Records.newRecord(QueryFederationQueuePoliciesResponse.class); + return response; + } + + /** + * Returns the total size of the query result. + * It is mainly related to the filter conditions set by the user. + * + * @return The total size of the query result. + */ + public abstract int getTotalSize(); + + /** + * Sets the total size of the federationQueueWeights. + * + * @param totalSize The total size of the query result to be set. + */ + public abstract void setTotalSize(int totalSize); + + /** + * Returns the page. + * + * @return page. + */ + @Public + @Unstable + public abstract int getTotalPage(); + + /** + * Sets the page. + * + * @param page page. + */ + @Private + @Unstable + public abstract void setTotalPage(int page); + + /** + * Returns the current page number in the FederationQueuePolicies pagination. + * + * @return The current page number. + */ + @Public + @Unstable + public abstract int getCurrentPage(); + + /** + * Sets the current page in the FederationQueuePolicies pagination. + * + * @param currentPage The current page number. + */ + @Private + @Unstable + public abstract void setCurrentPage(int currentPage); + + + /** + * Retrieves the page size. + * + * @return The number of policies to display per page. + */ + @Public + @Unstable + public abstract int getPageSize(); + + /** + * Sets the page size for FederationQueuePolicies pagination. + * + * @param pageSize The number of policies to display per page. + */ + @Private + @Unstable + public abstract void setPageSize(int pageSize); + + /** + * Get a list of FederationQueueWeight objects of different queues. + * + * @return list of FederationQueueWeight. + */ + @Public + @Unstable + public abstract List getFederationQueueWeights(); + + /** + * Sets the FederationQueueWeights, which represent the weights of different queues. + * + * @param federationQueueWeights list of FederationQueueWeight. + */ + @Private + @Unstable + public abstract void setFederationQueueWeights( + List federationQueueWeights); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index aca7a4c0b8320..fcae14128d80f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -50,4 +50,5 @@ service ResourceManagerAdministrationProtocolService { rpc deregisterSubCluster(DeregisterSubClusterRequestProto) returns (DeregisterSubClusterResponseProto); rpc saveFederationQueuePolicy(SaveFederationQueuePolicyRequestProto) returns (SaveFederationQueuePolicyResponseProto); rpc batchSaveFederationQueuePolicies(BatchSaveFederationQueuePoliciesRequestProto) returns (BatchSaveFederationQueuePoliciesResponseProto); + rpc listFederationQueuePolicies(QueryFederationQueuePoliciesRequestProto) returns (QueryFederationQueuePoliciesResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index 06e11f913b6b1..a2945f1eb1ee1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -188,6 +188,21 @@ message BatchSaveFederationQueuePoliciesResponseProto { required string message = 1; } +message QueryFederationQueuePoliciesRequestProto { + optional int32 pageSize = 1; + optional int32 currentPage = 2; + optional string queue = 3; + repeated string queues = 4; +} + +message QueryFederationQueuePoliciesResponseProto { + optional int32 totalSize = 1; + optional int32 totalPage = 2; + optional int32 currentPage = 3; + optional int32 pageSize = 4; + repeated FederationQueueWeightProto federationQueueWeights = 5; +} + ////////////////////////////////////////////////////////////////// ///////////// RM Failover related records //////////////////////// ////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 788ef8feb1ea5..0aa02c8124a3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -65,6 +67,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid; import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkSubClusterQueueWeightRatioValid; @@ -85,7 +88,8 @@ public class RouterCLI extends Configured implements Tool { // Command2: policy .put("-policy", new UsageInfo( "[-s|--save [queue;router weight;amrm weight;headroomalpha]] " + - "[-bs|--batch-save [--format xml] [-f|--input-file fileName]]", + "[-bs|--batch-save [--format xml] [-f|--input-file fileName]]" + + "[-l|--list [--pageSize][--currentPage][--queue][--queues]]", "We provide a set of commands for Policy:" + " Include list policies, save policies, batch save policies. " + " (Note: The policy type will be directly read from the" + @@ -122,6 +126,12 @@ public class RouterCLI extends Configured implements Tool { private static final String OPTION_FORMAT = "format"; private static final String OPTION_FILE = "f"; private static final String OPTION_INPUT_FILE = "input-file"; + private static final String OPTION_L = "l"; + private static final String OPTION_LIST = "list"; + private static final String OPTION_PAGE_SIZE = "pageSize"; + private static final String OPTION_CURRENT_PAGE = "currentPage"; + private static final String OPTION_QUEUE = "queue"; + private static final String OPTION_QUEUES = "queues"; private static final String CMD_POLICY = "-policy"; private static final String FORMAT_XML = "xml"; @@ -134,6 +144,12 @@ public class RouterCLI extends Configured implements Tool { private static final String XML_TAG_QUEUE = "queue"; private static final String XML_TAG_NAME = "name"; + private static final String LIST_POLICIES_TITLE = + "Yarn Federation Queue Policies"; + // Columns information + private static final List LIST_POLICIES_HEADER = Arrays.asList( + "Queue Name", "AMRM Weight", "Router Weight"); + public RouterCLI() { super(); } @@ -191,7 +207,8 @@ private static void printHelp() { .append("routeradmin\n") .append(" [-deregisterSubCluster [-sc|--subClusterId [subCluster Id]]\n") .append(" [-policy [-s|--save [queue;router weight;amrm weight;headroomalpha] " + - "[-bs|--batch-save [--format xml,json] [-f|--input-file fileName]]]\n") + "[-bs|--batch-save [--format xml,json] [-f|--input-file fileName]]] " + + "[-l|--list [--pageSize][--currentPage][--queue][--queues]]\n") .append(" [-help [cmd]]").append("\n"); StringBuilder helpBuilder = new StringBuilder(); System.out.println(summary); @@ -346,11 +363,26 @@ private int handlePolicy(String[] args) Option fileOpt = new Option("f", "input-file", true, "The location of the input configuration file. "); formatOpt.setOptionalArg(true); - + Option listOpt = new Option(OPTION_L, OPTION_LIST, false, + "We can display the configured queue strategy according to the parameters."); + Option pageSizeOpt = new Option(null, "pageSize", true, + "The number of policies displayed per page."); + Option currentPageOpt = new Option(null, "currentPage", true, + "Since users may configure numerous policies, we will choose to display them in pages. " + + "This parameter represents the page number to be displayed."); + Option queueOpt = new Option(null, "queue", true, + "the queue we need to filter. example: root.a"); + Option queuesOpt = new Option(null, "queues", true, + "list of queues to filter. example: root.a,root.b,root.c"); opts.addOption(saveOpt); opts.addOption(batchSaveOpt); opts.addOption(formatOpt); opts.addOption(fileOpt); + opts.addOption(listOpt); + opts.addOption(pageSizeOpt); + opts.addOption(currentPageOpt); + opts.addOption(queueOpt); + opts.addOption(queuesOpt); // Parse command line arguments. CommandLine cliParser; @@ -396,6 +428,31 @@ private int handlePolicy(String[] args) // Batch SavePolicies. return handBatchSavePolicies(format, filePath); + } else if(cliParser.hasOption(OPTION_L) || cliParser.hasOption(OPTION_LIST)) { + + int pageSize = 10; + if (cliParser.hasOption(OPTION_PAGE_SIZE)) { + pageSize = Integer.parseInt(cliParser.getOptionValue(OPTION_PAGE_SIZE)); + } + + int currentPage = 1; + if (cliParser.hasOption(OPTION_CURRENT_PAGE)) { + currentPage = Integer.parseInt(cliParser.getOptionValue(OPTION_CURRENT_PAGE)); + } + + String queue = null; + if (cliParser.hasOption(OPTION_QUEUE)) { + queue = cliParser.getOptionValue(OPTION_QUEUE); + } + + List queues = null; + if (cliParser.hasOption(OPTION_QUEUES)) { + String tmpQueues = cliParser.getOptionValue(OPTION_QUEUES); + queues = Arrays.stream(tmpQueues.split(",")).collect(Collectors.toList()); + } + + // List Policies. + return handListPolicies(pageSize, currentPage, queue, queues); } else { // printUsage printUsage(args[0]); @@ -618,6 +675,46 @@ private String parsePolicyWeightsNode(Element queueElement, String weightType) { return StringUtils.join(amRmPolicyWeights, ","); } + /** + * Handles the list federation policies based on the specified parameters. + * + * @param pageSize Records displayed per page. + * @param currentPage The current page number. + * @param queue The name of the queue to be filtered. + * @param queues list of queues to filter. + * @return 0, success; 1, failed. + */ + protected int handListPolicies(int pageSize, int currentPage, String queue, List queues) { + LOG.info("List Federation Policies, pageSize = {}, currentPage = {}, queue = {}, queues = {}", + pageSize, currentPage, queue, queues); + try { + PrintWriter writer = new PrintWriter(new OutputStreamWriter( + System.out, Charset.forName(StandardCharsets.UTF_8.name()))); + QueryFederationQueuePoliciesRequest request = + QueryFederationQueuePoliciesRequest.newInstance(pageSize, currentPage, queue, queues); + ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); + QueryFederationQueuePoliciesResponse response = + adminProtocol.listFederationQueuePolicies(request); + System.out.println("TotalPage = " + response.getTotalPage()); + + FormattingCLIUtils formattingCLIUtils = new FormattingCLIUtils(LIST_POLICIES_TITLE) + .addHeaders(LIST_POLICIES_HEADER); + List federationQueueWeights = response.getFederationQueueWeights(); + federationQueueWeights.forEach(federationQueueWeight -> { + String queueName = federationQueueWeight.getQueue(); + String amrmWeight = federationQueueWeight.getAmrmWeight(); + String routerWeight = federationQueueWeight.getRouterWeight(); + formattingCLIUtils.addLine(queueName, amrmWeight, routerWeight); + }); + writer.print(formattingCLIUtils.render()); + writer.flush(); + return EXIT_SUCCESS; + } catch (YarnException | IOException e) { + LOG.error("handleSavePolicy error.", e); + return EXIT_ERROR; + } + } + @Override public int run(String[] args) throws Exception { YarnConfiguration yarnConf = getConf() == null ? diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index 476ba75263dae..6ed83826dfa58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.junit.Before; import org.junit.Test; import org.mockito.stubbing.Answer; @@ -78,6 +80,19 @@ public void setup() throws Exception { return SaveFederationQueuePolicyResponse.newInstance("success"); }); + when(admin.listFederationQueuePolicies(any(QueryFederationQueuePoliciesRequest.class))) + .thenAnswer((Answer) invocationOnMock -> { + // Step1. parse request. + Object obj = invocationOnMock.getArgument(0); + QueryFederationQueuePoliciesRequest request = (QueryFederationQueuePoliciesRequest) obj; + String queue = request.getQueue(); + List weights = new ArrayList<>(); + FederationQueueWeight weight = FederationQueueWeight.newInstance( + "SC-1:0.8,SC-2:0.2", "SC-1:0.6,SC-2:0.4", "1", queue, "test"); + weights.add(weight); + return QueryFederationQueuePoliciesResponse.newInstance(1, 1, 1, 10, weights); + }); + Configuration config = new Configuration(); config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); @@ -240,4 +255,15 @@ public void testParsePoliciesByXml() throws Exception { assertEquals("SC-1:0.8,SC-2:0.2", queueWeight2.getAmrmWeight()); assertEquals("SC-1:0.6,SC-2:0.4", queueWeight2.getRouterWeight()); } + + @Test + public void testListPolicies() throws Exception { + PrintStream oldOutPrintStream = System.out; + ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); + System.setOut(new PrintStream(dataOut)); + oldOutPrintStream.println(dataOut); + + String[] args = {"-policy", "-l", "--queue", "root.a"}; + assertEquals(0, rmAdminCLI.run(args)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index 7107f8014e46f..36aecf141ad7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.DeregisterSubClusterRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SaveFederationQueuePolicyRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.BatchSaveFederationQueuePoliciesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; @@ -84,6 +85,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -118,6 +121,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesResponsePBImpl; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -400,4 +405,18 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies return null; } } + + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + QueryFederationQueuePoliciesRequestProto requestProto = + ((QueryFederationQueuePoliciesRequestPBImpl) request).getProto(); + try { + return new QueryFederationQueuePoliciesResponsePBImpl( + proxy.listFederationQueuePolicies(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index 01feef9d9feae..0eab80b9ad770 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -32,6 +32,8 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshClusterMaxPriorityRequestProto; @@ -83,6 +85,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl; @@ -117,6 +121,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SaveFederationQueuePolicyResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.BatchSaveFederationQueuePoliciesResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.QueryFederationQueuePoliciesResponsePBImpl; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -422,4 +428,21 @@ public BatchSaveFederationQueuePoliciesResponseProto batchSaveFederationQueuePol throw new ServiceException(e); } } + + @Override + public QueryFederationQueuePoliciesResponseProto listFederationQueuePolicies( + RpcController controller, QueryFederationQueuePoliciesRequestProto proto) + throws ServiceException { + QueryFederationQueuePoliciesRequest request = + new QueryFederationQueuePoliciesRequestPBImpl(proto); + try { + QueryFederationQueuePoliciesResponse response = + real.listFederationQueuePolicies(request); + return ((QueryFederationQueuePoliciesResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesRequestPBImpl.java new file mode 100644 index 0000000000000..b56ec663ce582 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesRequestPBImpl.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesRequestProto; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; + +import java.util.ArrayList; +import java.util.List; + +public class QueryFederationQueuePoliciesRequestPBImpl + extends QueryFederationQueuePoliciesRequest { + + private QueryFederationQueuePoliciesRequestProto proto = + QueryFederationQueuePoliciesRequestProto.getDefaultInstance(); + private QueryFederationQueuePoliciesRequestProto.Builder builder = null; + private boolean viaProto = false; + private List queues = null; + + public QueryFederationQueuePoliciesRequestPBImpl() { + builder = QueryFederationQueuePoliciesRequestProto.newBuilder(); + } + + public QueryFederationQueuePoliciesRequestPBImpl( + QueryFederationQueuePoliciesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public void setPageSize(int pageSize) { + maybeInitBuilder(); + Preconditions.checkNotNull(builder); + builder.setPageSize(pageSize); + } + + @Override + public int getPageSize() { + QueryFederationQueuePoliciesRequestProtoOrBuilder p = viaProto ? proto : builder; + boolean hasPageSize = p.hasPageSize(); + if (hasPageSize) { + return p.getPageSize(); + } + return 0; + } + + @Override + public void setCurrentPage(int currentPage) { + maybeInitBuilder(); + Preconditions.checkNotNull(builder); + builder.setCurrentPage(currentPage); + } + + @Override + public int getCurrentPage() { + QueryFederationQueuePoliciesRequestProtoOrBuilder p = viaProto ? proto : builder; + boolean hasCurrentPage = p.hasCurrentPage(); + if (hasCurrentPage) { + return p.getCurrentPage(); + } + return 0; + } + + @Override + public String getQueue() { + QueryFederationQueuePoliciesRequestProtoOrBuilder p = viaProto ? proto : builder; + boolean hasQueue = p.hasQueue(); + if (hasQueue) { + return p.getQueue(); + } + return null; + } + + @Override + public void setQueue(String queue) { + maybeInitBuilder(); + if (queue == null) { + builder.clearQueue(); + return; + } + builder.setQueue(queue); + } + + @Override + public List getQueues() { + if (this.queues != null) { + return this.queues; + } + initQueues(); + return this.queues; + } + + @Override + public void setQueues(List pQueues) { + if (pQueues == null || pQueues.isEmpty()) { + maybeInitBuilder(); + if (this.queues != null) { + this.queues.clear(); + } + return; + } + if (this.queues == null) { + this.queues = new ArrayList<>(); + } + this.queues.clear(); + this.queues.addAll(pQueues); + } + + private void initQueues() { + if (this.queues != null) { + return; + } + QueryFederationQueuePoliciesRequestProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getQueuesList(); + this.queues = new ArrayList<>(); + this.queues.addAll(list); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + public QueryFederationQueuePoliciesRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.queues != null) { + addQueuesToProto(); + } + } + + private void addQueuesToProto() { + maybeInitBuilder(); + builder.clearQueue(); + if (this.queues == null) { + return; + } + builder.addAllQueues(this.queues); + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = QueryFederationQueuePoliciesRequestProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesResponsePBImpl.java new file mode 100644 index 0000000000000..8cfae749d3ceb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/QueryFederationQueuePoliciesResponsePBImpl.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.yarn.proto.YarnProtos.FederationQueueWeightProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.QueryFederationQueuePoliciesResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; + +import java.util.ArrayList; +import java.util.List; + +public class QueryFederationQueuePoliciesResponsePBImpl + extends QueryFederationQueuePoliciesResponse { + + private QueryFederationQueuePoliciesResponseProto proto = + QueryFederationQueuePoliciesResponseProto.getDefaultInstance(); + private QueryFederationQueuePoliciesResponseProto.Builder builder = null; + private boolean viaProto = false; + private List federationQueueWeights = null; + + public QueryFederationQueuePoliciesResponsePBImpl() { + builder = QueryFederationQueuePoliciesResponseProto.newBuilder(); + } + + public QueryFederationQueuePoliciesResponsePBImpl( + QueryFederationQueuePoliciesResponseProto proto) { + this.proto = proto; + this.viaProto = true; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + if (this.federationQueueWeights != null) { + for (FederationQueueWeight federationQueueWeight : federationQueueWeights) { + FederationQueueWeightPBImpl federationQueueWeightPBImpl = + (FederationQueueWeightPBImpl) federationQueueWeight; + builder.addFederationQueueWeights(federationQueueWeightPBImpl.getProto()); + } + } + proto = builder.build(); + viaProto = true; + } + + public QueryFederationQueuePoliciesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = QueryFederationQueuePoliciesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public int getTotalSize() { + QueryFederationQueuePoliciesResponseProtoOrBuilder p = viaProto ? proto : builder; + boolean hasTotalSize = p.hasTotalSize(); + if (hasTotalSize) { + return p.getTotalSize(); + } + return 0; + } + + @Override + public void setTotalSize(int totalSize) { + maybeInitBuilder(); + Preconditions.checkNotNull(builder); + builder.setTotalSize(totalSize); + } + + @Override + public int getTotalPage() { + QueryFederationQueuePoliciesResponseProtoOrBuilder p = viaProto ? proto : builder; + boolean hasTotalPage = p.hasTotalPage(); + if (hasTotalPage) { + return p.getTotalPage(); + } + return 0; + } + + @Override + public void setTotalPage(int totalPage) { + maybeInitBuilder(); + Preconditions.checkNotNull(builder); + builder.setTotalPage(totalPage); + } + + @Override + public int getCurrentPage() { + QueryFederationQueuePoliciesResponseProtoOrBuilder p = viaProto ? proto : builder; + boolean hasCurrentPage = p.hasCurrentPage(); + if (hasCurrentPage) { + return p.getCurrentPage(); + } + return 0; + } + + @Override + public void setCurrentPage(int currentPage) { + maybeInitBuilder(); + Preconditions.checkNotNull(builder); + builder.setCurrentPage(currentPage); + } + + @Override + public int getPageSize() { + QueryFederationQueuePoliciesResponseProtoOrBuilder p = viaProto ? proto : builder; + boolean hasPageSize = p.hasPageSize(); + if (hasPageSize) { + return p.getPageSize(); + } + return 0; + } + + @Override + public void setPageSize(int pageSize) { + Preconditions.checkNotNull(builder); + builder.setPageSize(pageSize); + } + + private void initFederationQueueWeightsMapping() { + if (this.federationQueueWeights != null) { + return; + } + + QueryFederationQueuePoliciesResponseProtoOrBuilder p = viaProto ? proto : builder; + List queryFederationQueuePoliciesProtoList = + p.getFederationQueueWeightsList(); + + List fqWeights = new ArrayList<>(); + if (queryFederationQueuePoliciesProtoList == null || + queryFederationQueuePoliciesProtoList.size() == 0) { + this.federationQueueWeights = fqWeights; + return; + } + + for (FederationQueueWeightProto federationQueueWeightProto : + queryFederationQueuePoliciesProtoList) { + fqWeights.add(new FederationQueueWeightPBImpl(federationQueueWeightProto)); + } + + this.federationQueueWeights = fqWeights; + } + + + @Override + public List getFederationQueueWeights() { + initFederationQueueWeightsMapping(); + return this.federationQueueWeights; + } + + @Override + public void setFederationQueueWeights(List pfederationQueueWeights) { + maybeInitBuilder(); + if (federationQueueWeights == null) { + federationQueueWeights = new ArrayList<>(); + } + if(pfederationQueueWeights == null) { + builder.clearFederationQueueWeights(); + return; + } + federationQueueWeights.clear(); + federationQueueWeights.addAll(pfederationQueueWeights); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index 0fc52898434d5..ed66dadefb497 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -179,6 +179,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; /** @@ -980,6 +982,12 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies return null; } + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + return null; + } + @VisibleForTesting public HashMap> getApplicationContainerIdMap() { return applicationContainerIdMap; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 7f1c7b93e946b..8f9e8caa53670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -101,6 +101,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; @@ -1095,6 +1097,14 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies " Please call Router's batchSaveFederationQueuePolicies to set Policies."); } + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + throw new YarnException("It is not allowed to call the RM's " + + " listFederationQueuePolicies. " + + " Please call Router's listFederationQueuePolicies to list Policies."); + } + private void validateAttributesExists( List nodesToAttributes) throws IOException { NodeAttributesManager nodeAttributesManager = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java index 6503765ede118..d0e4825fed39b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java @@ -153,6 +153,8 @@ public final class RouterMetrics { private MutableGaugeInt numSaveFederationQueuePolicyFailedRetrieved; @Metric("# of batchSaveFederationQueuePolicies failed to be retrieved") private MutableGaugeInt numBatchSaveFederationQueuePoliciesFailedRetrieved; + @Metric("# of listFederationQueuePolicies failed to be retrieved") + private MutableGaugeInt numListFederationQueuePoliciesFailedRetrieved; @Metric("# of refreshAdminAcls failed to be retrieved") private MutableGaugeInt numRefreshAdminAclsFailedRetrieved; @Metric("# of refreshServiceAcls failed to be retrieved") @@ -303,6 +305,8 @@ public final class RouterMetrics { private MutableRate totalSucceededSaveFederationQueuePolicyRetrieved; @Metric("Total number of successful Retrieved BatchSaveFederationQueuePolicies and latency(ms)") private MutableRate totalSucceededBatchSaveFederationQueuePoliciesRetrieved; + @Metric("Total number of successful Retrieved ListFederationQueuePolicies and latency(ms)") + private MutableRate totalSucceededListFederationQueuePoliciesFailedRetrieved; @Metric("Total number of successful Retrieved RefreshAdminAcls and latency(ms)") private MutableRate totalSucceededRefreshAdminAclsRetrieved; @Metric("Total number of successful Retrieved RefreshServiceAcls and latency(ms)") @@ -391,6 +395,7 @@ public final class RouterMetrics { private MutableQuantiles refreshDeregisterSubClusterLatency; private MutableQuantiles saveFederationQueuePolicyLatency; private MutableQuantiles batchSaveFederationQueuePoliciesLatency; + private MutableQuantiles listFederationQueuePoliciesLatency; private MutableQuantiles refreshAdminAclsLatency; private MutableQuantiles refreshServiceAclsLatency; private MutableQuantiles replaceLabelsOnNodesLatency; @@ -609,6 +614,10 @@ private RouterMetrics() { "batchSaveFederationQueuePoliciesLatency", "latency of batch save federationqueuepolicies timeouts", "ops", "latency", 10); + listFederationQueuePoliciesLatency = registry.newQuantiles( + "listFederationQueuePoliciesLatency", + "latency of list federationqueuepolicies timeouts", "ops", "latency", 10); + refreshAdminAclsLatency = registry.newQuantiles("refreshAdminAclsLatency", "latency of refresh admin acls timeouts", "ops", "latency", 10); @@ -948,6 +957,11 @@ public long getNumSucceededBatchSaveFederationQueuePoliciesRetrieved() { return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().numSamples(); } + @VisibleForTesting + public long getNumSucceededListFederationQueuePoliciesFailedRetrieved() { + return totalSucceededListFederationQueuePoliciesFailedRetrieved.lastStat().numSamples(); + } + @VisibleForTesting public long getNumSucceededRefreshAdminAclsRetrieved() { return totalSucceededRefreshAdminAclsRetrieved.lastStat().numSamples(); @@ -1303,6 +1317,11 @@ public double getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved() { return totalSucceededBatchSaveFederationQueuePoliciesRetrieved.lastStat().mean(); } + @VisibleForTesting + public double getLatencySucceededListFederationQueuePoliciesRetrieved() { + return totalSucceededListFederationQueuePoliciesFailedRetrieved.lastStat().mean(); + } + @VisibleForTesting public double getLatencySucceededRefreshAdminAclsRetrieved() { return totalSucceededRefreshAdminAclsRetrieved.lastStat().mean(); @@ -1606,6 +1625,10 @@ public int getBatchSaveFederationQueuePoliciesFailedRetrieved() { return numBatchSaveFederationQueuePoliciesFailedRetrieved.value(); } + public int getListFederationQueuePoliciesFailedRetrieved() { + return numListFederationQueuePoliciesFailedRetrieved.value(); + } + public int getNumRefreshAdminAclsFailedRetrieved() { return numRefreshAdminAclsFailedRetrieved.value(); } @@ -1968,6 +1991,11 @@ public void succeededBatchSaveFederationQueuePoliciesRetrieved(long duration) { batchSaveFederationQueuePoliciesLatency.add(duration); } + public void succeededListFederationQueuePoliciesRetrieved(long duration) { + totalSucceededListFederationQueuePoliciesFailedRetrieved.add(duration); + listFederationQueuePoliciesLatency.add(duration); + } + public void succeededRefreshAdminAclsRetrieved(long duration) { totalSucceededRefreshAdminAclsRetrieved.add(duration); refreshAdminAclsLatency.add(duration); @@ -2254,6 +2282,10 @@ public void incrBatchSaveFederationQueuePoliciesFailedRetrieved() { numBatchSaveFederationQueuePoliciesFailedRetrieved.incr(); } + public void incrListFederationQueuePoliciesFailedRetrieved() { + numListFederationQueuePoliciesFailedRetrieved.incr(); + } + public void incrRefreshAdminAclsFailedRetrieved() { numRefreshAdminAclsFailedRetrieved.incr(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java index ed0698b1a812c..23517b97b4e19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java @@ -62,6 +62,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -225,4 +227,10 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { return rmAdminProxy.batchSaveFederationQueuePolicies(request); } + + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + return rmAdminProxy.listFederationQueuePolicies(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index a80e31f67d509..268522c1b1921 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router.rmadmin; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; @@ -68,6 +69,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -84,6 +87,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import java.util.ArrayList; import java.util.Map; @@ -995,6 +999,213 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies throw new YarnException("Unable to batchSaveFederationQueuePolicies."); } + /** + * List the Queue Policies for the Federation. + * + * @param request QueryFederationQueuePolicies Request. + * @return QueryFederationQueuePolicies Response. + * + * @throws YarnException indicates exceptions from yarn servers. + * @throws IOException io error occurs. + */ + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + + // Parameter validation. + if (request == null) { + routerMetrics.incrListFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing ListFederationQueuePolicies Request.", null); + } + + if (request.getPageSize() <= 0) { + routerMetrics.incrListFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "PageSize cannot be negative or zero.", null); + } + + if (request.getCurrentPage() <= 0) { + routerMetrics.incrListFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "CurrentPage cannot be negative or zero.", null); + } + + try { + QueryFederationQueuePoliciesResponse response = null; + + long startTime = clock.getTime(); + String queue = request.getQueue(); + List queues = request.getQueues(); + int currentPage = request.getCurrentPage(); + int pageSize = request.getPageSize(); + + // Print log + LOG.info("queue = {}, queues={}, currentPage={}, pageSize={}", + queue, queues, currentPage, pageSize); + + Map policiesConfigurations = + federationFacade.getPoliciesConfigurations(); + + // If the queue is not empty, filter according to the queue. + if (StringUtils.isNotBlank(queue)) { + response = filterPoliciesConfigurationsByQueue(queue, policiesConfigurations, + pageSize, currentPage); + } else if(CollectionUtils.isNotEmpty(queues)) { + // If queues are not empty, filter by queues, which may return multiple results. + // We filter by pagination. + response = filterPoliciesConfigurationsByQueues(queues, policiesConfigurations, + pageSize, currentPage); + } + long stopTime = clock.getTime(); + routerMetrics.succeededListFederationQueuePoliciesRetrieved(stopTime - startTime); + return response; + } catch (Exception e) { + routerMetrics.incrListFederationQueuePoliciesFailedRetrieved(); + RouterServerUtil.logAndThrowException(e, + "Unable to ListFederationQueuePolicies due to exception. " + e.getMessage()); + } + + routerMetrics.incrListFederationQueuePoliciesFailedRetrieved(); + throw new YarnException("Unable to listFederationQueuePolicies."); + } + + /** + * According to the configuration information of the queue filtering queue, + * this part should only return 1 result. + * + * @param queue queueName. + * @param policiesConfigurations policy configurations. + * @param pageSize Items per page. + * @param currentPage The number of pages to be queried. + * @return federation queue policies response. + * @throws YarnException indicates exceptions from yarn servers. + * + */ + private QueryFederationQueuePoliciesResponse filterPoliciesConfigurationsByQueue(String queue, + Map policiesConfigurations, + int pageSize, int currentPage) throws YarnException { + + // Step1. Check the parameters, if the policy list is empty, return empty directly. + if (MapUtils.isEmpty(policiesConfigurations)) { + return null; + } + SubClusterPolicyConfiguration policyConf = policiesConfigurations.getOrDefault(queue, null); + if(policyConf == null) { + return null; + } + + // Step2. Parse the parameters. + List federationQueueWeights = new ArrayList<>(); + FederationQueueWeight federationQueueWeight = parseFederationQueueWeight(queue, policyConf); + federationQueueWeights.add(federationQueueWeight); + + // Step3. Return result. + return QueryFederationQueuePoliciesResponse.newInstance( + 1, 1, currentPage, pageSize, federationQueueWeights); + } + + /** + * Filter queue configuration information based on the queue list. + * + * @param queues The name of the queue. + * @param policiesConfigurations policy configurations. + * @param pageSize Items per page. + * @param currentPage The number of pages to be queried. + * @return federation queue policies response. + * @throws YarnException indicates exceptions from yarn servers. + */ + private QueryFederationQueuePoliciesResponse filterPoliciesConfigurationsByQueues( + List queues, Map policiesConfigurations, + int pageSize, int currentPage) throws YarnException { + + // Step1. Check the parameters, if the policy list is empty, return empty directly. + if (MapUtils.isEmpty(policiesConfigurations)) { + return null; + } + + // Step2. Filtering for Queue Policies. + List federationQueueWeights = new ArrayList<>(); + for (String queue : queues) { + SubClusterPolicyConfiguration policyConf = policiesConfigurations.getOrDefault(queue, null); + if(policyConf == null) { + continue; + } + FederationQueueWeight federationQueueWeight = parseFederationQueueWeight(queue, policyConf); + if (federationQueueWeight != null) { + federationQueueWeights.add(federationQueueWeight); + } + } + + int startIndex = (currentPage - 1) * pageSize; + int endIndex = Math.min(startIndex + pageSize, federationQueueWeights.size()); + List subFederationQueueWeights = + federationQueueWeights.subList(startIndex, endIndex); + + int totalSize = federationQueueWeights.size(); + int totalPage = + (totalSize % pageSize == 0) ? totalSize / pageSize : (totalSize / pageSize) + 1; + + // Step3. Returns the Queue Policies result. + return QueryFederationQueuePoliciesResponse.newInstance( + totalSize, totalPage, currentPage, pageSize, subFederationQueueWeights); + } + + /** + * Parses a FederationQueueWeight from the given queue and SubClusterPolicyConfiguration. + * + * @param queue The name of the queue. + * @param policyConf policy configuration. + * @return Queue weights for representing Federation. + * @throws YarnException YarnException indicates exceptions from yarn servers. + */ + private FederationQueueWeight parseFederationQueueWeight(String queue, + SubClusterPolicyConfiguration policyConf) throws YarnException { + + if (policyConf != null) { + ByteBuffer params = policyConf.getParams(); + WeightedPolicyInfo weightedPolicyInfo = WeightedPolicyInfo.fromByteBuffer(params); + Map amrmPolicyWeights = weightedPolicyInfo.getAMRMPolicyWeights(); + Map routerPolicyWeights = + weightedPolicyInfo.getRouterPolicyWeights(); + float headroomAlpha = weightedPolicyInfo.getHeadroomAlpha(); + String policyManagerClassName = policyConf.getType(); + + String amrmPolicyWeight = parsePolicyWeights(amrmPolicyWeights); + String routerPolicyWeight = parsePolicyWeights(routerPolicyWeights); + + FederationQueueWeight.checkSubClusterQueueWeightRatioValid(amrmPolicyWeight); + FederationQueueWeight.checkSubClusterQueueWeightRatioValid(routerPolicyWeight); + + return FederationQueueWeight.newInstance(routerPolicyWeight, amrmPolicyWeight, + String.valueOf(headroomAlpha), queue, policyManagerClassName); + } + + return null; + } + + /** + * Parses the policy weights from the provided policyWeights map. + * returns a string similar to the following: + * SC-1:0.7,SC-2:0.3 + * + * @param policyWeights + * A map containing SubClusterIdInfo as keys and corresponding weight values. + * @return A string representation of the parsed policy weights. + */ + protected String parsePolicyWeights(Map policyWeights) { + if (MapUtils.isEmpty(policyWeights)) { + return null; + } + List policyWeightList = new ArrayList<>(); + for (Map.Entry entry : policyWeights.entrySet()) { + SubClusterIdInfo key = entry.getKey(); + Float value = entry.getValue(); + policyWeightList.add(key.toId() + ":" + value); + } + return StringUtils.join(policyWeightList, ","); + } + /** * Save FederationQueuePolicy. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index 3b760e28f7889..fc2278d3bb318 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -69,6 +69,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; @@ -410,4 +412,11 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies RequestInterceptorChainWrapper pipeline = getInterceptorChain(); return pipeline.getRootInterceptor().batchSaveFederationQueuePolicies(request); } + + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().listFederationQueuePolicies(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java index f62ddb27962e4..60d5e5303c7fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java @@ -633,6 +633,11 @@ public void getBatchSaveFederationQueuePoliciesFailedRetrieved() { LOG.info("Mocked: failed BatchSaveFederationQueuePolicies call"); metrics.incrBatchSaveFederationQueuePoliciesFailedRetrieved(); } + + public void getListFederationQueuePoliciesFailedRetrieved() { + LOG.info("Mocked: failed ListFederationQueuePolicies call"); + metrics.incrListFederationQueuePoliciesFailedRetrieved(); + } } // Records successes for all calls @@ -974,6 +979,12 @@ public void getBatchSaveFederationQueuePoliciesRetrieved(long duration) { " call with duration {}", duration); metrics.succeededBatchSaveFederationQueuePoliciesRetrieved(duration); } + + public void getListFederationQueuePoliciesRetrieved(long duration) { + LOG.info("Mocked: successful ListFederationQueuePoliciesRetrieved " + + " call with duration {}", duration); + metrics.succeededListFederationQueuePoliciesRetrieved(duration); + } } @Test @@ -2277,4 +2288,27 @@ public void testGetBatchSaveFederationQueuePoliciesRetrieved() { metrics.getLatencySucceededBatchSaveFederationQueuePoliciesRetrieved(), ASSERT_DOUBLE_DELTA); } + + @Test + public void testListFederationQueuePoliciesFailedRetrieved() { + long totalBadBefore = metrics.getListFederationQueuePoliciesFailedRetrieved(); + badSubCluster.getListFederationQueuePoliciesFailedRetrieved(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getListFederationQueuePoliciesFailedRetrieved()); + } + + @Test + public void testListFederationQueuePoliciesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededListFederationQueuePoliciesFailedRetrieved(); + goodSubCluster.getListFederationQueuePoliciesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededListFederationQueuePoliciesFailedRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededListFederationQueuePoliciesRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getListFederationQueuePoliciesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededListFederationQueuePoliciesFailedRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededListFederationQueuePoliciesRetrieved(), ASSERT_DOUBLE_DELTA); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java index 263926e728b26..6fe218c7b2a00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java @@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; /** * Mock interceptor that does not do anything other than forwarding it to the @@ -177,4 +179,10 @@ public BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies BatchSaveFederationQueuePoliciesRequest request) throws YarnException, IOException { return getNextInterceptor().batchSaveFederationQueuePolicies(request); } + + @Override + public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( + QueryFederationQueuePoliciesRequest request) throws YarnException, IOException { + return getNextInterceptor().listFederationQueuePolicies(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java index 20266d5b54b6e..8439b01222e2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java @@ -62,6 +62,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; @@ -82,6 +84,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.HashSet; @@ -802,10 +805,10 @@ private FederationQueueWeight generateFederationQueueWeight( * Generating Policy Weight Data. * * @param pSubClusters set of sub-clusters. - * @return policy Weight String, like SC-1:0.7,SC-2:0. + * @return policy Weight String, like SC-1:0.7,SC-2:0.3 */ private String generatePolicyWeight(List pSubClusters) { - List weights = generateWeights(subClusters.size()); + List weights = generateWeights(pSubClusters.size()); List subClusterWeight = new ArrayList<>(); for (int i = 0; i < pSubClusters.size(); i++) { String subCluster = pSubClusters.get(i); @@ -844,4 +847,123 @@ private List generateWeights(int n) { return formattedRandomNumbers; } + + @Test + public void testParsePolicyWeights() { + Map policyWeights = new LinkedHashMap<>(); + SubClusterIdInfo sc1 = new SubClusterIdInfo("SC-1"); + policyWeights.put(sc1, 0.7f); + SubClusterIdInfo sc2 = new SubClusterIdInfo("SC-2"); + policyWeights.put(sc2, 0.3f); + String policyWeight = interceptor.parsePolicyWeights(policyWeights); + assertEquals("SC-1:0.7,SC-2:0.3", policyWeight); + } + + @Test + public void testFilterPoliciesConfigurationsByQueues() throws Exception { + // SubClusters : SC-1,SC-2 + List subClusterLists = new ArrayList<>(); + subClusterLists.add("SC-1"); + subClusterLists.add("SC-2"); + + // We initialize 26 queues, queue root.a~root.z + List federationQueueWeights = new ArrayList<>(); + for (char letter = 'a'; letter <= 'z'; letter++) { + FederationQueueWeight leaf = + generateFederationQueueWeight("root." + letter, subClusterLists); + federationQueueWeights.add(leaf); + } + + // Save Queue Policies in Batches + BatchSaveFederationQueuePoliciesRequest request = + BatchSaveFederationQueuePoliciesRequest.newInstance(federationQueueWeights); + + BatchSaveFederationQueuePoliciesResponse policiesResponse = + interceptor.batchSaveFederationQueuePolicies(request); + + assertNotNull(policiesResponse); + assertNotNull(policiesResponse.getMessage()); + assertEquals("batch save policies success.", policiesResponse.getMessage()); + + // We query 12 queues, root.a ~ root.l + List queues = new ArrayList<>(); + for (char letter = 'a'; letter <= 'l'; letter++) { + queues.add("root." + letter); + } + + // Queue1: We query page 1, 10 items per page, and the returned result should be 10 items. + // TotalPage should be 2, TotalSize should be 12. + QueryFederationQueuePoliciesRequest request1 = + QueryFederationQueuePoliciesRequest.newInstance(10, 1, "", queues); + QueryFederationQueuePoliciesResponse response1 = + interceptor.listFederationQueuePolicies(request1); + assertNotNull(response1); + assertEquals(1, response1.getCurrentPage()); + assertEquals(10, response1.getPageSize()); + assertEquals(2, response1.getTotalPage()); + assertEquals(12, response1.getTotalSize()); + List federationQueueWeights1 = response1.getFederationQueueWeights(); + assertNotNull(federationQueueWeights1); + assertEquals(10, federationQueueWeights1.size()); + + // Queue2: We query page 1, 12 items per page, and the returned result should be 12 items. + // TotalPage should be 1, TotalSize should be 12. + QueryFederationQueuePoliciesRequest request2 = + QueryFederationQueuePoliciesRequest.newInstance(12, 1, "", queues); + QueryFederationQueuePoliciesResponse response2 = + interceptor.listFederationQueuePolicies(request2); + assertNotNull(response2); + assertEquals(1, response2.getCurrentPage()); + assertEquals(12, response2.getPageSize()); + assertEquals(1, response2.getTotalPage()); + assertEquals(12, response2.getTotalSize()); + List federationQueueWeights2 = response2.getFederationQueueWeights(); + assertNotNull(federationQueueWeights2); + assertEquals(12, federationQueueWeights2.size()); + + // Queue3: Boundary limit exceeded + // We filter 12 queues, should return 12 records, 12 per page, + // should return 1 page, but we are going to return page 2. + QueryFederationQueuePoliciesRequest request3 = + QueryFederationQueuePoliciesRequest.newInstance(12, 2, "", queues); + QueryFederationQueuePoliciesResponse response3 = + interceptor.listFederationQueuePolicies(request3); + assertNotNull(response3); + assertEquals(2, response3.getCurrentPage()); + assertEquals(12, response3.getPageSize()); + assertEquals(1, response2.getTotalPage()); + assertEquals(12, response3.getTotalSize()); + List federationQueueWeights3 = response3.getFederationQueueWeights(); + assertNotNull(federationQueueWeights3); + assertEquals(0, federationQueueWeights3.size()); + + // Queue4: Boundary limit exceeded + // We pass in some negative parameters and we will get some exceptions + QueryFederationQueuePoliciesRequest request4 = + QueryFederationQueuePoliciesRequest.newInstance(-1, 2, "", queues); + LambdaTestUtils.intercept(YarnException.class, "PageSize cannot be negative or zero.", + () -> interceptor.listFederationQueuePolicies(request4)); + + // Queue5: Boundary limit exceeded + // We pass in some negative parameters and we will get some exceptions + QueryFederationQueuePoliciesRequest request5 = + QueryFederationQueuePoliciesRequest.newInstance(10, -1, "", queues); + LambdaTestUtils.intercept(YarnException.class, "CurrentPage cannot be negative or zero.", + () -> interceptor.listFederationQueuePolicies(request5)); + + // Queue6: We use Queue as the condition, + // at this time we will only get the only one return value. + QueryFederationQueuePoliciesRequest request6 = + QueryFederationQueuePoliciesRequest.newInstance(10, 1, "root.a", null); + QueryFederationQueuePoliciesResponse response6 = + interceptor.listFederationQueuePolicies(request6); + assertNotNull(response6); + assertEquals(1, response6.getCurrentPage()); + assertEquals(10, response6.getPageSize()); + assertEquals(1, response6.getTotalPage()); + assertEquals(1, response6.getTotalSize()); + List federationQueueWeights6 = response6.getFederationQueueWeights(); + assertNotNull(federationQueueWeights6); + assertEquals(1, federationQueueWeights6.size()); + } } From 07c8b69f7cccae98be6e4d338baea4f0a58610f7 Mon Sep 17 00:00:00 2001 From: dannytbecker <43830149+dannytbecker@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:17:12 -0700 Subject: [PATCH 013/155] HDFS-17167. Add config to startup NameNode as Observer (#6013) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hdfs/server/namenode/BackupNode.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 14 ++++- .../src/main/resources/hdfs-default.xml | 9 +++ .../server/namenode/ha/TestObserverNode.java | 60 +++++++++++++++++++ 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 7c6683dfc176b..f0450b0778dad 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1177,6 +1177,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_PLUGINS_KEY = "dfs.namenode.plugins"; public static final String DFS_WEB_UGI_KEY = "dfs.web.ugi"; public static final String DFS_NAMENODE_STARTUP_KEY = "dfs.namenode.startup"; + public static final String DFS_NAMENODE_OBSERVER_ENABLED_KEY = "dfs.namenode.observer.enabled"; + public static final boolean DFS_NAMENODE_OBSERVER_ENABLED_DEFAULT = false; public static final String DFS_DATANODE_KEYTAB_FILE_KEY = "dfs.datanode.keytab.file"; public static final String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = HdfsClientConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index 105f172812bba..a3d9746be1224 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -435,7 +435,7 @@ protected String getNameServiceId(Configuration conf) { } @Override - protected HAState createHAState(StartupOption startOpt) { + protected HAState createHAState(Configuration conf) { return new BackupState(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 4aa81152fa9ee..388f06f0e33ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -163,6 +163,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_OBSERVER_ENABLED_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_OBSERVER_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY; @@ -1125,7 +1127,7 @@ protected NameNode(Configuration conf, NamenodeRole role) + " this namenode/service.", clientNamenodeAddress); } this.haEnabled = HAUtil.isHAEnabled(conf, nsId); - state = createHAState(getStartupOption(conf)); + state = createHAState(conf); this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf); this.haContext = createHAContext(); try { @@ -1161,11 +1163,17 @@ private void stopAtException(Exception e){ } } - protected HAState createHAState(StartupOption startOpt) { + protected HAState createHAState(Configuration conf) { + StartupOption startOpt = getStartupOption(conf); if (!haEnabled || startOpt == StartupOption.UPGRADE || startOpt == StartupOption.UPGRADEONLY) { return ACTIVE_STATE; - } else if (startOpt == StartupOption.OBSERVER) { + } else if (conf.getBoolean(DFS_NAMENODE_OBSERVER_ENABLED_KEY, + DFS_NAMENODE_OBSERVER_ENABLED_DEFAULT) + || startOpt == StartupOption.OBSERVER) { + // Set Observer state using config instead of startup option + // This allows other startup options to be used when starting observer. + // e.g. rollingUpgrade return OBSERVER_STATE; } else { return STANDBY_STATE; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 8d85d7cc3c0af..c8f5993adbbb7 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2801,6 +2801,15 @@ + + dfs.namenode.observer.enabled + false + + This causes NameNode on startup to become an observer node if + set to true, otherwise startup is no different. + + + dfs.namenode.enable.retrycache true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java index 48f2cb185e2e0..a293cb4d17c47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_OBSERVER_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState; import static org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider.*; @@ -215,6 +216,65 @@ public void testSimpleRead() throws Exception { assertSentTo(0); } + @Test + public void testConfigStartup() throws Exception { + int nnIdx = dfsCluster.getNumNameNodes() - 1; + + // Transition all current observers to standby + for (int i = 0; i < dfsCluster.getNumNameNodes(); i++) { + if (dfsCluster.getNameNode(i).isObserverState()) { + dfsCluster.transitionToStandby(i); + } + } + + // Confirm that the namenode at nnIdx is standby + assertTrue("The NameNode is observer despite being transitioned to standby", + dfsCluster.getNameNode(nnIdx).isStandbyState()); + + // Restart the NameNode with observer startup option as false + dfsCluster.getConfiguration(nnIdx) + .setBoolean(DFS_NAMENODE_OBSERVER_ENABLED_KEY, false); + dfsCluster.restartNameNode(nnIdx); + + // Verify that the NameNode is not in Observer state + dfsCluster.waitNameNodeUp(nnIdx); + assertTrue("The NameNode started as Observer despite " + + DFS_NAMENODE_OBSERVER_ENABLED_KEY + " being false", + dfsCluster.getNameNode(nnIdx).isStandbyState()); + + dfs.mkdir(testPath, FsPermission.getDefault()); + assertSentTo(0); + + // The first request goes to the active because it has not refreshed yet; + // the second would go to the observer if it was not in standby + dfsCluster.rollEditLogAndTail(0); + dfs.getFileStatus(testPath); + dfs.getFileStatus(testPath); + assertSentTo(0); + + Path testPath2 = new Path(testPath, "test2"); + // Restart the NameNode with the observer startup option as true + dfsCluster.getConfiguration(nnIdx) + .setBoolean(DFS_NAMENODE_OBSERVER_ENABLED_KEY, true); + dfsCluster.restartNameNode(nnIdx); + + // Check that the NameNode is in Observer state + dfsCluster.waitNameNodeUp(nnIdx); + assertTrue("The NameNode did not start as Observer despite " + + DFS_NAMENODE_OBSERVER_ENABLED_KEY + " being true", + dfsCluster.getNameNode(nnIdx).isObserverState()); + + dfs.mkdir(testPath2, FsPermission.getDefault()); + assertSentTo(0); + + // The first request goes to the active because it has not refreshed yet; + // the second will properly go to the observer + dfsCluster.rollEditLogAndTail(0); + dfs.getFileStatus(testPath2); + dfs.getFileStatus(testPath2); + assertSentTo(nnIdx); + } + @Test public void testFailover() throws Exception { Path testPath2 = new Path(testPath, "test2"); From 9342ecf6ccd5c7ef443a0eb722852d2addc1d5db Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 6 Sep 2023 21:32:36 -0400 Subject: [PATCH 014/155] HADOOP-18870. CURATOR-599 change broke functionality introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi --- .../hadoop/util/curator/ZKCuratorManager.java | 12 ++++++- .../util/curator/TestZKCuratorManager.java | 35 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index 81ee4663e1c8a..4df7977432918 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -549,7 +549,17 @@ public HadoopZookeeperFactory(String zkPrincipal, String kerberosPrincipal, public ZooKeeper newZooKeeper(String connectString, int sessionTimeout, Watcher watcher, boolean canBeReadOnly ) throws Exception { - ZKClientConfig zkClientConfig = new ZKClientConfig(); + return this.newZooKeeper(connectString, sessionTimeout, + watcher, canBeReadOnly, new ZKClientConfig()); + } + + @Override + public ZooKeeper newZooKeeper(String connectString, int sessionTimeout, + Watcher watcher, boolean canBeReadOnly, ZKClientConfig zkClientConfig + ) throws Exception { + if (zkClientConfig == null) { + zkClientConfig = new ZKClientConfig(); + } if (zkPrincipal != null) { LOG.info("Configuring zookeeper to use {} as the server principal", zkPrincipal); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java index fd15a0c2b1bf4..4365e43e49139 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java @@ -22,10 +22,15 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import javax.security.auth.login.AppConfigurationEntry; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -193,6 +198,36 @@ public void testJaasConfiguration() throws Exception { } } + @Test + public void testCuratorFrameworkFactory() throws Exception{ + // By not explicitly calling the NewZooKeeper method validate that the Curator override works. + ZKClientConfig zkClientConfig = new ZKClientConfig(); + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.ZK_ADDRESS, this.server.getConnectString()); + int numRetries = conf.getInt(CommonConfigurationKeys.ZK_NUM_RETRIES, + CommonConfigurationKeys.ZK_NUM_RETRIES_DEFAULT); + int zkSessionTimeout = conf.getInt(CommonConfigurationKeys.ZK_TIMEOUT_MS, + CommonConfigurationKeys.ZK_TIMEOUT_MS_DEFAULT); + int zkRetryInterval = conf.getInt( + CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS, + CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS_DEFAULT); + RetryNTimes retryPolicy = new RetryNTimes(numRetries, zkRetryInterval); + + CuratorFramework client = CuratorFrameworkFactory.builder() + .connectString(conf.get(CommonConfigurationKeys.ZK_ADDRESS)) + .zkClientConfig(zkClientConfig) + .sessionTimeoutMs(zkSessionTimeout).retryPolicy(retryPolicy) + .authorization(new ArrayList<>()) + .zookeeperFactory(new ZKCuratorManager.HadoopZookeeperFactory( + "foo1", "bar1", "bar1.keytab", false, + new ZKCuratorManager.TruststoreKeystore(conf)) + + ).build(); + client.start(); + validateJaasConfiguration(ZKCuratorManager.HadoopZookeeperFactory.JAAS_CLIENT_ENTRY, + "bar1", "bar1.keytab", client.getZookeeperClient().getZooKeeper()); + } + private void validateJaasConfiguration(String clientConfig, String principal, String keytab, ZooKeeper zk) { assertEquals("Validate that expected clientConfig is set in ZK config", clientConfig, From 3ab983fbae70a789bb0cb759d5be2faccd4cabd1 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 8 Sep 2023 07:52:54 +0800 Subject: [PATCH 015/155] YARN-11434. [Router] UGI conf doesn't read user overridden configurations on Router startup. (#6011) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../GlobalPolicyGenerator.java | 2 ++ .../TestGlobalPolicyGenerator.java | 13 +++++++++++++ .../apache/hadoop/yarn/server/router/Router.java | 2 ++ .../hadoop/yarn/server/router/TestRouter.java | 13 +++++++++++++ 4 files changed, 30 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index 80434ecd8f22e..dc21d6ef36cd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -35,6 +35,7 @@ import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; @@ -111,6 +112,7 @@ protected void initAndStart(Configuration conf, boolean hasToReboot) { @Override protected void serviceInit(Configuration conf) throws Exception { + UserGroupInformation.setConfiguration(conf); // Set up the context this.gpgContext.setStateStoreFacade(FederationStateStoreFacade.getInstance(conf)); GPGPolicyFacade gpgPolicyFacade = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java index bc9a325c47ebd..4f8521dd8aeac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java @@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.service.Service; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Test; @@ -55,4 +57,15 @@ public void testGpgWithFederation() throws InterruptedException, TimeoutExceptio return (services.size() == 1 && gpg.getWebApp() != null); }, 100, 5000); } + + @Test + public void testUserProvidedUGIConf() throws Exception { + String errMsg = "Invalid attribute value for " + + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION + " of DUMMYAUTH"; + Configuration dummyConf = new YarnConfiguration(); + dummyConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "DUMMYAUTH"); + GlobalPolicyGenerator gpg = new GlobalPolicyGenerator(); + LambdaTestUtils.intercept(IllegalArgumentException.class, errMsg, () -> gpg.init(dummyConf)); + gpg.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 993fb81f3a695..62153242f7cf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -33,6 +33,7 @@ import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; @@ -120,6 +121,7 @@ protected void doSecureLogin() throws IOException { @Override protected void serviceInit(Configuration config) throws Exception { this.conf = config; + UserGroupInformation.setConfiguration(this.conf); // ClientRM Proxy clientRMProxyService = createClientRMProxyService(); addService(clientRMProxyService); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java index 89438d72d0ce9..2bc202f61866d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.security.http.CrossOriginFilter; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.WebApp; import org.eclipse.jetty.servlet.FilterHolder; @@ -185,6 +186,18 @@ public void testRouterSupportCrossOrigin() throws ServletException, IOException router.stop(); } + @Test + public void testUserProvidedUGIConf() throws Exception { + String errMsg = "Invalid attribute value for " + + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION + " of DUMMYAUTH"; + Configuration dummyConf = new YarnConfiguration(); + dummyConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "DUMMYAUTH"); + Router router = new Router(); + LambdaTestUtils.intercept(IllegalArgumentException.class, errMsg, + () -> router.init(dummyConf)); + router.stop(); + } + private class HttpServletResponseForRouterTest implements HttpServletResponse { private final Map headers = new HashMap<>(1); From bf605c8accbf1ad714ff8cb4d0cea89bfb981762 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Fri, 8 Sep 2023 15:23:26 +0800 Subject: [PATCH 016/155] YARN-11564. Fix wrong config in yarn-default.xml (#6030) Contributed by Chenyu Zheng. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 408295b8f4844..f82540b8f46cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5153,7 +5153,7 @@ The number of threads to use for the Router scheduled executor service. - yarn.router.subcluster.cleaner.interval.time + yarn.router.scheduled.executor.threads 1 From c5e9510b54932946f792cfc88acf8b022b5d2fe8 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 9 Sep 2023 09:50:53 +0800 Subject: [PATCH 017/155] YARN-8980. Mapreduce application container start fail after AM restart. (#5975) Contributed by Chenyu Zheng. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../amrmproxy/FederationInterceptor.java | 26 +++ .../resourcemanager/DefaultAMSProcessor.java | 7 +- .../TestWorkPreservingUnmanagedAM.java | 155 +++++++++++++++++- 3 files changed, 176 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java index e7fcb1c3785c3..9c4c2c72e5c5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java @@ -98,6 +98,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.MonotonicClock; import org.apache.hadoop.yarn.util.resource.Resources; +import org.eclipse.jetty.util.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -260,6 +261,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor { private final MonotonicClock clock = new MonotonicClock(); + /* + * For UAM, keepContainersAcrossApplicationAttempts is always true. + * When re-register to RM, RM will clear node set and regenerate NMToken for transferred + * container. But If keepContainersAcrossApplicationAttempts of AM is false, AM may not + * called getNMTokensFromPreviousAttempts, so the NMToken which is pass from + * RegisterApplicationMasterResponse will be missing. Here we cache these NMToken, + * then pass to AM in allocate stage. + * */ + private Set nmTokenMapFromRegisterSecondaryCluster; + /** * Creates an instance of the FederationInterceptor class. */ @@ -278,6 +289,7 @@ public FederationInterceptor() { this.finishAMCalled = false; this.lastSCResponseTime = new ConcurrentHashMap<>(); this.lastAMHeartbeatTime = this.clock.getTime(); + this.nmTokenMapFromRegisterSecondaryCluster = new ConcurrentHashSet<>(); } /** @@ -453,6 +465,7 @@ public void recover(Map recoveredDataMap) { // RegisterApplicationMaster RegisterApplicationMasterResponse response = this.uamPool.registerApplicationMaster(keyScId, this.amRegistrationRequest); + nmTokenMapFromRegisterSecondaryCluster.addAll(response.getNMTokensFromPreviousAttempts()); // Set sub-cluster to be timed out initially lastSCResponseTime.put(subClusterId, clock.getTime() - subClusterTimeOut); @@ -1096,6 +1109,8 @@ protected void reAttachUAMAndMergeRegisterResponse( if (registerResponse != null) { LOG.info("Merging register response for {}", appId); mergeRegisterResponse(homeResponse, registerResponse); + nmTokenMapFromRegisterSecondaryCluster.addAll( + registerResponse.getNMTokensFromPreviousAttempts()); } } catch (Exception e) { LOG.warn("Reattaching UAM failed for ApplicationId: " + appId, e); @@ -1434,6 +1449,17 @@ private void mergeAllocateResponses(AllocateResponse mergedResponse) { } } } + // When re-register RM, client may not cache the NMToken from register response. + // Here we pass these NMToken in allocate stage. + if (nmTokenMapFromRegisterSecondaryCluster.size() > 0) { + List duplicateNmToken = new ArrayList(nmTokenMapFromRegisterSecondaryCluster); + nmTokenMapFromRegisterSecondaryCluster.removeAll(duplicateNmToken); + if (!isNullOrEmpty(mergedResponse.getNMTokens())) { + mergedResponse.getNMTokens().addAll(duplicateNmToken); + } else { + mergedResponse.setNMTokens(duplicateNmToken); + } + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 98fb9b349424e..96e65855fee9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -168,14 +168,13 @@ public void registerApplicationMaster( // and corresponding NM tokens. if (app.getApplicationSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { + // Clear the node set remembered by the secret manager. Necessary + // for UAM restart because we use the same attemptId. + rmContext.getNMTokenSecretManager().clearNodeSetForAttempt(applicationAttemptId); List transferredContainers = getScheduler() .getTransferredContainers(applicationAttemptId); if (!transferredContainers.isEmpty()) { response.setContainersFromPreviousAttempts(transferredContainers); - // Clear the node set remembered by the secret manager. Necessary - // for UAM restart because we use the same attemptId. - rmContext.getNMTokenSecretManager() - .clearNodeSetForAttempt(applicationAttemptId); List nmTokens = new ArrayList(); for (Container container : transferredContainers) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java index 3ed5cfd4f7baf..3c57dfc19e61c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java @@ -21,15 +21,19 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; @@ -71,6 +75,7 @@ protected void testUAMRestart(boolean keepContainers) throws Exception { MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm.registerNode(); + Set tokenCacheClientSide = new HashSet(); // create app and launch the UAM boolean unamanged = true; @@ -98,14 +103,19 @@ protected void testUAMRestart(boolean keepContainers) throws Exception { // Allocate two containers to UAM int numContainers = 3; - List conts = am.allocate("127.0.0.1", 1000, numContainers, - new ArrayList()).getAllocatedContainers(); + AllocateResponse allocateResponse = + am.allocate("127.0.0.1", 1000, numContainers, new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + List conts = allocateResponse.getAllocatedContainers(); while (conts.size() < numContainers) { nm.nodeHeartbeat(true); - conts.addAll(am.allocate(new ArrayList(), - new ArrayList()).getAllocatedContainers()); + allocateResponse = + am.allocate(new ArrayList(), new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + conts.addAll(allocateResponse.getAllocatedContainers()); Thread.sleep(100); } + checkNMTokenForContainer(tokenCacheClientSide, conts); // Release one container List releaseList = @@ -127,6 +137,10 @@ protected void testUAMRestart(boolean keepContainers) throws Exception { RegisterApplicationMasterResponse response = null; try { response = am.registerAppAttempt(false); + // When AM restart, it means nmToken in client side should be missing + tokenCacheClientSide.clear(); + response.getNMTokensFromPreviousAttempts() + .forEach(token -> tokenCacheClientSide.add(token.getNodeId())); } catch (InvalidApplicationMasterRequestException e) { Assert.assertEquals(false, keepContainers); return; @@ -142,14 +156,124 @@ protected void testUAMRestart(boolean keepContainers) throws Exception { numContainers = 1; am.allocate("127.0.0.1", 1000, numContainers, new ArrayList()); nm.nodeHeartbeat(true); - conts = am.allocate(new ArrayList(), - new ArrayList()).getAllocatedContainers(); + allocateResponse = am.allocate(new ArrayList(), new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + conts = allocateResponse.getAllocatedContainers(); while (conts.size() < numContainers) { nm.nodeHeartbeat(true); - conts.addAll(am.allocate(new ArrayList(), - new ArrayList()).getAllocatedContainers()); + allocateResponse = + am.allocate(new ArrayList(), new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + conts.addAll(allocateResponse.getAllocatedContainers()); Thread.sleep(100); } + checkNMTokenForContainer(tokenCacheClientSide, conts); + + rm.stop(); + } + + protected void testUAMRestartWithoutTransferContainer(boolean keepContainers) throws Exception { + // start RM + MockRM rm = new MockRM(); + rm.start(); + MockNM nm = + new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); + nm.registerNode(); + Set tokenCacheClientSide = new HashSet(); + + // create app and launch the UAM + boolean unamanged = true; + int maxAttempts = 1; + boolean waitForAccepted = true; + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(200, rm) + .withAppName("") + .withUser(UserGroupInformation.getCurrentUser().getShortUserName()) + .withAcls(null) + .withUnmanagedAM(unamanged) + .withQueue(null) + .withMaxAppAttempts(maxAttempts) + .withCredentials(null) + .withAppType(null) + .withWaitForAppAcceptedState(waitForAccepted) + .withKeepContainers(keepContainers) + .build(); + RMApp app = MockRMAppSubmitter.submit(rm, data); + + MockAM am = MockRM.launchUAM(app, rm, nm); + + // Register for the first time + am.registerAppAttempt(); + + // Allocate two containers to UAM + int numContainers = 3; + AllocateResponse allocateResponse = + am.allocate("127.0.0.1", 1000, numContainers, new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + List conts = allocateResponse.getAllocatedContainers(); + while (conts.size() < numContainers) { + nm.nodeHeartbeat(true); + allocateResponse = + am.allocate(new ArrayList(), new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + conts.addAll(allocateResponse.getAllocatedContainers()); + Thread.sleep(100); + } + checkNMTokenForContainer(tokenCacheClientSide, conts); + + // Release all containers, then there are no transfer containfer app attempt + List releaseList = new ArrayList(); + releaseList.add(conts.get(0).getId()); + releaseList.add(conts.get(1).getId()); + releaseList.add(conts.get(2).getId()); + List finishedConts = + am.allocate(new ArrayList(), releaseList) + .getCompletedContainersStatuses(); + while (finishedConts.size() < releaseList.size()) { + nm.nodeHeartbeat(true); + finishedConts + .addAll(am + .allocate(new ArrayList(), + new ArrayList()) + .getCompletedContainersStatuses()); + Thread.sleep(100); + } + + // Register for the second time + RegisterApplicationMasterResponse response = null; + try { + response = am.registerAppAttempt(false); + // When AM restart, it means nmToken in client side should be missing + tokenCacheClientSide.clear(); + response.getNMTokensFromPreviousAttempts() + .forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + } catch (InvalidApplicationMasterRequestException e) { + Assert.assertEquals(false, keepContainers); + return; + } + Assert.assertEquals("RM should not allow second register" + + " for UAM without keep container flag ", true, keepContainers); + + // Expecting the zero running containers previously + Assert.assertEquals(0, response.getContainersFromPreviousAttempts().size()); + Assert.assertEquals(0, response.getNMTokensFromPreviousAttempts().size()); + + // Allocate one more containers to UAM, just to be safe + numContainers = 1; + am.allocate("127.0.0.1", 1000, numContainers, new ArrayList()); + nm.nodeHeartbeat(true); + allocateResponse = am.allocate(new ArrayList(), new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + conts = allocateResponse.getAllocatedContainers(); + while (conts.size() < numContainers) { + nm.nodeHeartbeat(true); + allocateResponse = + am.allocate(new ArrayList(), new ArrayList()); + allocateResponse.getNMTokens().forEach(token -> tokenCacheClientSide.add(token.getNodeId())); + conts.addAll(allocateResponse.getAllocatedContainers()); + Thread.sleep(100); + } + checkNMTokenForContainer(tokenCacheClientSide, conts); rm.stop(); } @@ -164,4 +288,19 @@ public void testUAMRestartNoKeepContainers() throws Exception { testUAMRestart(false); } + @Test(timeout = 600000) + public void testUAMRestartKeepContainersWithoutTransferContainer() throws Exception { + testUAMRestartWithoutTransferContainer(true); + } + + @Test(timeout = 600000) + public void testUAMRestartNoKeepContainersWithoutTransferContainer() throws Exception { + testUAMRestartWithoutTransferContainer(false); + } + + private void checkNMTokenForContainer(Set cacheToken, List containers) { + for (Container container : containers) { + Assert.assertTrue(cacheToken.contains(container.getNodeId())); + } + } } From 3bd6a751edcb099cbd354b240684d4467209069d Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Mon, 11 Sep 2023 11:57:52 +0800 Subject: [PATCH 018/155] HDFS-17177. ErasureCodingWork should ignore the deleted block while reconstructing blocks (#6024) --- .../blockmanagement/ErasureCodingWork.java | 18 +++++++++++++----- .../blockmanagement/ReplicationWork.java | 5 +++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java index 147f4c3fd624a..b8c396696ab11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java @@ -48,7 +48,7 @@ public ErasureCodingWork(String blockPoolId, BlockInfo block, this.blockPoolId = blockPoolId; this.liveBlockIndices = liveBlockIndices; this.liveBusyBlockIndices = liveBusyBlockIndices; - this.excludeReconstructedIndices=excludeReconstrutedIndices; + this.excludeReconstructedIndices = excludeReconstrutedIndices; LOG.debug("Creating an ErasureCodingWork to {} reconstruct ", block); } @@ -62,10 +62,18 @@ void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set excludedNodes) { // TODO: new placement policy for EC considering multiple writers - DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( - getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], - getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), - storagePolicySuite.getPolicy(getStoragePolicyID()), null); + DatanodeStorageInfo[] chosenTargets = null; + // HDFS-14720. If the block is deleted, the block size will become + // BlockCommand.NO_ACK (LONG.MAX_VALUE) . This kind of block we don't need + // to send for replication or reconstruction + if (!getBlock().isDeleted()) { + chosenTargets = blockplacement.chooseTarget( + getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], + getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), + storagePolicySuite.getPolicy(getStoragePolicyID()), null); + } else { + LOG.warn("ErasureCodingWork could not need choose targets for {}", getBlock()); + } setTargets(chosenTargets); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java index 771751f21e094..15e5d5cdc2729 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.net.Node; import java.util.List; @@ -47,11 +46,13 @@ assert getSrcNodes().length > 0 // HDFS-14720 If the block is deleted, the block size will become // BlockCommand.NO_ACK (LONG.MAX_VALUE) . This kind of block we don't need // to send for replication or reconstruction - if (getBlock().getNumBytes() != BlockCommand.NO_ACK) { + if (!getBlock().isDeleted()) { chosenTargets = blockplacement.chooseTarget(getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); + } else { + LOG.warn("ReplicationWork could not need choose targets for {}", getBlock()); } setTargets(chosenTargets); } finally { From e4550e1aefcd344eeff0a0fbdec043b0ce71661e Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:28:31 +0800 Subject: [PATCH 019/155] YARN-6476. Advanced Federation UI based on YARN UI v2. (#5920) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../hadoop/yarn/server/router/Router.java | 50 ++++++++++++++++++- .../hadoop/yarn/server/router/TestRouter.java | 2 +- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 62153242f7cf3..56bb91db52667 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.URL; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -38,6 +39,7 @@ import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.webapp.WebApps.Builder; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebServiceClient; +import org.eclipse.jetty.webapp.WebAppContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,6 +109,8 @@ public class Router extends CompositeService { private static final String METRICS_NAME = "Router"; + private static final String UI2_WEBAPP_NAME = "/ui2"; + private ScheduledThreadPoolExecutor scheduledExecutorService; private SubClusterCleaner subClusterCleaner; @@ -226,7 +231,50 @@ public void startWepApp() { String[] proxyParts = proxyHostAndPort.split(":"); builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); } - webApp = builder.start(new RouterWebApp(this)); + webApp = builder.start(new RouterWebApp(this), getUIWebAppContext()); + } + + private WebAppContext getUIWebAppContext() { + WebAppContext uiWebAppContext = null; + boolean isWebUI2Enabled = conf.getBoolean(YarnConfiguration.YARN_WEBAPP_UI2_ENABLE, + YarnConfiguration.DEFAULT_YARN_WEBAPP_UI2_ENABLE); + + if(isWebUI2Enabled) { + String onDiskPath = conf.get(YarnConfiguration.YARN_WEBAPP_UI2_WARFILE_PATH); + uiWebAppContext = new WebAppContext(); + uiWebAppContext.setContextPath(UI2_WEBAPP_NAME); + + if (null == onDiskPath) { + String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war"; + URL url = getClass().getClassLoader().getResource(war); + if (null == url) { + onDiskPath = getWebAppsPath("ui2"); + } else { + onDiskPath = url.getFile(); + } + } + + if (onDiskPath == null || onDiskPath.isEmpty()) { + LOG.error("No war file or webapps found for yarn federation!"); + } else { + if (onDiskPath.endsWith(".war")) { + uiWebAppContext.setWar(onDiskPath); + LOG.info("Using war file at: {}.", onDiskPath); + } else { + uiWebAppContext.setResourceBase(onDiskPath); + LOG.info("Using webapps at: {}.", onDiskPath); + } + } + } + return uiWebAppContext; + } + + private String getWebAppsPath(String appName) { + URL url = getClass().getClassLoader().getResource("webapps/" + appName); + if (url == null) { + return ""; + } + return url.toString(); } public static String getProxyHostAndPort(Configuration conf) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java index 2bc202f61866d..5fb5e045e6630 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java @@ -143,7 +143,7 @@ public void testRouterSupportCrossOrigin() throws ServletException, IOException WebAppContext webAppContext = httpServer2.getWebAppContext(); ServletHandler servletHandler = webAppContext.getServletHandler(); FilterHolder holder = servletHandler.getFilter("Cross Origin Filter"); - CrossOriginFilter filter = CrossOriginFilter.class.cast(holder.getFilter()); + CrossOriginFilter filter = (CrossOriginFilter) holder.getFilter(); // 1. Simulate [example.com] for access HttpServletRequest mockReq = Mockito.mock(HttpServletRequest.class); From 81d90fd65b3c0c7ac66ebae78e22dcb240a0547b Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 11 Sep 2023 14:30:25 +0100 Subject: [PATCH 020/155] HADOOP-18073. S3A: Upgrade AWS SDK to V2 (#5995) This patch migrates the S3A connector to use the V2 AWS SDK. This is a significant change at the source code level. Any applications using the internal extension/override points in the filesystem connector are likely to break. This includes but is not limited to: - Code invoking methods on the S3AFileSystem class which used classes from the V1 SDK. - The ability to define the factory for the `AmazonS3` client, and to retrieve it from the S3AFileSystem. There is a new factory API and a special interface S3AInternals to access a limited set of internal classes and operations. - Delegation token and auditing extensions. - Classes trying to integrate with the AWS SDK. All standard V1 credential providers listed in the option fs.s3a.aws.credentials.provider will be automatically remapped to their V2 equivalent. Other V1 Credential Providers are supported, but only if the V1 SDK is added back to the classpath. The SDK Signing plugin has changed; all v1 signers are incompatible. There is no support for the S3 "v2" signing algorithm. Finally, the aws-sdk-bundle JAR has been replaced by the shaded V2 equivalent, "bundle.jar", which is now exported by the hadoop-aws module. Consult the document aws_sdk_upgrade for the full details. Contributed by Ahmar Suhail + some bits by Steve Loughran --- LICENSE-binary | 1 + .../fs/statistics/StoreStatisticNames.java | 4 + .../src/main/resources/core-default.xml | 52 +- hadoop-project/pom.xml | 22 +- .../dev-support/findbugs-exclude.xml | 5 + hadoop-tools/hadoop-aws/pom.xml | 16 +- .../hadoop/fs/s3a/AWSBadRequestException.java | 4 +- .../hadoop/fs/s3a/AWSClientIOException.java | 18 +- .../fs/s3a/AWSCredentialProviderList.java | 110 +- .../hadoop/fs/s3a/AWSNoResponseException.java | 9 +- .../hadoop/fs/s3a/AWSRedirectException.java | 4 +- .../hadoop/fs/s3a/AWSS3IOException.java | 25 +- .../hadoop/fs/s3a/AWSServiceIOException.java | 37 +- .../fs/s3a/AWSServiceThrottledException.java | 9 +- .../hadoop/fs/s3a/AWSStatus500Exception.java | 9 +- .../s3a/AnonymousAWSCredentialsProvider.java | 18 +- .../org/apache/hadoop/fs/s3a/ArnResource.java | 12 +- .../org/apache/hadoop/fs/s3a/Constants.java | 16 +- .../CredentialInitializationException.java | 18 +- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 430 ++----- .../hadoop/fs/s3a/FailureInjectionPolicy.java | 2 +- .../fs/s3a/InconsistentAmazonS3Client.java | 345 ------ .../fs/s3a/InconsistentS3ClientFactory.java | 54 - .../org/apache/hadoop/fs/s3a/Invoker.java | 21 +- .../org/apache/hadoop/fs/s3a/Listing.java | 53 +- .../apache/hadoop/fs/s3a/MultipartUtils.java | 49 +- .../fs/s3a/ProgressableProgressListener.java | 46 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 124 +- .../apache/hadoop/fs/s3a/S3ADataBlocks.java | 8 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 1099 ++++++++++------- .../apache/hadoop/fs/s3a/S3AInputStream.java | 116 +- .../apache/hadoop/fs/s3a/S3AInternals.java | 118 ++ .../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 56 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 681 +++------- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 187 ++- .../apache/hadoop/fs/s3a/S3ListRequest.java | 16 +- .../apache/hadoop/fs/s3a/S3ListResult.java | 61 +- .../hadoop/fs/s3a/S3ObjectAttributes.java | 27 - .../s3a/SharedInstanceCredentialProvider.java | 1 - .../fs/s3a/SimpleAWSCredentialsProvider.java | 30 +- .../org/apache/hadoop/fs/s3a/Statistic.java | 5 + .../s3a/TemporaryAWSCredentialsProvider.java | 7 +- .../org/apache/hadoop/fs/s3a/Tristate.java | 45 +- .../org/apache/hadoop/fs/s3a/UploadInfo.java | 12 +- .../hadoop/fs/s3a/WriteOperationHelper.java | 202 ++- .../apache/hadoop/fs/s3a/WriteOperations.java | 100 +- .../fs/s3a/adapter/AwsV1BindingSupport.java | 118 ++ .../V1ToV2AwsCredentialProviderAdapter.java | 165 +++ .../hadoop/fs/s3a/adapter/package-info.java | 31 + .../hadoop/fs/s3a/api/RequestFactory.java | 224 ++-- .../fs/s3a/audit/AWSAuditEventCallbacks.java | 105 +- .../fs/s3a/audit/AWSRequestAnalyzer.java | 135 +- .../hadoop/fs/s3a/audit/AuditIntegration.java | 54 +- .../hadoop/fs/s3a/audit/AuditManagerS3A.java | 21 +- .../fs/s3a/audit/S3AAuditConstants.java | 13 +- .../s3a/audit/impl/ActiveAuditManagerS3A.java | 419 ++++--- .../fs/s3a/audit/impl/LoggingAuditor.java | 125 +- .../s3a/audit/impl/NoopAuditManagerS3A.java | 16 +- .../audit/impl/S3AInternalAuditConstants.java | 16 +- .../auth/AbstractAWSCredentialProvider.java | 13 +- .../AbstractSessionCredentialsProvider.java | 28 +- .../auth/AssumedRoleCredentialProvider.java | 95 +- .../auth/CredentialProviderListFactory.java | 303 +++++ .../auth/IAMInstanceCredentialsProvider.java | 49 +- .../s3a/auth/MarshalledCredentialBinding.java | 58 +- .../auth/MarshalledCredentialProvider.java | 5 +- .../fs/s3a/auth/NoAuthWithAWSException.java | 2 +- .../hadoop/fs/s3a/auth/STSClientFactory.java | 141 ++- .../hadoop/fs/s3a/auth/SignerFactory.java | 114 ++ .../hadoop/fs/s3a/auth/SignerManager.java | 8 +- .../EncryptionSecretOperations.java | 38 +- .../s3a/auth/delegation/RoleTokenBinding.java | 2 +- .../auth/delegation/S3ADelegationTokens.java | 5 - .../auth/delegation/SessionTokenBinding.java | 61 +- .../fs/s3a/commit/AbstractS3ACommitter.java | 9 +- .../hadoop/fs/s3a/commit/PutTracker.java | 4 +- .../s3a/commit/files/SinglePendingCommit.java | 13 +- .../fs/s3a/commit/impl/CommitOperations.java | 54 +- .../s3a/commit/magic/MagicCommitTracker.java | 28 +- .../hadoop/fs/s3a/impl/AWSCannedACL.java | 43 + .../hadoop/fs/s3a/impl/AWSClientConfig.java | 394 ++++++ .../apache/hadoop/fs/s3a/impl/AWSHeaders.java | 98 ++ .../fs/s3a/impl/BulkDeleteRetryHandler.java | 12 +- .../fs/s3a/impl/ChangeDetectionPolicy.java | 125 +- .../hadoop/fs/s3a/impl/ChangeTracker.java | 71 +- .../impl/ConfigureShadedAWSSocketFactory.java | 13 +- .../hadoop/fs/s3a/impl/CopyOutcome.java | 80 -- .../hadoop/fs/s3a/impl/DeleteOperation.java | 17 +- .../hadoop/fs/s3a/impl/ErrorTranslation.java | 14 +- .../hadoop/fs/s3a/impl/HeaderProcessing.java | 206 +-- .../fs/s3a/impl/InstantiationIOException.java | 180 +++ .../hadoop/fs/s3a/impl/InternalConstants.java | 51 +- ...t.java => MultiObjectDeleteException.java} | 67 +- .../hadoop/fs/s3a/impl/NetworkBinding.java | 11 +- .../fs/s3a/impl/OperationCallbacks.java | 15 +- .../hadoop/fs/s3a/impl/ProgressListener.java | 26 + .../fs/s3a/impl/ProgressListenerEvent.java | 29 + .../hadoop/fs/s3a/impl/RenameOperation.java | 23 +- .../fs/s3a/impl/RequestFactoryImpl.java | 663 +++++----- .../fs/s3a/impl/S3AMultipartUploader.java | 31 +- .../hadoop/fs/s3a/impl/SDKStreamDrainer.java | 43 +- .../hadoop/fs/s3a/impl/V2Migration.java | 72 +- .../fs/s3a/prefetch/S3ARemoteObject.java | 54 +- .../s3a/prefetch/S3ARemoteObjectReader.java | 7 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 8 +- .../fs/s3a/select/BlockingEnumeration.java | 156 +++ .../hadoop/fs/s3a/select/SelectBinding.java | 126 +- .../select/SelectEventStreamPublisher.java | 124 ++ .../fs/s3a/select/SelectInputStream.java | 32 +- .../s3a/select/SelectObjectContentHelper.java | 114 ++ .../impl/AwsStatisticsCollector.java | 166 ++- .../hadoop/fs/s3a/tools/MarkerTool.java | 17 +- .../fs/s3a/tools/MarkerToolOperations.java | 13 +- .../s3a/tools/MarkerToolOperationsImpl.java | 11 +- .../tools/hadoop-aws/assumed_roles.md | 2 +- .../markdown/tools/hadoop-aws/auditing.md | 48 +- .../tools/hadoop-aws/aws_sdk_upgrade.md | 340 ++++- .../tools/hadoop-aws/aws_sdk_v2_changelog.md | 340 +++++ .../tools/hadoop-aws/delegation_tokens.md | 2 +- .../site/markdown/tools/hadoop-aws/index.md | 142 +-- .../markdown/tools/hadoop-aws/s3_select.md | 43 +- .../site/markdown/tools/hadoop-aws/testing.md | 69 +- .../tools/hadoop-aws/troubleshooting_s3a.md | 95 +- .../hadoop/fs/s3a/AbstractS3AMockTest.java | 30 +- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 8 + .../hadoop/fs/s3a/EncryptionTestUtils.java | 20 +- .../s3a/ITestS3AAWSCredentialsProvider.java | 164 ++- .../fs/s3a/ITestS3ABucketExistence.java | 41 +- .../hadoop/fs/s3a/ITestS3ACannedACLs.java | 31 +- .../s3a/ITestS3AClientSideEncryptionKms.java | 6 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 236 ++-- .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 2 +- .../ITestS3AEncryptionSSEKMSDefaultKey.java | 8 +- ...estS3AEncryptionWithDefaultS3Settings.java | 3 +- .../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 200 ++- .../fs/s3a/ITestS3AFailureHandling.java | 28 +- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 35 +- .../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 5 +- .../hadoop/fs/s3a/ITestS3ARequesterPays.java | 2 +- .../hadoop/fs/s3a/ITestS3AStorageClass.java | 1 - .../fs/s3a/ITestS3ATemporaryCredentials.java | 30 +- .../hadoop/fs/s3a/MockS3AFileSystem.java | 17 +- .../hadoop/fs/s3a/MockS3ClientFactory.java | 49 +- .../hadoop/fs/s3a/MultipartTestUtils.java | 35 +- .../hadoop/fs/s3a/S3ATestConstants.java | 5 + .../apache/hadoop/fs/s3a/S3ATestUtils.java | 17 +- .../apache/hadoop/fs/s3a/TestArnResource.java | 9 +- .../org/apache/hadoop/fs/s3a/TestInvoker.java | 105 +- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 266 ++-- .../fs/s3a/TestS3ABlockOutputStream.java | 11 +- .../hadoop/fs/s3a/TestS3ADeleteOnExit.java | 25 +- .../fs/s3a/TestS3AExceptionTranslation.java | 208 +++- .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 95 +- .../fs/s3a/TestS3AInputStreamRetry.java | 130 +- .../apache/hadoop/fs/s3a/TestS3AProxy.java | 18 +- .../apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 50 +- .../fs/s3a/TestStreamChangeTracker.java | 107 +- .../fs/s3a/TestWildflyAndOpenSSLBinding.java | 9 +- .../adapter/TestV1CredentialsProvider.java | 222 ++++ .../fs/s3a/audit/AbstractAuditingTest.java | 98 +- .../hadoop/fs/s3a/audit/AuditTestSupport.java | 4 +- .../fs/s3a/audit/ITestAuditManager.java | 18 +- ...ava => SimpleAWSExecutionInterceptor.java} | 37 +- .../fs/s3a/audit/TestAuditIntegration.java | 82 +- .../fs/s3a/audit/TestAuditSpanLifecycle.java | 11 +- .../audit/TestHttpReferrerAuditHeader.java | 55 +- .../fs/s3a/audit/TestLoggingAuditor.java | 41 +- .../hadoop/fs/s3a/auth/ITestAssumeRole.java | 36 +- .../hadoop/fs/s3a/auth/ITestCustomSigner.java | 47 +- .../hadoop/fs/s3a/auth/RoleTestUtils.java | 7 +- .../s3a/auth/TestMarshalledCredentials.java | 10 +- .../hadoop/fs/s3a/auth/TestSignerManager.java | 41 +- .../delegation/CountInvocationsProvider.java | 34 +- .../ITestSessionDelegationInFilesystem.java | 30 +- .../ITestSessionDelegationTokens.java | 28 +- .../TestS3ADelegationTokenSupport.java | 1 - .../s3a/commit/AbstractITCommitProtocol.java | 4 +- .../s3a/commit/staging/StagingTestBase.java | 178 ++- .../staging/TestDirectoryCommitterScale.java | 7 +- .../commit/staging/TestStagingCommitter.java | 31 +- .../TestStagingPartitionedTaskCommit.java | 9 +- .../ITestS3AFileContextStatistics.java | 3 +- .../s3a/impl/ITestPartialRenamesDeletes.java | 1 - .../fs/s3a/impl/ITestRenameDeleteRace.java | 5 +- .../fs/s3a/impl/TestHeaderProcessing.java | 32 +- .../fs/s3a/impl/TestNetworkBinding.java | 43 - .../fs/s3a/impl/TestRequestFactory.java | 109 +- .../fs/s3a/impl/TestSDKStreamDrainer.java | 16 +- .../ITestDirectoryMarkerListing.java | 25 +- .../fs/s3a/prefetch/MockS3ARemoteObject.java | 27 +- .../fs/s3a/prefetch/S3APrefetchFakes.java | 48 +- .../s3a/scale/AbstractSTestS3AHugeFiles.java | 25 +- .../ILoadTestS3ABulkDeleteThrottling.java | 13 +- .../scale/ITestS3ADirectoryPerformance.java | 22 +- .../scale/ITestS3AHugeFilesStorageClass.java | 4 +- .../fs/s3a/select/AbstractS3SelectTest.java | 4 +- .../fs/s3a/select/ITestS3SelectLandsat.java | 2 +- .../hadoop/fs/s3a/select/StreamPublisher.java | 89 ++ .../s3a/select/TestBlockingEnumeration.java | 200 +++ .../TestSelectEventStreamPublisher.java | 190 +++ .../ITestAWSStatisticCollection.java | 4 +- .../hadoop/fs/s3a/test/ExtraAssertions.java | 2 +- .../s3a/test/MinimalOperationCallbacks.java | 14 +- .../MinimalWriteOperationHelperCallbacks.java | 16 +- .../hadoop/fs/s3a/tools/ITestMarkerTool.java | 7 - 205 files changed, 8987 insertions(+), 5900 deletions(-) delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{MultiObjectDeleteSupport.java => MultiObjectDeleteException.java} (62%) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListener.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListenerEvent.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/BlockingEnumeration.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectEventStreamPublisher.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectObjectContentHelper.java create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_v2_changelog.md create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java rename hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/{SimpleAWSRequestHandler.java => SimpleAWSExecutionInterceptor.java} (56%) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestBlockingEnumeration.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java diff --git a/LICENSE-binary b/LICENSE-binary index e7f9edbdf76d0..b20d382f7bd18 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -364,6 +364,7 @@ org.objenesis:objenesis:2.6 org.xerial.snappy:snappy-java:1.1.10.1 org.yaml:snakeyaml:2.0 org.wildfly.openssl:wildfly-openssl:1.1.3.Final +software.amazon.awssdk:bundle:jar:2.20.128 -------------------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java index c04c1bb47fcea..3a8927aba493e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java @@ -407,6 +407,10 @@ public final class StoreStatisticNames { public static final String MULTIPART_UPLOAD_LIST = "multipart_upload_list"; + /** Probe for store region: {@value}. */ + public static final String STORE_REGION_PROBE + = "store_region_probe"; + private StoreStatisticNames() { } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 14ffe3d9dec0e..9564a56e9e40d 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1387,17 +1387,24 @@ AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication. + + fs.s3a.session.token + Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider + as one of the providers. + + + fs.s3a.aws.credentials.provider org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, - com.amazonaws.auth.EnvironmentVariableCredentialsProvider, + software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider Comma-separated class names of credential provider classes which implement - com.amazonaws.auth.AWSCredentialsProvider. + software.amazon.awssdk.auth.credentials.AwsCredentialsProvider. When S3A delegation tokens are not enabled, this list will be used to directly authenticate with S3 and other AWS services. @@ -1405,43 +1412,6 @@ token binding it may be used to communicate wih the STS endpoint to request session/role credentials. - - These are loaded and queried in sequence for a valid set of credentials. - Each listed class must implement one of the following means of - construction, which are attempted in order: - * a public constructor accepting java.net.URI and - org.apache.hadoop.conf.Configuration, - * a public constructor accepting org.apache.hadoop.conf.Configuration, - * a public static method named getInstance that accepts no - arguments and returns an instance of - com.amazonaws.auth.AWSCredentialsProvider, or - * a public default constructor. - - Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows - anonymous access to a publicly accessible S3 bucket without any credentials. - Please note that allowing anonymous access to an S3 bucket compromises - security and therefore is unsuitable for most use cases. It can be useful - for accessing public data sets without requiring AWS credentials. - - If unspecified, then the default list of credential provider classes, - queried in sequence, is: - * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks - for session login secrets in the Hadoop configuration. - * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider: - Uses the values of fs.s3a.access.key and fs.s3a.secret.key. - * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports - configuration of AWS access key ID and secret access key in - environment variables named AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, - and AWS_SESSION_TOKEN as documented in the AWS SDK. - * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up - IAM credentials of any EC2 VM or AWS container in which the process is running. - - - - - fs.s3a.session.token - Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider - as one of the providers. @@ -1539,10 +1509,10 @@ Note: for job submission to actually collect these tokens, Kerberos must be enabled. - Options are: + Bindings available in hadoop-aws are: org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding - and org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding + org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9fca0fa159b87..cba562dd14036 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -184,6 +184,8 @@ 900 1.12.499 2.7.1 + 2.20.128 + 1.0.1 1.11.2 2.1 0.7 @@ -1128,15 +1130,31 @@ com.amazonaws - aws-java-sdk-bundle + aws-java-sdk-core ${aws-java-sdk.version} - io.netty + * + * + + + + + software.amazon.awssdk + bundle + ${aws-java-sdk-v2.version} + + + * * + + software.amazon.eventstream + eventstream + ${aws.eventstream.version} + org.apache.mina mina-core diff --git a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml index 39a9e51ac8125..359ac0e80dd1b 100644 --- a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml +++ b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml @@ -64,6 +64,11 @@ + + + + + diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 3bd973567c115..c5f921a874c1f 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -494,11 +494,25 @@ test test-jar + + com.amazonaws - aws-java-sdk-bundle + aws-java-sdk-core + provided + + + software.amazon.awssdk + bundle compile + + software.amazon.eventstream + eventstream + test + org.assertj assertj-core diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java index 482c5a1db7a1f..c5867eeab4f4d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; /** * A 400 "Bad Request" exception was received. @@ -36,7 +36,7 @@ public class AWSBadRequestException extends AWSServiceIOException { * @param cause the underlying cause */ public AWSBadRequestException(String operation, - AmazonServiceException cause) { + AwsServiceException cause) { super(operation, cause); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java index d3c5f888c7370..b61667d1c502b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java @@ -18,29 +18,28 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonClientException; -import com.amazonaws.SdkBaseException; +import software.amazon.awssdk.core.exception.SdkException; import org.apache.hadoop.util.Preconditions; import java.io.IOException; /** - * IOException equivalent of an {@link AmazonClientException}. + * IOException equivalent of an {@link SdkException}. */ public class AWSClientIOException extends IOException { private final String operation; public AWSClientIOException(String operation, - SdkBaseException cause) { + SdkException cause) { super(cause); Preconditions.checkArgument(operation != null, "Null 'operation' argument"); Preconditions.checkArgument(cause != null, "Null 'cause' argument"); this.operation = operation; } - public AmazonClientException getCause() { - return (AmazonClientException) super.getCause(); + public SdkException getCause() { + return (SdkException) super.getCause(); } @Override @@ -48,4 +47,11 @@ public String getMessage() { return operation + ": " + getCause().getMessage(); } + /** + * Query inner cause for retryability. + * @return what the cause says. + */ + public boolean retryable() { + return getCause().retryable(); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index f4d0a8d091249..d89795c68d3da 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -19,6 +19,8 @@ package org.apache.hadoop.fs.s3a; import java.io.Closeable; +import java.io.IOException; +import java.nio.file.AccessDeniedException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -27,21 +29,22 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import com.amazonaws.AmazonClientException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.Preconditions; + +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; /** * A list of providers. @@ -51,17 +54,17 @@ *
    *
  1. Allows extra providers to be added dynamically.
    2. *
  2. If any provider in the chain throws an exception other than - * an {@link AmazonClientException}, that is rethrown, rather than + * an {@link SdkException}, that is rethrown, rather than * swallowed.
    3. *
  3. Has some more diagnostics.
    4. - *
  4. On failure, the last "relevant" AmazonClientException raised is + *
  5. On failure, the last "relevant" {@link SdkException} raised is * rethrown; exceptions other than 'no credentials' have priority.
    6. - *
  6. Special handling of {@link AnonymousAWSCredentials}.
    7. + *
  7. Special handling of {@link AnonymousCredentialsProvider}.
    8. *
*/ -@InterfaceAudience.Private +@InterfaceAudience.LimitedPrivate("extensions") @InterfaceStability.Evolving -public final class AWSCredentialProviderList implements AWSCredentialsProvider, +public final class AWSCredentialProviderList implements AwsCredentialsProvider, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger( @@ -73,9 +76,9 @@ public final class AWSCredentialProviderList implements AWSCredentialsProvider, CREDENTIALS_REQUESTED_WHEN_CLOSED = "Credentials requested after provider list was closed"; - private final List providers = new ArrayList<>(1); + private final List providers = new ArrayList<>(1); private boolean reuseLastProvider = true; - private AWSCredentialsProvider lastProvider; + private AwsCredentialsProvider lastProvider; private final AtomicInteger refCount = new AtomicInteger(1); @@ -98,17 +101,17 @@ public AWSCredentialProviderList() { * @param providers provider list. */ public AWSCredentialProviderList( - Collection providers) { + Collection providers) { this.providers.addAll(providers); } /** - * Create with an initial list of providers. + * Create with an initial list of SDK V2 credential providers. * @param name name for error messages, may be "" * @param providerArgs provider list. */ public AWSCredentialProviderList(final String name, - final AWSCredentialsProvider... providerArgs) { + final AwsCredentialsProvider... providerArgs) { setName(name); Collections.addAll(providers, providerArgs); } @@ -126,11 +129,11 @@ public void setName(final String name) { } /** - * Add a new provider. - * @param p provider + * Add a new SDK V2 provider. + * @param provider provider */ - public void add(AWSCredentialsProvider p) { - providers.add(p); + public void add(AwsCredentialsProvider provider) { + providers.add(provider); } /** @@ -142,16 +145,11 @@ public void addAll(AWSCredentialProviderList other) { } /** - * Refresh all child entries. + * Was an implementation of the v1 refresh; now just + * a no-op. */ - @Override + @Deprecated public void refresh() { - if (isClosed()) { - return; - } - for (AWSCredentialsProvider provider : providers) { - provider.refresh(); - } } /** @@ -160,7 +158,7 @@ public void refresh() { * @return a set of credentials (possibly anonymous), for authenticating. */ @Override - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { if (isClosed()) { LOG.warn(CREDENTIALS_REQUESTED_WHEN_CLOSED); throw new NoAuthWithAWSException(name + @@ -168,18 +166,18 @@ public AWSCredentials getCredentials() { } checkNotEmpty(); if (reuseLastProvider && lastProvider != null) { - return lastProvider.getCredentials(); + return lastProvider.resolveCredentials(); } - AmazonClientException lastException = null; - for (AWSCredentialsProvider provider : providers) { + SdkException lastException = null; + for (AwsCredentialsProvider provider : providers) { try { - AWSCredentials credentials = provider.getCredentials(); + AwsCredentials credentials = provider.resolveCredentials(); Preconditions.checkNotNull(credentials, "Null credentials returned by %s", provider); - if ((credentials.getAWSAccessKeyId() != null && - credentials.getAWSSecretKey() != null) - || (credentials instanceof AnonymousAWSCredentials)) { + if ((credentials.accessKeyId() != null && credentials.secretAccessKey() != null) || ( + provider instanceof AnonymousCredentialsProvider + || provider instanceof AnonymousAWSCredentialsProvider)) { lastProvider = provider; LOG.debug("Using credentials from {}", provider); return credentials; @@ -196,7 +194,7 @@ public AWSCredentials getCredentials() { } LOG.debug("No credentials from {}: {}", provider, e.toString()); - } catch (AmazonClientException e) { + } catch (SdkException e) { lastException = e; LOG.debug("No credentials provided by {}: {}", provider, e.toString(), e); @@ -222,14 +220,13 @@ public AWSCredentials getCredentials() { * * @return providers */ - @VisibleForTesting - List getProviders() { + public List getProviders() { return providers; } /** * Verify that the provider list is not empty. - * @throws AmazonClientException if there are no providers. + * @throws SdkException if there are no providers. */ public void checkNotEmpty() { if (providers.isEmpty()) { @@ -255,9 +252,11 @@ public String listProviderNames() { */ @Override public String toString() { - return "AWSCredentialProviderList[" + - name + - "refcount= " + refCount.get() + ": [" + + return "AWSCredentialProviderList" + + " name=" + name + + "; refcount= " + refCount.get() + + "; size="+ providers.size() + + ": [" + StringUtils.join(providers, ", ") + ']' + (lastProvider != null ? (" last provider: " + lastProvider) : ""); } @@ -317,7 +316,7 @@ public void close() { } // do this outside the synchronized block. - for (AWSCredentialsProvider p : providers) { + for (AwsCredentialsProvider p : providers) { if (p instanceof Closeable) { IOUtils.closeStream((Closeable) p); } else if (p instanceof AutoCloseable) { @@ -333,4 +332,27 @@ public void close() { public int size() { return providers.size(); } + + + /** + * Translate an exception if it or its inner exception is an + * {@link CredentialInitializationException}. + * If this condition is not met, null is returned. + * @param path path of operation. + * @param throwable exception + * @return a translated exception or null. + */ + public static IOException maybeTranslateCredentialException(String path, + Throwable throwable) { + if (throwable instanceof CredentialInitializationException) { + // the exception raised by AWSCredentialProvider list if the + // credentials were not accepted, + return (AccessDeniedException)new AccessDeniedException(path, null, + throwable.toString()).initCause(throwable); + } else if (throwable.getCause() instanceof CredentialInitializationException) { + return maybeTranslateCredentialException(path, throwable.getCause()); + } else { + return null; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java index e6a23b2361da9..b8562714b1aae 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java @@ -18,14 +18,19 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; /** * Status code 443, no response from server. This is considered idempotent. */ public class AWSNoResponseException extends AWSServiceIOException { public AWSNoResponseException(String operation, - AmazonServiceException cause) { + AwsServiceException cause) { super(operation, cause); } + + @Override + public boolean retryable() { + return true; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java index bb337ee5eebda..cb478482a8ed4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; /** * Request is redirected. @@ -32,7 +32,7 @@ public class AWSRedirectException extends AWSServiceIOException { * @param cause the underlying cause */ public AWSRedirectException(String operation, - AmazonServiceException cause) { + AwsServiceException cause) { super(operation, cause); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java index 014d217b6a4fb..de1dd8b4a7a18 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSS3IOException.java @@ -18,14 +18,13 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.AmazonS3Exception; +import software.amazon.awssdk.services.s3.model.S3Exception; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import java.util.Map; - /** - * Wrap a {@link AmazonS3Exception} as an IOE, relaying all + * Wrap a {@link S3Exception} as an IOE, relaying all * getters. */ @InterfaceAudience.Public @@ -38,24 +37,12 @@ public class AWSS3IOException extends AWSServiceIOException { * @param cause the underlying cause */ public AWSS3IOException(String operation, - AmazonS3Exception cause) { + S3Exception cause) { super(operation, cause); } - public AmazonS3Exception getCause() { - return (AmazonS3Exception) super.getCause(); - } - - public String getErrorResponseXml() { - return getCause().getErrorResponseXml(); - } - - public Map getAdditionalDetails() { - return getCause().getAdditionalDetails(); - } - - public String getExtendedRequestId() { - return getCause().getExtendedRequestId(); + public S3Exception getCause() { + return (S3Exception) super.getCause(); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java index a9c2c9840203f..434ec8df292ac 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceIOException.java @@ -18,13 +18,15 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; + /** * A specific exception from AWS operations. - * The exception must always be created with an {@link AmazonServiceException}. + * The exception must always be created with an {@link AwsServiceException}. * The attributes of this exception can all be directly accessed. */ @InterfaceAudience.Public @@ -37,36 +39,27 @@ public class AWSServiceIOException extends AWSClientIOException { * @param cause the underlying cause */ public AWSServiceIOException(String operation, - AmazonServiceException cause) { + AwsServiceException cause) { super(operation, cause); } - public AmazonServiceException getCause() { - return (AmazonServiceException) super.getCause(); - } - - public String getRequestId() { - return getCause().getRequestId(); + public AwsServiceException getCause() { + return (AwsServiceException) super.getCause(); } - public String getServiceName() { - return getCause().getServiceName(); + public String requestId() { + return getCause().requestId(); } - public String getErrorCode() { - return getCause().getErrorCode(); + public AwsErrorDetails awsErrorDetails() { + return getCause().awsErrorDetails(); } - public int getStatusCode() { - return getCause().getStatusCode(); + public int statusCode() { + return getCause().statusCode(); } - public String getRawResponseContent() { - return getCause().getRawResponseContent(); + public String extendedRequestId() { + return getCause().extendedRequestId(); } - - public boolean isRetryable() { - return getCause().isRetryable(); - } - } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java index 131cea7562242..5cd2eb9d320f3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; /** * Exception raised when a service was throttled. @@ -36,7 +36,12 @@ public class AWSServiceThrottledException extends AWSServiceIOException { * @param cause the underlying cause */ public AWSServiceThrottledException(String operation, - AmazonServiceException cause) { + AwsServiceException cause) { super(operation, cause); } + + @Override + public boolean retryable() { + return true; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java index 83be294fac7cd..ecfe5da1455cd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; /** * A 500 response came back from a service. @@ -31,7 +31,12 @@ */ public class AWSStatus500Exception extends AWSServiceIOException { public AWSStatus500Exception(String operation, - AmazonServiceException cause) { + AwsServiceException cause) { super(operation, cause); } + + @Override + public boolean retryable() { + return true; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index 564c03bf731d7..dcfc2a03b1232 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -18,9 +18,10 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.auth.AWSCredentials; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -35,23 +36,18 @@ * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Stable -@Deprecated -public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { +public class AnonymousAWSCredentialsProvider implements AwsCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"; - public AWSCredentials getCredentials() { - return new AnonymousAWSCredentials(); + public AwsCredentials resolveCredentials() { + return AnonymousCredentialsProvider.create().resolveCredentials(); } - public void refresh() {} - @Override public String toString() { return getClass().getSimpleName(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java index a85f26223ffcf..98745b295b5d3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java @@ -20,7 +20,7 @@ import javax.annotation.Nonnull; -import com.amazonaws.arn.Arn; +import software.amazon.awssdk.arns.Arn; /** * Represents an Arn Resource, this can be an accesspoint or bucket. @@ -126,14 +126,14 @@ public String getEndpoint() { public static ArnResource accessPointFromArn(String arn) throws IllegalArgumentException { Arn parsed = Arn.fromString(arn); - if (parsed.getRegion().isEmpty() || parsed.getAccountId().isEmpty() || - parsed.getResourceAsString().isEmpty()) { + if (!parsed.region().isPresent() || !parsed.accountId().isPresent() || + parsed.resourceAsString().isEmpty()) { throw new IllegalArgumentException( String.format("Access Point Arn %s has an invalid format or missing properties", arn)); } - String resourceName = parsed.getResource().getResource(); - return new ArnResource(resourceName, parsed.getAccountId(), parsed.getRegion(), - parsed.getPartition(), arn); + String resourceName = parsed.resource().resource(); + return new ArnResource(resourceName, parsed.accountId().get(), parsed.region().get(), + parsed.partition(), arn); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 4e35dc17317f6..02f496abde000 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -140,7 +140,6 @@ private Constants() { public static final String ASSUMED_ROLE_POLICY = "fs.s3a.assumed.role.policy"; - @SuppressWarnings("deprecation") public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT = SimpleAWSCredentialsProvider.NAME; @@ -597,7 +596,7 @@ private Constants() { public static final String SIGNING_ALGORITHM_STS = "fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_STS.toLowerCase() - + "signing-algorithm"; + + ".signing-algorithm"; public static final String S3N_FOLDER_SUFFIX = "_$folder$"; public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size"; @@ -739,14 +738,21 @@ private Constants() { public static final String STREAM_READ_GAUGE_INPUT_POLICY = "stream_read_gauge_input_policy"; + /** + * S3 Client Factory implementation class: {@value}. + * Unstable and incompatible between v1 and v2 SDK versions. + */ @InterfaceAudience.Private @InterfaceStability.Unstable public static final String S3_CLIENT_FACTORY_IMPL = "fs.s3a.s3.client.factory.impl"; + /** + * Default factory: + * {@code org.apache.hadoop.fs.s3a.DefaultS3ClientFactory}. + */ @InterfaceAudience.Private @InterfaceStability.Unstable - @SuppressWarnings("deprecation") public static final Class DEFAULT_S3_CLIENT_FACTORY_IMPL = DefaultS3ClientFactory.class; @@ -1299,4 +1305,8 @@ private Constants() { */ public static final int DEFAULT_PREFETCH_MAX_BLOCKS_COUNT = 4; + /** + * The bucket region header. + */ + public static final String BUCKET_REGION_HEADER = "x-amz-bucket-region"; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java index 2f0cfd37ad37c..c6a1b8fcc713b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonClientException; +import software.amazon.awssdk.core.exception.SdkClientException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -26,22 +26,22 @@ /** * Exception which Hadoop's AWSCredentialsProvider implementations should * throw when there is a problem with the credential setup. This - * is a subclass of {@link AmazonClientException} which sets - * {@link #isRetryable()} to false, so as to fail fast. + * is a subclass of {@link SdkClientException} which sets + * {@link #retryable()} to false, so as to fail fast. * This is used in credential providers and elsewhere. * When passed through {@code S3AUtils.translateException()} it - * is mapped to an AccessDeniedException. As a result, the Invoker - * code will automatically translate + * is mapped to an AccessDeniedException. */ @InterfaceAudience.Public @InterfaceStability.Stable -public class CredentialInitializationException extends AmazonClientException { +public class CredentialInitializationException extends SdkClientException { + public CredentialInitializationException(String message, Throwable t) { - super(message, t); + super(builder().message(message).cause(t)); } public CredentialInitializationException(String message) { - super(message); + super(builder().message(message)); } /** @@ -49,7 +49,7 @@ public CredentialInitializationException(String message) { * @return false, always. */ @Override - public boolean isRetryable() { + public boolean retryable() { return false; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index f724f86e4afcd..98c72d276628e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -20,32 +20,26 @@ import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.SdkClientException; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.regions.RegionUtils; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Builder; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.AmazonS3EncryptionClientV2Builder; -import com.amazonaws.services.s3.AmazonS3EncryptionV2; -import com.amazonaws.services.s3.S3ClientOptions; -import com.amazonaws.services.s3.internal.ServiceUtils; -import com.amazonaws.services.s3.model.CryptoConfigurationV2; -import com.amazonaws.services.s3.model.CryptoMode; -import com.amazonaws.services.s3.model.CryptoRangeGetMode; -import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; -import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; -import com.amazonaws.util.AwsHostNameUtils; -import com.amazonaws.util.RuntimeHttpUtils; -import org.apache.hadoop.util.Preconditions; -import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.s3a.impl.AWSClientConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3BaseClientBuilder; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.transfer.s3.S3TransferManager; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -54,15 +48,11 @@ import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector; import org.apache.hadoop.fs.store.LogExactlyOnce; -import static com.amazonaws.services.s3.Headers.REQUESTER_PAYS_HEADER; -import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; -import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CENTRAL_REGION; -import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING; -import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; -import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; -import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; -import static org.apache.hadoop.fs.s3a.S3AUtils.translateException; +import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3; + /** * The default {@link S3ClientFactory} implementation. @@ -71,12 +61,9 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -@SuppressWarnings("deprecation") public class DefaultS3ClientFactory extends Configured implements S3ClientFactory { - private static final String S3_SERVICE_NAME = "s3"; - private static final String REQUESTER_PAYS_HEADER_VALUE = "requester"; /** @@ -85,310 +72,161 @@ public class DefaultS3ClientFactory extends Configured protected static final Logger LOG = LoggerFactory.getLogger(DefaultS3ClientFactory.class); - /** - * A one-off warning of default region chains in use. - */ - private static final LogExactlyOnce WARN_OF_DEFAULT_REGION_CHAIN = - new LogExactlyOnce(LOG); - - /** - * Warning message printed when the SDK Region chain is in use. - */ - private static final String SDK_REGION_CHAIN_IN_USE = - "S3A filesystem client is using" - + " the SDK region resolution chain."; /** Exactly once log to inform about ignoring the AWS-SDK Warnings for CSE. */ private static final LogExactlyOnce IGNORE_CSE_WARN = new LogExactlyOnce(LOG); - /** Bucket name. */ - private String bucket; - - /** - * Create the client by preparing the AwsConf configuration - * and then invoking {@code buildAmazonS3Client()}. - */ @Override - public AmazonS3 createS3Client( + public S3Client createS3Client( final URI uri, final S3ClientCreationParameters parameters) throws IOException { - Configuration conf = getConf(); - bucket = uri.getHost(); - final ClientConfiguration awsConf = S3AUtils - .createAwsConf(conf, - bucket, - Constants.AWS_SERVICE_IDENTIFIER_S3); - // add any headers - parameters.getHeaders().forEach((h, v) -> - awsConf.addHeader(h, v)); - if (parameters.isRequesterPays()) { - // All calls must acknowledge requester will pay via header. - awsConf.addHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE); - } + Configuration conf = getConf(); + String bucket = uri.getHost(); + + ApacheHttpClient.Builder httpClientBuilder = AWSClientConfig + .createHttpClientBuilder(conf) + .proxyConfiguration(AWSClientConfig.createProxyConfiguration(conf, bucket)); + return configureClientBuilder(S3Client.builder(), parameters, conf, bucket) + .httpClientBuilder(httpClientBuilder) + .build(); + } - // When EXPERIMENTAL_AWS_INTERNAL_THROTTLING is false - // throttling is explicitly disabled on the S3 client so that - // all failures are collected in S3A instrumentation, and its - // retry policy is the only one used. - // This may cause problems in copy/rename. - awsConf.setUseThrottleRetries( - conf.getBoolean(EXPERIMENTAL_AWS_INTERNAL_THROTTLING, - EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT)); + @Override + public S3AsyncClient createS3AsyncClient( + final URI uri, + final S3ClientCreationParameters parameters) throws IOException { - if (!StringUtils.isEmpty(parameters.getUserAgentSuffix())) { - awsConf.setUserAgentSuffix(parameters.getUserAgentSuffix()); - } + Configuration conf = getConf(); + String bucket = uri.getHost(); + + NettyNioAsyncHttpClient.Builder httpClientBuilder = AWSClientConfig + .createAsyncHttpClientBuilder(conf) + .proxyConfiguration(AWSClientConfig.createAsyncProxyConfiguration(conf, bucket)); + + MultipartConfiguration multipartConfiguration = MultipartConfiguration.builder() + .minimumPartSizeInBytes(parameters.getMinimumPartSize()) + .thresholdInBytes(parameters.getMultiPartThreshold()) + .build(); + + return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket) + .httpClientBuilder(httpClientBuilder) + .multipartConfiguration(multipartConfiguration) + .multipartEnabled(true) + .build(); + } - // Get the encryption method for this bucket. - S3AEncryptionMethods encryptionMethods = - getEncryptionAlgorithm(bucket, conf); - try { - // If CSE is enabled then build a S3EncryptionClient. - if (S3AEncryptionMethods.CSE_KMS.getMethod() - .equals(encryptionMethods.getMethod())) { - return buildAmazonS3EncryptionClient( - awsConf, - parameters); - } else { - return buildAmazonS3Client( - awsConf, - parameters); - } - } catch (SdkClientException e) { - // SDK refused to build. - throw translateException("creating AWS S3 client", uri.toString(), e); - } + @Override + public S3TransferManager createS3TransferManager(final S3AsyncClient s3AsyncClient) { + return S3TransferManager.builder() + .s3Client(s3AsyncClient) + .build(); } /** - * Create an {@link AmazonS3} client of type - * {@link AmazonS3EncryptionV2} if CSE is enabled. - * - * @param awsConf AWS configuration. - * @param parameters parameters. - * - * @return new AmazonS3 client. - * @throws IOException if lookupPassword() has any problem. + * Configure a sync or async S3 client builder. + * This method handles all shared configuration. + * @param builder S3 client builder + * @param parameters parameter object + * @param conf configuration object + * @param bucket bucket name + * @return the builder object + * @param S3 client builder type + * @param S3 client type */ - protected AmazonS3 buildAmazonS3EncryptionClient( - final ClientConfiguration awsConf, - final S3ClientCreationParameters parameters) throws IOException { + private , ClientT> BuilderT configureClientBuilder( + BuilderT builder, S3ClientCreationParameters parameters, Configuration conf, String bucket) + throws IOException { - AmazonS3 client; - AmazonS3EncryptionClientV2Builder builder = - new AmazonS3EncryptionClientV2Builder(); - Configuration conf = getConf(); + Region region = parameters.getRegion(); + LOG.debug("Using region {}", region); - // CSE-KMS Method - String kmsKeyId = getS3EncryptionKey(bucket, conf, true); - // Check if kmsKeyID is not null - Preconditions.checkArgument(!StringUtils.isBlank(kmsKeyId), "CSE-KMS " - + "method requires KMS key ID. Use " + S3_ENCRYPTION_KEY - + " property to set it. "); - - EncryptionMaterialsProvider materialsProvider = - new KMSEncryptionMaterialsProvider(kmsKeyId); - builder.withEncryptionMaterialsProvider(materialsProvider); - //Configure basic params of a S3 builder. - configureBasicParams(builder, awsConf, parameters); - - // Configuring endpoint. - AmazonS3EncryptionClientV2Builder.EndpointConfiguration epr - = createEndpointConfiguration(parameters.getEndpoint(), - awsConf, getConf().getTrimmed(AWS_REGION)); - configureEndpoint(builder, epr); - - // Create cryptoConfig. - CryptoConfigurationV2 cryptoConfigurationV2 = - new CryptoConfigurationV2(CryptoMode.AuthenticatedEncryption) - .withRangeGetMode(CryptoRangeGetMode.ALL); - if (epr != null) { - cryptoConfigurationV2 - .withAwsKmsRegion(RegionUtils.getRegion(epr.getSigningRegion())); - LOG.debug("KMS region used: {}", cryptoConfigurationV2.getAwsKmsRegion()); + URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf); + + if (endpoint != null) { + builder.endpointOverride(endpoint); + LOG.debug("Using endpoint {}", endpoint); } - builder.withCryptoConfiguration(cryptoConfigurationV2); - client = builder.build(); - IGNORE_CSE_WARN.info("S3 client-side encryption enabled: Ignore S3-CSE " - + "Warnings."); - return client; - } + S3Configuration serviceConfiguration = S3Configuration.builder() + .pathStyleAccessEnabled(parameters.isPathStyleAccess()) + .build(); - /** - * Use the Builder API to create an AWS S3 client. - *

- * This has a more complex endpoint configuration mechanism - * which initially caused problems; the - * {@code withForceGlobalBucketAccessEnabled(true)} - * command is critical here. - * @param awsConf AWS configuration - * @param parameters parameters - * @return new AmazonS3 client - * @throws SdkClientException if the configuration is invalid. - */ - protected AmazonS3 buildAmazonS3Client( - final ClientConfiguration awsConf, - final S3ClientCreationParameters parameters) { - AmazonS3ClientBuilder b = AmazonS3Client.builder(); - configureBasicParams(b, awsConf, parameters); - - // endpoint set up is a PITA - AwsClientBuilder.EndpointConfiguration epr - = createEndpointConfiguration(parameters.getEndpoint(), - awsConf, getConf().getTrimmed(AWS_REGION)); - configureEndpoint(b, epr); - final AmazonS3 client = b.build(); - return client; + return builder + .overrideConfiguration(createClientOverrideConfiguration(parameters, conf)) + .credentialsProvider(parameters.getCredentialSet()) + .region(region) + .serviceConfiguration(serviceConfiguration); } /** - * A method to configure basic AmazonS3Builder parameters. - * - * @param builder Instance of AmazonS3Builder used. - * @param awsConf ClientConfiguration used. - * @param parameters Parameters used to set in the builder. + * Create an override configuration for an S3 client. + * @param parameters parameter object + * @param conf configuration object + * @throws IOException any IOE raised, or translated exception + * @return the override configuration */ - private void configureBasicParams(AmazonS3Builder builder, - ClientConfiguration awsConf, S3ClientCreationParameters parameters) { - builder.withCredentials(parameters.getCredentialSet()); - builder.withClientConfiguration(awsConf); - builder.withPathStyleAccessEnabled(parameters.isPathStyleAccess()); + protected ClientOverrideConfiguration createClientOverrideConfiguration( + S3ClientCreationParameters parameters, Configuration conf) throws IOException { + final ClientOverrideConfiguration.Builder clientOverrideConfigBuilder = + AWSClientConfig.createClientConfigBuilder(conf, AWS_SERVICE_IDENTIFIER_S3); - if (parameters.getMetrics() != null) { - builder.withMetricsCollector( - new AwsStatisticsCollector(parameters.getMetrics())); - } - if (parameters.getRequestHandlers() != null) { - builder.withRequestHandlers( - parameters.getRequestHandlers().toArray(new RequestHandler2[0])); - } - if (parameters.getMonitoringListener() != null) { - builder.withMonitoringListener(parameters.getMonitoringListener()); - } + // add any headers + parameters.getHeaders().forEach((h, v) -> clientOverrideConfigBuilder.putHeader(h, v)); - } + if (parameters.isRequesterPays()) { + // All calls must acknowledge requester will pay via header. + clientOverrideConfigBuilder.putHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE); + } - /** - * A method to configure endpoint and Region for an AmazonS3Builder. - * - * @param builder Instance of AmazonS3Builder used. - * @param epr EndpointConfiguration used to set in builder. - */ - private void configureEndpoint( - AmazonS3Builder builder, - AmazonS3Builder.EndpointConfiguration epr) { - if (epr != null) { - // an endpoint binding was constructed: use it. - builder.withEndpointConfiguration(epr); - } else { - // no idea what the endpoint is, so tell the SDK - // to work it out at the cost of an extra HEAD request - builder.withForceGlobalBucketAccessEnabled(true); - // HADOOP-17771 force set the region so the build process doesn't halt. - String region = getConf().getTrimmed(AWS_REGION, AWS_S3_CENTRAL_REGION); - LOG.debug("fs.s3a.endpoint.region=\"{}\"", region); - if (!region.isEmpty()) { - // there's either an explicit region or we have fallen back - // to the central one. - LOG.debug("Using default endpoint; setting region to {}", region); - builder.setRegion(region); - } else { - // no region. - // allow this if people really want it; it is OK to rely on this - // when deployed in EC2. - WARN_OF_DEFAULT_REGION_CHAIN.warn(SDK_REGION_CHAIN_IN_USE); - LOG.debug(SDK_REGION_CHAIN_IN_USE); - } + if (!StringUtils.isEmpty(parameters.getUserAgentSuffix())) { + clientOverrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_SUFFIX, + parameters.getUserAgentSuffix()); } - } - /** - * Configure classic S3 client. - *

- * This includes: endpoint, Path Access and possibly other - * options. - * - * @param s3 S3 Client. - * @param endPoint s3 endpoint, may be empty - * @param pathStyleAccess enable path style access? - * @return S3 client - * @throws IllegalArgumentException if misconfigured - */ - protected static AmazonS3 configureAmazonS3Client(AmazonS3 s3, - final String endPoint, - final boolean pathStyleAccess) - throws IllegalArgumentException { - if (!endPoint.isEmpty()) { - try { - s3.setEndpoint(endPoint); - } catch (IllegalArgumentException e) { - String msg = "Incorrect endpoint: " + e.getMessage(); - LOG.error(msg); - throw new IllegalArgumentException(msg, e); + if (parameters.getExecutionInterceptors() != null) { + for (ExecutionInterceptor interceptor : parameters.getExecutionInterceptors()) { + clientOverrideConfigBuilder.addExecutionInterceptor(interceptor); } } - if (pathStyleAccess) { - LOG.debug("Enabling path style access!"); - s3.setS3ClientOptions(S3ClientOptions.builder() - .setPathStyleAccess(true) - .build()); + + if (parameters.getMetrics() != null) { + clientOverrideConfigBuilder.addMetricPublisher( + new AwsStatisticsCollector(parameters.getMetrics())); } - return s3; + + final RetryPolicy.Builder retryPolicyBuilder = AWSClientConfig.createRetryPolicyBuilder(conf); + clientOverrideConfigBuilder.retryPolicy(retryPolicyBuilder.build()); + + return clientOverrideConfigBuilder.build(); } /** - * Given an endpoint string, return an endpoint config, or null, if none - * is needed. - *

- * This is a pretty painful piece of code. It is trying to replicate - * what AwsClient.setEndpoint() does, because you can't - * call that setter on an AwsClient constructed via - * the builder, and you can't pass a metrics collector - * down except through the builder. - *

- * Note also that AWS signing is a mystery which nobody fully - * understands, especially given all problems surface in a - * "400 bad request" response, which, like all security systems, - * provides minimal diagnostics out of fear of leaking - * secrets. + * Given a endpoint string, create the endpoint URI. * * @param endpoint possibly null endpoint. - * @param awsConf config to build the URI from. - * @param awsRegion AWS S3 Region if the corresponding config is set. - * @return a configuration for the S3 client builder. + * @param conf config to build the URI from. + * @return an endpoint uri */ - @VisibleForTesting - public static AwsClientBuilder.EndpointConfiguration - createEndpointConfiguration( - final String endpoint, final ClientConfiguration awsConf, - String awsRegion) { - LOG.debug("Creating endpoint configuration for \"{}\"", endpoint); + private static URI getS3Endpoint(String endpoint, final Configuration conf) { + + boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); + + String protocol = secureConnections ? "https" : "http"; + if (endpoint == null || endpoint.isEmpty()) { - // the default endpoint...we should be using null at this point. - LOG.debug("Using default endpoint -no need to generate a configuration"); + // don't set an endpoint if none is configured, instead let the SDK figure it out. return null; } - final URI epr = RuntimeHttpUtils.toUri(endpoint, awsConf); - LOG.debug("Endpoint URI = {}", epr); - String region = awsRegion; - if (StringUtils.isBlank(region)) { - if (!ServiceUtils.isS3USStandardEndpoint(endpoint)) { - LOG.debug("Endpoint {} is not the default; parsing", epr); - region = AwsHostNameUtils.parseRegion( - epr.getHost(), - S3_SERVICE_NAME); - } else { - // US-east, set region == null. - LOG.debug("Endpoint {} is the standard one; declare region as null", - epr); - region = null; - } + if (!endpoint.contains("://")) { + endpoint = String.format("%s://%s", protocol, endpoint); + } + + try { + return new URI(endpoint); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); } - LOG.debug("Region for endpoint {}, URI {} is determined as {}", - endpoint, epr, region); - return new AwsClientBuilder.EndpointConfiguration(endpoint, region); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java index cfd7046e8abfa..4bf81817efc5d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java @@ -36,7 +36,7 @@ public class FailureInjectionPolicy { public static final String DEFAULT_DELAY_KEY_SUBSTRING = "DELAY_LISTING_ME"; private static final Logger LOG = - LoggerFactory.getLogger(InconsistentAmazonS3Client.class); + LoggerFactory.getLogger(FailureInjectionPolicy.class); /** * Probability of throttling a request. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java deleted file mode 100644 index c6d17a32b64b1..0000000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a; - -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.SdkClientException; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.MultipartUploadListing; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; -import org.apache.hadoop.util.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; - -/** - * A wrapper around {@link com.amazonaws.services.s3.AmazonS3} that injects - * failures. - * It used to also inject inconsistency, but this was removed with S3Guard; - * what is retained is the ability to throttle AWS operations and for the - * input stream to be inconsistent. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class InconsistentAmazonS3Client extends AmazonS3Client { - - private static final Logger LOG = - LoggerFactory.getLogger(InconsistentAmazonS3Client.class); - - private FailureInjectionPolicy policy; - - /** - * Counter of failures since last reset. - */ - private final AtomicLong failureCounter = new AtomicLong(0); - - - /** - * Instantiate. - * This subclasses a deprecated constructor of the parent - * {@code AmazonS3Client} class; we can't use the builder API because, - * that only creates the consistent client. - * @param credentials credentials to auth. - * @param clientConfiguration connection settings - * @param conf hadoop configuration. - */ - @SuppressWarnings("deprecation") - public InconsistentAmazonS3Client(AWSCredentialsProvider credentials, - ClientConfiguration clientConfiguration, Configuration conf) { - super(credentials, clientConfiguration); - policy = new FailureInjectionPolicy(conf); - } - - /** - * A way for tests to patch in a different fault injection policy at runtime. - * @param fs filesystem under test - * @param policy failure injection settings to set - * @throws Exception on failure - */ - public static void setFailureInjectionPolicy(S3AFileSystem fs, - FailureInjectionPolicy policy) throws Exception { - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("s3guard"); - InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3); - ic.replacePolicy(policy); - } - - private void replacePolicy(FailureInjectionPolicy pol) { - this.policy = pol; - } - - @Override - public String toString() { - return String.format("Inconsistent S3 Client: %s; failure count %d", - policy, failureCounter.get()); - } - - /** - * Convenience function for test code to cast from supertype. - * @param c supertype to cast from - * @return subtype, not null - * @throws Exception on error - */ - public static InconsistentAmazonS3Client castFrom(AmazonS3 c) throws - Exception { - InconsistentAmazonS3Client ic = null; - if (c instanceof InconsistentAmazonS3Client) { - ic = (InconsistentAmazonS3Client) c; - } - Preconditions.checkNotNull(ic, "Not an instance of " + - "InconsistentAmazonS3Client"); - return ic; - } - - @Override - public DeleteObjectsResult deleteObjects(DeleteObjectsRequest - deleteObjectsRequest) - throws AmazonClientException, AmazonServiceException { - maybeFail(); - return super.deleteObjects(deleteObjectsRequest); - } - - @Override - public void deleteObject(DeleteObjectRequest deleteObjectRequest) - throws AmazonClientException, AmazonServiceException { - String key = deleteObjectRequest.getKey(); - LOG.debug("key {}", key); - maybeFail(); - super.deleteObject(deleteObjectRequest); - } - - /* We should only need to override this version of putObject() */ - @Override - public PutObjectResult putObject(PutObjectRequest putObjectRequest) - throws AmazonClientException, AmazonServiceException { - LOG.debug("key {}", putObjectRequest.getKey()); - maybeFail(); - return super.putObject(putObjectRequest); - } - - /* We should only need to override these versions of listObjects() */ - @Override - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) - throws AmazonClientException, AmazonServiceException { - maybeFail(); - return super.listObjects(listObjectsRequest); - } - - /* consistent listing with possibility of failing. */ - @Override - public ListObjectsV2Result listObjectsV2(ListObjectsV2Request request) - throws AmazonClientException, AmazonServiceException { - maybeFail(); - return super.listObjectsV2(request); - } - - - @Override - public CompleteMultipartUploadResult completeMultipartUpload( - CompleteMultipartUploadRequest completeMultipartUploadRequest) - throws SdkClientException, AmazonServiceException { - maybeFail(); - return super.completeMultipartUpload(completeMultipartUploadRequest); - } - - @Override - public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest) - throws SdkClientException, AmazonServiceException { - maybeFail(); - return super.uploadPart(uploadPartRequest); - } - - @Override - public InitiateMultipartUploadResult initiateMultipartUpload( - InitiateMultipartUploadRequest initiateMultipartUploadRequest) - throws SdkClientException, AmazonServiceException { - maybeFail(); - return super.initiateMultipartUpload(initiateMultipartUploadRequest); - } - - @Override - public MultipartUploadListing listMultipartUploads( - ListMultipartUploadsRequest listMultipartUploadsRequest) - throws SdkClientException, AmazonServiceException { - maybeFail(); - return super.listMultipartUploads(listMultipartUploadsRequest); - } - - /** - * Set the probability of throttling a request. - * @param throttleProbability the probability of a request being throttled. - */ - public void setThrottleProbability(float throttleProbability) { - policy.setThrottleProbability(throttleProbability); - } - - /** - * Conditionally fail the operation. - * @param errorMsg description of failure - * @param statusCode http status code for error - * @throws AmazonClientException if the client chooses to fail - * the request. - */ - private void maybeFail(String errorMsg, int statusCode) - throws AmazonClientException { - // code structure here is to line up for more failures later - AmazonServiceException ex = null; - if (FailureInjectionPolicy.trueWithProbability(policy.getThrottleProbability())) { - // throttle the request - ex = new AmazonServiceException(errorMsg - + " count = " + (failureCounter.get() + 1), null); - ex.setStatusCode(statusCode); - } - - int failureLimit = policy.getFailureLimit(); - if (ex != null) { - long count = failureCounter.incrementAndGet(); - if (failureLimit == 0 - || (failureLimit > 0 && count < failureLimit)) { - throw ex; - } - } - } - - private void maybeFail() { - maybeFail("throttled", 503); - } - - /** - * Set the limit on failures before all operations pass through. - * This resets the failure count. - * @param limit limit; "0" means "no limit" - */ - public void setFailureLimit(int limit) { - policy.setFailureLimit(limit); - failureCounter.set(0); - } - - @Override - public S3Object getObject(GetObjectRequest var1) throws SdkClientException, - AmazonServiceException { - maybeFail(); - return super.getObject(var1); - } - - @Override - public S3Object getObject(String bucketName, String key) - throws SdkClientException, AmazonServiceException { - maybeFail(); - return super.getObject(bucketName, key); - - } - - /** Since ObjectListing is immutable, we just override it with wrapper. */ - @SuppressWarnings("serial") - private static class CustomObjectListing extends ObjectListing { - - private final List customListing; - private final List customPrefixes; - - CustomObjectListing(ObjectListing rawListing, - List customListing, - List customPrefixes) { - super(); - this.customListing = customListing; - this.customPrefixes = customPrefixes; - - this.setBucketName(rawListing.getBucketName()); - this.setCommonPrefixes(rawListing.getCommonPrefixes()); - this.setDelimiter(rawListing.getDelimiter()); - this.setEncodingType(rawListing.getEncodingType()); - this.setMarker(rawListing.getMarker()); - this.setMaxKeys(rawListing.getMaxKeys()); - this.setNextMarker(rawListing.getNextMarker()); - this.setPrefix(rawListing.getPrefix()); - this.setTruncated(rawListing.isTruncated()); - } - - @Override - public List getObjectSummaries() { - return customListing; - } - - @Override - public List getCommonPrefixes() { - return customPrefixes; - } - } - - @SuppressWarnings("serial") - private static class CustomListObjectsV2Result extends ListObjectsV2Result { - - private final List customListing; - private final List customPrefixes; - - CustomListObjectsV2Result(ListObjectsV2Result raw, - List customListing, List customPrefixes) { - super(); - this.customListing = customListing; - this.customPrefixes = customPrefixes; - - this.setBucketName(raw.getBucketName()); - this.setCommonPrefixes(raw.getCommonPrefixes()); - this.setDelimiter(raw.getDelimiter()); - this.setEncodingType(raw.getEncodingType()); - this.setStartAfter(raw.getStartAfter()); - this.setMaxKeys(raw.getMaxKeys()); - this.setContinuationToken(raw.getContinuationToken()); - this.setPrefix(raw.getPrefix()); - this.setTruncated(raw.isTruncated()); - } - - @Override - public List getObjectSummaries() { - return customListing; - } - - @Override - public List getCommonPrefixes() { - return customPrefixes; - } - } -} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java deleted file mode 100644 index 4bfcc8aba3af3..0000000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.services.s3.AmazonS3; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * S3 Client factory used for testing with eventual consistency fault injection. - * This client is for testing only; it is in the production - * {@code hadoop-aws} module to enable integration tests to use this - * just by editing the Hadoop configuration used to bring up the client. - * - * The factory uses the older constructor-based instantiation/configuration - * of the client, so does not wire up metrics, handlers etc. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class InconsistentS3ClientFactory extends DefaultS3ClientFactory { - - @Override - protected AmazonS3 buildAmazonS3Client( - final ClientConfiguration awsConf, - final S3ClientCreationParameters parameters) { - LOG.warn("** FAILURE INJECTION ENABLED. Do not run in production! **"); - LOG.warn("List inconsistency is no longer emulated; only throttling and read errors"); - InconsistentAmazonS3Client s3 - = new InconsistentAmazonS3Client( - parameters.getCredentialSet(), awsConf, getConf()); - configureAmazonS3Client(s3, - parameters.getEndpoint(), - parameters.isPathStyleAccess()); - return s3; - } -} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java index 67c8e7d809cc8..9b2c95a90c76f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java @@ -24,8 +24,7 @@ import java.util.concurrent.Future; import javax.annotation.Nullable; -import com.amazonaws.AmazonClientException; -import com.amazonaws.SdkBaseException; +import software.amazon.awssdk.core.exception.SdkException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +38,7 @@ import org.apache.hadoop.util.functional.InvocationRaisingIOE; import org.apache.hadoop.util.Preconditions; + import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration; /** @@ -120,7 +120,7 @@ public static T once(String action, String path, throws IOException { try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) { return operation.apply(); - } catch (AmazonClientException e) { + } catch (SdkException e) { throw S3AUtils.translateException(action, path, e); } } @@ -145,7 +145,7 @@ public static T onceTrackingDuration( throws IOException { try { return invokeTrackingDuration(tracker, operation); - } catch (AmazonClientException e) { + } catch (SdkException e) { throw S3AUtils.translateException(action, path, e); } } @@ -170,7 +170,7 @@ public static void once(String action, String path, /** * - * Wait for a future, translating AmazonClientException into an IOException. + * Wait for a future, translating SdkException into an IOException. * @param action action to execute (used in error messages) * @param path path of work (used in error messages) * @param future future to await for @@ -186,7 +186,7 @@ public static T onceInTheFuture(String action, throws IOException { try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) { return FutureIO.awaitFuture(future); - } catch (AmazonClientException e) { + } catch (SdkException e) { throw S3AUtils.translateException(action, path, e); } } @@ -444,7 +444,7 @@ public T retryUntranslated( * @param operation operation to execute * @return the result of the call * @throws IOException any IOE raised - * @throws SdkBaseException any AWS exception raised + * @throws SdkException any AWS exception raised * @throws RuntimeException : these are never caught and retries. */ @Retries.RetryRaw @@ -466,7 +466,7 @@ public T retryUntranslated( } // execute the operation, returning if successful return operation.apply(); - } catch (IOException | SdkBaseException e) { + } catch (IOException | SdkException e) { caught = e; } // you only get here if the operation didn't complete @@ -479,7 +479,7 @@ public T retryUntranslated( translated = (IOException) caught; } else { translated = S3AUtils.translateException(text, "", - (SdkBaseException)caught); + (SdkException) caught); } try { @@ -518,11 +518,10 @@ public T retryUntranslated( if (caught instanceof IOException) { throw (IOException) caught; } else { - throw (SdkBaseException) caught; + throw (SdkException) caught; } } - /** * Execute an operation; any exception raised is simply caught and * logged at debug. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index 6c39cc4b64240..490deaaab04d9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -18,7 +18,8 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.services.s3.model.CommonPrefix; +import software.amazon.awssdk.services.s3.model.S3Object; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.VisibleForTesting; @@ -277,19 +278,19 @@ public S3ListRequest createListObjectsRequest(String key, } /** - * Interface to implement by the logic deciding whether to accept a summary + * Interface to implement the logic deciding whether to accept a s3Object * entry or path as a valid file or directory. */ interface FileStatusAcceptor { /** - * Predicate to decide whether or not to accept a summary entry. + * Predicate to decide whether or not to accept a s3Object entry. * @param keyPath qualified path to the entry - * @param summary summary entry + * @param s3Object s3Object entry * @return true if the entry is accepted (i.e. that a status entry * should be generated. */ - boolean accept(Path keyPath, S3ObjectSummary summary); + boolean accept(Path keyPath, S3Object s3Object); /** * Predicate to decide whether or not to accept a prefix. @@ -451,21 +452,21 @@ private boolean buildNextStatusBatch(S3ListResult objects) { int added = 0, ignored = 0; // list to fill in with results. Initial size will be list maximum. List stats = new ArrayList<>( - objects.getObjectSummaries().size() + + objects.getS3Objects().size() + objects.getCommonPrefixes().size()); // objects - for (S3ObjectSummary summary : objects.getObjectSummaries()) { - String key = summary.getKey(); + for (S3Object s3Object : objects.getS3Objects()) { + String key = s3Object.key(); Path keyPath = getStoreContext().getContextAccessors().keyToPath(key); if (LOG.isDebugEnabled()) { - LOG.debug("{}: {}", keyPath, stringify(summary)); + LOG.debug("{}: {}", keyPath, stringify(s3Object)); } // Skip over keys that are ourselves and old S3N _$folder$ files - if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) { - S3AFileStatus status = createFileStatus(keyPath, summary, + if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) { + S3AFileStatus status = createFileStatus(keyPath, s3Object, listingOperationCallbacks.getDefaultBlockSize(keyPath), getStoreContext().getUsername(), - summary.getETag(), null, isCSEEnabled); + s3Object.eTag(), null, isCSEEnabled); LOG.debug("Adding: {}", status); stats.add(status); added++; @@ -476,11 +477,11 @@ private boolean buildNextStatusBatch(S3ListResult objects) { } // prefixes: always directories - for (String prefix : objects.getCommonPrefixes()) { + for (CommonPrefix prefix : objects.getCommonPrefixes()) { Path keyPath = getStoreContext() .getContextAccessors() - .keyToPath(prefix); - if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) { + .keyToPath(prefix.prefix()); + if (acceptor.accept(keyPath, prefix.prefix()) && filter.accept(keyPath)) { S3AFileStatus status = new S3AFileStatus(Tristate.FALSE, keyPath, getStoreContext().getUsername()); LOG.debug("Adding directory: {}", status); @@ -731,18 +732,18 @@ public AcceptFilesOnly(Path qualifiedPath) { } /** - * Reject a summary entry if the key path is the qualified Path, or + * Reject a s3Object entry if the key path is the qualified Path, or * it ends with {@code "_$folder$"}. * @param keyPath key path of the entry - * @param summary summary entry + * @param s3Object s3Object entry * @return true if the entry is accepted (i.e. that a status entry * should be generated. */ @Override - public boolean accept(Path keyPath, S3ObjectSummary summary) { + public boolean accept(Path keyPath, S3Object s3Object) { return !keyPath.equals(qualifiedPath) - && !summary.getKey().endsWith(S3N_FOLDER_SUFFIX) - && !objectRepresentsDirectory(summary.getKey()); + && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) + && !objectRepresentsDirectory(s3Object.key()); } /** @@ -767,8 +768,8 @@ public boolean accept(FileStatus status) { */ static class AcceptAllButS3nDirs implements FileStatusAcceptor { - public boolean accept(Path keyPath, S3ObjectSummary summary) { - return !summary.getKey().endsWith(S3N_FOLDER_SUFFIX); + public boolean accept(Path keyPath, S3Object s3Object) { + return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX); } public boolean accept(Path keyPath, String prefix) { @@ -799,17 +800,17 @@ public AcceptAllButSelfAndS3nDirs(Path qualifiedPath) { } /** - * Reject a summary entry if the key path is the qualified Path, or + * Reject a s3Object entry if the key path is the qualified Path, or * it ends with {@code "_$folder$"}. * @param keyPath key path of the entry - * @param summary summary entry + * @param s3Object s3Object entry * @return true if the entry is accepted (i.e. that a status entry * should be generated.) */ @Override - public boolean accept(Path keyPath, S3ObjectSummary summary) { + public boolean accept(Path keyPath, S3Object s3Object) { return !keyPath.equals(qualifiedPath) && - !summary.getKey().endsWith(S3N_FOLDER_SUFFIX); + !s3Object.key().endsWith(S3N_FOLDER_SUFFIX); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java index d8c820cd8a121..296ec18dcf18d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java @@ -23,13 +23,14 @@ import java.util.NoSuchElementException; import javax.annotation.Nullable; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.MultipartUploadListing; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse; +import software.amazon.awssdk.services.s3.model.MultipartUpload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.api.RequestFactory; import org.apache.hadoop.fs.s3a.impl.StoreContext; @@ -43,7 +44,7 @@ * MultipartUtils upload-specific functions for use by S3AFileSystem and Hadoop * CLI. * The Audit span active when - * {@link #listMultipartUploads(StoreContext, AmazonS3, String, int)} + * {@link #listMultipartUploads(StoreContext, S3Client, String, int)} * was invoked is retained for all subsequent operations. */ public final class MultipartUtils { @@ -67,7 +68,7 @@ private MultipartUtils() { } */ static MultipartUtils.UploadIterator listMultipartUploads( final StoreContext storeContext, - AmazonS3 s3, + S3Client s3, @Nullable String prefix, int maxKeys) throws IOException { @@ -84,14 +85,14 @@ static MultipartUtils.UploadIterator listMultipartUploads( * at the time the iterator was constructed. */ static class ListingIterator implements - RemoteIterator { + RemoteIterator { private final String prefix; private final RequestFactory requestFactory; private final int maxKeys; - private final AmazonS3 s3; + private final S3Client s3; private final Invoker invoker; private final AuditSpan auditSpan; @@ -101,7 +102,7 @@ static class ListingIterator implements /** * Most recent listing results. */ - private MultipartUploadListing listing; + private ListMultipartUploadsResponse listing; /** * Indicator that this is the first listing. @@ -114,7 +115,7 @@ static class ListingIterator implements private int listCount = 0; ListingIterator(final StoreContext storeContext, - AmazonS3 s3, + S3Client s3, @Nullable String prefix, int maxKeys) throws IOException { this.storeContext = storeContext; @@ -153,7 +154,7 @@ public boolean hasNext() throws IOException { */ @Override @Retries.RetryTranslated - public MultipartUploadListing next() throws IOException { + public ListMultipartUploadsResponse next() throws IOException { if (firstListing) { firstListing = false; } else { @@ -171,32 +172,34 @@ public MultipartUploadListing next() throws IOException { public String toString() { return "Upload iterator: prefix " + prefix + "; list count " + listCount - + "; upload count " + listing.getMultipartUploads().size() + + "; upload count " + listing.uploads().size() + "; isTruncated=" + listing.isTruncated(); } @Retries.RetryTranslated private void requestNextBatch() throws IOException { try (AuditSpan span = auditSpan.activate()) { - ListMultipartUploadsRequest req = requestFactory - .newListMultipartUploadsRequest(prefix); + ListMultipartUploadsRequest.Builder requestBuilder = requestFactory + .newListMultipartUploadsRequestBuilder(prefix); if (!firstListing) { - req.setKeyMarker(listing.getNextKeyMarker()); - req.setUploadIdMarker(listing.getNextUploadIdMarker()); + requestBuilder.keyMarker(listing.nextKeyMarker()); + requestBuilder.uploadIdMarker(listing.nextUploadIdMarker()); } - req.setMaxUploads(maxKeys); + requestBuilder.maxUploads(maxKeys); + + ListMultipartUploadsRequest request = requestBuilder.build(); LOG.debug("[{}], Requesting next {} uploads prefix {}, " + "next key {}, next upload id {}", listCount, maxKeys, prefix, - req.getKeyMarker(), req.getUploadIdMarker()); + request.keyMarker(), request.uploadIdMarker()); listCount++; listing = invoker.retry("listMultipartUploads", prefix, true, trackDurationOfOperation(storeContext.getInstrumentation(), MULTIPART_UPLOAD_LIST.getSymbol(), - () -> s3.listMultipartUploads(req))); + () -> s3.listMultipartUploads(requestBuilder.build()))); LOG.debug("Listing found {} upload(s)", - listing.getMultipartUploads().size()); + listing.uploads().size()); LOG.debug("New listing state: {}", this); } } @@ -216,14 +219,14 @@ public static class UploadIterator */ private ListingIterator lister; /** Current listing: the last upload listing we fetched. */ - private MultipartUploadListing listing; + private ListMultipartUploadsResponse listing; /** Iterator over the current listing. */ private ListIterator batchIterator; @Retries.RetryTranslated public UploadIterator( final StoreContext storeContext, - AmazonS3 s3, + S3Client s3, int maxKeys, @Nullable String prefix) throws IOException { @@ -249,7 +252,7 @@ public MultipartUpload next() throws IOException { private boolean requestNextBatch() throws IOException { if (lister.hasNext()) { listing = lister.next(); - batchIterator = listing.getMultipartUploads().listIterator(); + batchIterator = listing.uploads().listIterator(); return batchIterator.hasNext(); } return false; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java index 0ce022aa88588..7ee6c55c191b7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProgressableProgressListener.java @@ -18,59 +18,55 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.event.ProgressEvent; -import com.amazonaws.event.ProgressEventType; -import com.amazonaws.event.ProgressListener; -import com.amazonaws.services.s3.transfer.Upload; +import software.amazon.awssdk.transfer.s3.model.ObjectTransfer; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; import org.apache.hadoop.util.Progressable; import org.slf4j.Logger; -import static com.amazonaws.event.ProgressEventType.TRANSFER_COMPLETED_EVENT; -import static com.amazonaws.event.ProgressEventType.TRANSFER_PART_STARTED_EVENT; /** * Listener to progress from AWS regarding transfers. */ -public class ProgressableProgressListener implements ProgressListener { +public class ProgressableProgressListener implements TransferListener { private static final Logger LOG = S3AFileSystem.LOG; private final S3AFileSystem fs; private final String key; private final Progressable progress; private long lastBytesTransferred; - private final Upload upload; /** * Instantiate. * @param fs filesystem: will be invoked with statistics updates * @param key key for the upload - * @param upload source of events * @param progress optional callback for progress. */ public ProgressableProgressListener(S3AFileSystem fs, String key, - Upload upload, Progressable progress) { this.fs = fs; this.key = key; - this.upload = upload; this.progress = progress; this.lastBytesTransferred = 0; } @Override - public void progressChanged(ProgressEvent progressEvent) { - if (progress != null) { - progress.progress(); - } + public void transferInitiated(TransferListener.Context.TransferInitiated context) { + fs.incrementWriteOperations(); + } + + @Override + public void transferComplete(TransferListener.Context.TransferComplete context) { + fs.incrementWriteOperations(); + } + + @Override + public void bytesTransferred(TransferListener.Context.BytesTransferred context) { - // There are 3 http ops here, but this should be close enough for now - ProgressEventType pet = progressEvent.getEventType(); - if (pet == TRANSFER_PART_STARTED_EVENT || - pet == TRANSFER_COMPLETED_EVENT) { - fs.incrementWriteOperations(); + if(progress != null) { + progress.progress(); } - long transferred = upload.getProgress().getBytesTransferred(); + long transferred = context.progressSnapshot().transferredBytes(); long delta = transferred - lastBytesTransferred; fs.incrementPutProgressStatistics(key, delta); lastBytesTransferred = transferred; @@ -79,11 +75,13 @@ public void progressChanged(ProgressEvent progressEvent) { /** * Method to invoke after upload has completed. * This can handle race conditions in setup/teardown. + * @param upload upload which has just completed. * @return the number of bytes which were transferred after the notification */ - public long uploadCompleted() { - long delta = upload.getProgress().getBytesTransferred() - - lastBytesTransferred; + public long uploadCompleted(ObjectTransfer upload) { + + long delta = + upload.progress().snapshot().transferredBytes() - lastBytesTransferred; if (delta > 0) { LOG.debug("S3A write delta changed after finished: {} bytes", delta); fs.incrementPutProgressStatistics(key, delta); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 43a2b7e0dbd5b..de0f59154e995 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -31,15 +31,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import com.amazonaws.SdkBaseException; -import com.amazonaws.event.ProgressEvent; -import com.amazonaws.event.ProgressEventType; -import com.amazonaws.event.ProgressListener; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.UploadPartRequest; - +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +import org.apache.hadoop.fs.s3a.impl.ProgressListener; +import org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; import org.apache.hadoop.util.Preconditions; @@ -69,6 +70,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.Statistic.*; +import static org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent.*; import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation; @@ -192,7 +194,7 @@ class S3ABlockOutputStream extends OutputStream implements this.executorService = MoreExecutors.listeningDecorator( builder.executorService); this.multiPartUpload = null; - final Progressable progress = builder.progress; + Progressable progress = builder.progress; this.progressListener = (progress instanceof ProgressListener) ? (ProgressListener) progress : new ProgressableListener(progress); @@ -439,7 +441,7 @@ public void close() throws IOException { uploadCurrentBlock(true); } // wait for the partial uploads to finish - final List partETags = + final List partETags = multiPartUpload.waitForAllPartUploads(); bytes = bytesSubmitted; @@ -597,27 +599,28 @@ private long putObject() throws IOException { final PutObjectRequest putObjectRequest = uploadData.hasFile() ? writeOperationHelper.createPutObjectRequest( key, - uploadData.getFile(), - builder.putOptions) + uploadData.getFile().length(), + builder.putOptions, + true) : writeOperationHelper.createPutObjectRequest( key, - uploadData.getUploadStream(), size, - builder.putOptions); - BlockUploadProgress callback = - new BlockUploadProgress( - block, progressListener, now()); - putObjectRequest.setGeneralProgressListener(callback); + builder.putOptions, + false); + + BlockUploadProgress progressCallback = + new BlockUploadProgress(block, progressListener, now()); statistics.blockUploadQueued(size); - ListenableFuture putObjectResult = + ListenableFuture putObjectResult = executorService.submit(() -> { try { // the putObject call automatically closes the input // stream afterwards. - return writeOperationHelper.putObject( - putObjectRequest, - builder.putOptions, - statistics); + PutObjectResponse response = + writeOperationHelper.putObject(putObjectRequest, builder.putOptions, uploadData, + uploadData.hasFile(), statistics); + progressCallback.progressChanged(REQUEST_BYTE_TRANSFER_EVENT); + return response; } finally { cleanupWithLogger(LOG, uploadData, block); } @@ -761,7 +764,7 @@ protected IOStatisticsAggregator getThreadIOStatistics() { */ private class MultiPartUpload { private final String uploadId; - private final List> partETagsFutures; + private final List> partETagsFutures; private int partsSubmitted; private int partsUploaded; private long bytesSubmitted; @@ -866,18 +869,19 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block, final int currentPartNumber = partETagsFutures.size() + 1; final UploadPartRequest request; final S3ADataBlocks.BlockUploadData uploadData; + final RequestBody requestBody; try { uploadData = block.startUpload(); - request = writeOperationHelper.newUploadPartRequest( + requestBody = uploadData.hasFile() + ? RequestBody.fromFile(uploadData.getFile()) + : RequestBody.fromInputStream(uploadData.getUploadStream(), size); + + request = writeOperationHelper.newUploadPartRequestBuilder( key, uploadId, currentPartNumber, - size, - uploadData.getUploadStream(), - uploadData.getFile(), - 0L); - request.setLastPart(isLast); - } catch (SdkBaseException aws) { + size).build(); + } catch (SdkException aws) { // catch and translate IOException e = translateException("upload", key, aws); // failure to start the upload. @@ -888,28 +892,38 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block, noteUploadFailure(e); throw e; } - BlockUploadProgress callback = - new BlockUploadProgress( - block, progressListener, now()); - request.setGeneralProgressListener(callback); + + BlockUploadProgress progressCallback = + new BlockUploadProgress(block, progressListener, now()); + statistics.blockUploadQueued(block.dataSize()); - ListenableFuture partETagFuture = + ListenableFuture partETagFuture = executorService.submit(() -> { // this is the queued upload operation // do the upload try { LOG.debug("Uploading part {} for id '{}'", currentPartNumber, uploadId); - PartETag partETag = writeOperationHelper.uploadPart(request, statistics) - .getPartETag(); + + progressCallback.progressChanged(TRANSFER_PART_STARTED_EVENT); + + UploadPartResponse response = writeOperationHelper + .uploadPart(request, requestBody, statistics); LOG.debug("Completed upload of {} to part {}", - block, partETag.getETag()); + block, response.eTag()); LOG.debug("Stream statistics of {}", statistics); partsUploaded++; - return partETag; + + progressCallback.progressChanged(TRANSFER_PART_COMPLETED_EVENT); + + return CompletedPart.builder() + .eTag(response.eTag()) + .partNumber(currentPartNumber) + .build(); } catch (IOException e) { // save immediately. noteUploadFailure(e); + progressCallback.progressChanged(TRANSFER_PART_FAILED_EVENT); throw e; } finally { // close the stream and block @@ -924,7 +938,7 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block, * @return list of results * @throws IOException IO Problems */ - private List waitForAllPartUploads() throws IOException { + private List waitForAllPartUploads() throws IOException { LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size()); try { return Futures.allAsList(partETagsFutures).get(); @@ -948,7 +962,7 @@ private List waitForAllPartUploads() throws IOException { */ private void cancelAllActiveFutures() { LOG.debug("Cancelling futures"); - for (ListenableFuture future : partETagsFutures) { + for (ListenableFuture future : partETagsFutures) { future.cancel(true); } } @@ -960,7 +974,7 @@ private void cancelAllActiveFutures() { * @param partETags list of partial uploads * @throws IOException on any problem */ - private void complete(List partETags) + private void complete(List partETags) throws IOException { maybeRethrowUploadFailure(); AtomicInteger errorCount = new AtomicInteger(0); @@ -1005,22 +1019,24 @@ private IOException abort() { } } + /** * The upload progress listener registered for events returned * during the upload of a single block. * It updates statistics and handles the end of the upload. * Transfer failures are logged at WARN. */ - private final class BlockUploadProgress implements ProgressListener { + private final class BlockUploadProgress { + private final S3ADataBlocks.DataBlock block; private final ProgressListener nextListener; private final Instant transferQueueTime; private Instant transferStartTime; + private long size; /** * Track the progress of a single block upload. * @param block block to monitor - * @param nextListener optional next progress listener * @param transferQueueTime time the block was transferred * into the queue */ @@ -1029,20 +1045,17 @@ private BlockUploadProgress(S3ADataBlocks.DataBlock block, Instant transferQueueTime) { this.block = block; this.transferQueueTime = transferQueueTime; + this.size = block.dataSize(); this.nextListener = nextListener; } - @Override - public void progressChanged(ProgressEvent progressEvent) { - ProgressEventType eventType = progressEvent.getEventType(); - long bytesTransferred = progressEvent.getBytesTransferred(); + public void progressChanged(ProgressListenerEvent eventType) { - long size = block.dataSize(); switch (eventType) { case REQUEST_BYTE_TRANSFER_EVENT: // bytes uploaded - statistics.bytesTransferred(bytesTransferred); + statistics.bytesTransferred(size); break; case TRANSFER_PART_STARTED_EVENT: @@ -1057,6 +1070,7 @@ public void progressChanged(ProgressEvent progressEvent) { statistics.blockUploadCompleted( Duration.between(transferStartTime, now()), size); + statistics.bytesTransferred(size); break; case TRANSFER_PART_FAILED_EVENT: @@ -1071,13 +1085,13 @@ public void progressChanged(ProgressEvent progressEvent) { } if (nextListener != null) { - nextListener.progressChanged(progressEvent); + nextListener.progressChanged(eventType, size); } } } /** - * Bridge from AWS {@code ProgressListener} to Hadoop {@link Progressable}. + * Bridge from {@link ProgressListener} to Hadoop {@link Progressable}. */ private static class ProgressableListener implements ProgressListener { private final Progressable progress; @@ -1086,7 +1100,7 @@ private static class ProgressableListener implements ProgressListener { this.progress = progress; } - public void progressChanged(ProgressEvent progressEvent) { + public void progressChanged(ProgressListenerEvent eventType, int bytesTransferred) { if (progress != null) { progress.progress(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java index b20d8e859aa88..1c6facfd54f8c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java @@ -47,7 +47,7 @@ * Set of classes to support output streaming into blocks which are then * uploaded as to S3 as a single PUT, or as part of a multipart request. */ -final class S3ADataBlocks { +public final class S3ADataBlocks { private static final Logger LOG = LoggerFactory.getLogger(S3ADataBlocks.class); @@ -101,7 +101,7 @@ static BlockFactory createFactory(S3AFileSystem owner, * It can be one of a file or an input stream. * When closed, any stream is closed. Any source file is untouched. */ - static final class BlockUploadData implements Closeable { + public static final class BlockUploadData implements Closeable { private final File file; private final InputStream uploadStream; @@ -109,7 +109,7 @@ static final class BlockUploadData implements Closeable { * File constructor; input stream will be null. * @param file file to upload */ - BlockUploadData(File file) { + public BlockUploadData(File file) { Preconditions.checkArgument(file.exists(), "No file: " + file); this.file = file; this.uploadStream = null; @@ -119,7 +119,7 @@ static final class BlockUploadData implements Closeable { * Stream constructor, file field will be null. * @param uploadStream stream to upload */ - BlockUploadData(InputStream uploadStream) { + public BlockUploadData(InputStream uploadStream) { Preconditions.checkNotNull(uploadStream, "rawUploadStream"); this.uploadStream = uploadStream; this.file = null; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 2c828a5ef35fe..e192135b9f312 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -21,17 +21,20 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.InterruptedIOException; import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.AccessDeniedException; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -41,6 +44,7 @@ import java.util.Objects; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -48,43 +52,53 @@ import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.Headers; -import com.amazonaws.services.s3.model.CannedAccessControlList; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.SelectObjectContentResult; -import com.amazonaws.services.s3.model.StorageClass; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; -import com.amazonaws.services.s3.transfer.Copy; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerConfiguration; -import com.amazonaws.services.s3.transfer.Upload; -import com.amazonaws.services.s3.transfer.model.CopyResult; -import com.amazonaws.services.s3.transfer.model.UploadResult; -import com.amazonaws.event.ProgressListener; - +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; +import software.amazon.awssdk.transfer.s3.model.CompletedCopy; +import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.model.Copy; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CopyRequest; +import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.UploadFileRequest; + +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,11 +122,12 @@ import org.apache.hadoop.fs.s3a.auth.SignerManager; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; +import org.apache.hadoop.fs.s3a.impl.AWSCannedACL; +import org.apache.hadoop.fs.s3a.impl.AWSHeaders; import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; import org.apache.hadoop.fs.s3a.impl.ContextAccessors; import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation; -import org.apache.hadoop.fs.s3a.impl.CopyOutcome; import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder; import org.apache.hadoop.fs.s3a.impl.DeleteOperation; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; @@ -122,6 +137,7 @@ import org.apache.hadoop.fs.s3a.impl.InternalConstants; import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; import org.apache.hadoop.fs.s3a.impl.MkdirOperation; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; import org.apache.hadoop.fs.s3a.impl.OpenFileSupport; import org.apache.hadoop.fs.s3a.impl.OperationCallbacks; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; @@ -131,7 +147,6 @@ import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; -import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl; @@ -141,6 +156,7 @@ import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; +import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.fs.store.audit.AuditEntryPoint; import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -213,6 +229,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.INITIALIZE_SPAN; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList; import static org.apache.hadoop.fs.s3a.auth.RolePolicies.STATEMENT_ALLOW_KMS_RW; import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3Operations; import static org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.TokenIssuingPolicy.NoTokensAvailable; @@ -224,14 +241,14 @@ import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_INACCESSIBLE; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_301_MOVED_PERMANENTLY; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion; import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.logDnsLookup; @@ -275,7 +292,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private URI uri; private Path workingDir; private String username; - private AmazonS3 s3; + private S3Client s3Client; + /** Async client is used for transfer manager and s3 select. */ + private S3AsyncClient s3AsyncClient; // initial callback policy is fail-once; it's there just to assist // some mock tests and other codepaths trying to call the low level // APIs on an uninitialized filesystem. @@ -294,7 +313,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private Listing listing; private long partSize; private boolean enableMultiObjectsDelete; - private TransferManager transfers; + private S3TransferManager transferManager; private ExecutorService boundedThreadPool; private ThreadPoolExecutor unboundedThreadPool; @@ -313,10 +332,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private int executorCapacity; private long multiPartThreshold; public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); + /** Exactly once log to warn about setting the region in config to avoid probe. */ + private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG); private static final Logger PROGRESS = LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress"); private LocalDirAllocator directoryAllocator; - private CannedAccessControlList cannedACL; + private String cannedACL; /** * This must never be null; until initialized it just declares that there @@ -364,6 +385,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private AWSCredentialProviderList credentials; private SignerManager signerManager; + private S3AInternals s3aInternals; /** * Page size for deletions. @@ -429,6 +451,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private String scheme = FS_S3A; + private final static Map BUCKET_REGIONS = new HashMap<>(); + /** Add any deprecated keys. */ @SuppressWarnings("deprecation") private static void addDeprecatedKeys() { @@ -510,6 +534,8 @@ public void initialize(URI name, Configuration originalConf) super.initialize(uri, conf); setConf(conf); + s3aInternals = createS3AInternals(); + // look for encryption data // DT Bindings may override this setEncryptionSecrets( @@ -583,9 +609,6 @@ public void initialize(URI name, Configuration originalConf) // the encryption algorithms) bindAWSClient(name, delegationTokensEnabled); - initTransferManager(); - - // This initiates a probe against S3 for the bucket existing. doBucketProbing(); @@ -653,7 +676,7 @@ public void initialize(URI name, Configuration originalConf) AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1); vectoredIOContext = populateVectoredIOContext(conf); scheme = (this.uri != null && this.uri.getScheme() != null) ? this.uri.getScheme() : FS_S3A; - } catch (AmazonClientException e) { + } catch (SdkException e) { // amazon client exception: stop all services then throw the translation cleanupWithLogger(LOG, span); stopAllServices(); @@ -705,7 +728,7 @@ private void setCSEGauge() { * bucket existence check is not done to improve performance of * S3AFileSystem initialization. When set to 1 or 2, bucket existence check * will be performed which is potentially slow. - * If 3 or higher: warn and use the v2 check. + * If 3 or higher: warn and skip check. * Also logging DNS address of the s3 endpoint if the bucket probe value is * greater than 0 else skipping it for increased performance. * @throws UnknownStoreException the bucket is absent @@ -722,18 +745,14 @@ private void doBucketProbing() throws IOException { LOG.debug("skipping check for bucket existence"); break; case 1: - logDnsLookup(getConf()); - verifyBucketExists(); - break; case 2: logDnsLookup(getConf()); - verifyBucketExistsV2(); + verifyBucketExists(); break; default: // we have no idea what this is, assume it is from a later release. - LOG.warn("Unknown bucket probe option {}: {}; falling back to check #2", + LOG.warn("Unknown bucket probe option {}: {}; skipping check for bucket existence", S3A_BUCKET_PROBE, bucketProbe); - verifyBucketExistsV2(); break; } } @@ -826,54 +845,37 @@ protected static S3AStorageStatistics createStorageStatistics( } /** - * Verify that the bucket exists. This does not check permissions, - * not even read access. + * Verify that the bucket exists. * Retry policy: retrying, translated. * @throws UnknownStoreException the bucket is absent * @throws IOException any other problem talking to S3 */ @Retries.RetryTranslated - protected void verifyBucketExists() - throws UnknownStoreException, IOException { - if (!invoker.retry("doesBucketExist", bucket, true, - trackDurationOfOperation(getDurationTrackerFactory(), - STORE_EXISTS_PROBE.getSymbol(), - () -> s3.doesBucketExist(bucket)))) { - throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does " - + "not exist"); - } - } + protected void verifyBucketExists() throws UnknownStoreException, IOException { - /** - * Verify that the bucket exists. This will correctly throw an exception - * when credentials are invalid. - * Retry policy: retrying, translated. - * @throws UnknownStoreException the bucket is absent - * @throws IOException any other problem talking to S3 - */ - @Retries.RetryTranslated - protected void verifyBucketExistsV2() - throws UnknownStoreException, IOException { - if (!invoker.retry("doesBucketExistV2", bucket, true, - trackDurationOfOperation(getDurationTrackerFactory(), - STORE_EXISTS_PROBE.getSymbol(), - () -> { - // Bug in SDK always returns `true` for AccessPoint ARNs with `doesBucketExistV2()` - // expanding implementation to use ARNs and buckets correctly + if(!trackDurationAndSpan( + STORE_EXISTS_PROBE, bucket, null, () -> + invoker.retry("doesBucketExist", bucket, true, () -> { try { - s3.getBucketAcl(bucket); - } catch (AmazonServiceException ex) { - int statusCode = ex.getStatusCode(); - if (statusCode == SC_404 || - (statusCode == SC_403 && ex.getMessage().contains(AP_INACCESSIBLE))) { + if (BUCKET_REGIONS.containsKey(bucket)) { + return true; + } + s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()); + return true; + } catch (AwsServiceException ex) { + int statusCode = ex.statusCode(); + if (statusCode == SC_404_NOT_FOUND || + (statusCode == SC_403_FORBIDDEN && accessPoint != null)) { return false; } } return true; }))) { - throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does " - + "not exist"); + + throw new UnknownStoreException("s3a://" + bucket + "/", + " Bucket does " + "not exist. " + "Accessing with " + ENDPOINT + " set to " + + getConf().getTrimmed(ENDPOINT, null)); } } @@ -914,7 +916,6 @@ public Listing getListing() { * @param dtEnabled are delegation tokens enabled? * @throws IOException failure. */ - @SuppressWarnings("deprecation") private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { Configuration conf = getConf(); credentials = null; @@ -927,7 +928,6 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { // with it if so. LOG.debug("Using delegation tokens"); - V2Migration.v1DelegationTokenCredentialProvidersUsed(); S3ADelegationTokens tokens = new S3ADelegationTokens(); this.delegationTokens = Optional.of(tokens); tokens.bindToFileSystem(getCanonicalUri(), @@ -954,7 +954,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { uaSuffix = tokens.getUserAgentField(); } else { // DT support is disabled, so create the normal credential chain - credentials = createAWSCredentialProviderSet(name, conf); + credentials = createAWSCredentialProviderList(name, conf); } LOG.debug("Using credential provider {}", credentials); Class s3ClientFactoryClass = conf.getClass( @@ -965,8 +965,14 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { ? conf.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT) : accessPoint.getEndpoint(); - S3ClientFactory.S3ClientCreationParameters parameters = null; - parameters = new S3ClientFactory.S3ClientCreationParameters() + String configuredRegion = accessPoint == null + ? conf.getTrimmed(AWS_REGION) + : accessPoint.getRegion(); + + Region region = getS3Region(configuredRegion); + + S3ClientFactory.S3ClientCreationParameters parameters = + new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(credentials) .withPathUri(name) .withEndpoint(endpoint) @@ -974,11 +980,97 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { .withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false)) .withUserAgentSuffix(uaSuffix) .withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS)) - .withRequestHandlers(auditManager.createRequestHandlers()); + .withExecutionInterceptors(auditManager.createExecutionInterceptors()) + .withMinimumPartSize(partSize) + .withMultipartThreshold(multiPartThreshold) + .withTransferManagerExecutor(unboundedThreadPool) + .withRegion(region); + + S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf); + s3Client = clientFactory.createS3Client(getUri(), parameters); + createS3AsyncClient(clientFactory, parameters); + transferManager = clientFactory.createS3TransferManager(getS3AsyncClient()); + } + + /** + * Creates and configures the S3AsyncClient. + * Uses synchronized method to suppress spotbugs error. + * + * @param clientFactory factory used to create S3AsyncClient + * @param parameters parameter object + * @throws IOException on any IO problem + */ + private void createS3AsyncClient(S3ClientFactory clientFactory, + S3ClientFactory.S3ClientCreationParameters parameters) throws IOException { + s3AsyncClient = clientFactory.createS3AsyncClient(getUri(), parameters); + } + + /** + * Get the bucket region. + * + * @param region AWS S3 Region set in the config. This property may not be set, in which case + * ask S3 for the region. + * @return region of the bucket. + */ + private Region getS3Region(String region) throws IOException { + + if (!StringUtils.isBlank(region)) { + return Region.of(region); + } + + Region cachedRegion = BUCKET_REGIONS.get(bucket); + + if (cachedRegion != null) { + LOG.debug("Got region {} for bucket {} from cache", cachedRegion, bucket); + return cachedRegion; + } + + Region s3Region = trackDurationAndSpan(STORE_REGION_PROBE, bucket, null, + () -> invoker.retry("getS3Region", bucket, true, () -> { + try { + + SET_REGION_WARNING.warn( + "Getting region for bucket {} from S3, this will slow down FS initialisation. " + + "To avoid this, set the region using property {}", bucket, + FS_S3A_BUCKET_PREFIX + bucket + ".endpoint.region"); + + // build a s3 client with region eu-west-1 that can be used to get the region of the + // bucket. Using eu-west-1, as headBucket() doesn't work with us-east-1. This is because + // us-east-1 uses the endpoint s3.amazonaws.com, which resolves bucket.s3.amazonaws.com + // to the actual region the bucket is in. As the request is signed with us-east-1 and + // not the bucket's region, it fails. + S3Client getRegionS3Client = + S3Client.builder().region(Region.EU_WEST_1).credentialsProvider(credentials) + .build(); + + HeadBucketResponse headBucketResponse = + getRegionS3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()); + + Region bucketRegion = Region.of( + headBucketResponse.sdkHttpResponse().headers().get(BUCKET_REGION_HEADER).get(0)); + BUCKET_REGIONS.put(bucket, bucketRegion); + + return bucketRegion; + } catch (S3Exception exception) { + if (exception.statusCode() == SC_301_MOVED_PERMANENTLY) { + Region bucketRegion = Region.of( + exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER) + .get(0)); + BUCKET_REGIONS.put(bucket, bucketRegion); + + return bucketRegion; + } + + if (exception.statusCode() == SC_404_NOT_FOUND) { + throw new UnknownStoreException("s3a://" + bucket + "/", + " Bucket does " + "not exist"); + } + + throw exception; + } + })); - s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf) - .createS3Client(getUri(), - parameters); + return s3Region; } /** @@ -1078,12 +1170,14 @@ protected RequestFactory createRequestFactory() { .toUpperCase(Locale.US); StorageClass storageClass = null; if (!storageClassConf.isEmpty()) { - try { - storageClass = StorageClass.fromValue(storageClassConf); - } catch (IllegalArgumentException e) { + storageClass = StorageClass.fromValue(storageClassConf); + + if (storageClass.equals(StorageClass.UNKNOWN_TO_SDK_VERSION)) { LOG.warn("Unknown storage class property {}: {}; falling back to default storage class", STORAGE_CLASS, storageClassConf); + storageClass = null; } + } else { LOG.debug("Unset storage class property {}; falling back to default storage class", STORAGE_CLASS); @@ -1110,6 +1204,14 @@ public RequestFactory getRequestFactory() { return requestFactory; } + /** + * Get the S3 Async client. + * @return the async s3 client. + */ + private S3AsyncClient getS3AsyncClient() { + return s3AsyncClient; + } + /** * Implementation of all operations used by delegation tokens. */ @@ -1150,22 +1252,10 @@ public EncryptionSecrets getEncryptionSecrets() { return encryptionSecrets; } - private void initTransferManager() { - TransferManagerConfiguration transferConfiguration = - new TransferManagerConfiguration(); - transferConfiguration.setMinimumUploadPartSize(partSize); - transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); - transferConfiguration.setMultipartCopyPartSize(partSize); - transferConfiguration.setMultipartCopyThreshold(multiPartThreshold); - - transfers = new TransferManager(s3, unboundedThreadPool); - transfers.setConfiguration(transferConfiguration); - } - private void initCannedAcls(Configuration conf) { String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL); if (!cannedACLName.isEmpty()) { - cannedACL = CannedAccessControlList.valueOf(cannedACLName); + cannedACL = AWSCannedACL.valueOf(cannedACLName).toString(); } else { cannedACL = null; } @@ -1198,12 +1288,22 @@ private void initMultipartUploads(Configuration conf) throws IOException { public void abortOutstandingMultipartUploads(long seconds) throws IOException { Preconditions.checkArgument(seconds >= 0); - Date purgeBefore = - new Date(new Date().getTime() - seconds * 1000); + Instant purgeBefore = + Instant.now().minusSeconds(seconds); LOG.debug("Purging outstanding multipart uploads older than {}", purgeBefore); invoker.retry("Purging multipart uploads", bucket, true, - () -> transfers.abortMultipartUploads(bucket, purgeBefore)); + () -> { + MultipartUtils.UploadIterator uploadIterator = + MultipartUtils.listMultipartUploads(createStoreContext(), s3Client, null, maxKeys); + + while (uploadIterator.hasNext()) { + MultipartUpload upload = uploadIterator.next(); + if (upload.initiated().compareTo(purgeBefore) < 0) { + abortMultipartUpload(upload); + } + } + }); } /** @@ -1252,81 +1352,122 @@ public int getDefaultPort() { } /** - * Returns the S3 client used by this filesystem. - * This is for internal use within the S3A code itself. - * @return AmazonS3Client + * Set the client -used in mocking tests to force in a different client. + * @param client client. */ - private AmazonS3 getAmazonS3Client() { - return s3; + protected void setAmazonS3Client(S3Client client) { + Preconditions.checkNotNull(client, "clientV2"); + LOG.debug("Setting S3V2 client to {}", client); + s3Client = client; } /** - * Returns the S3 client used by this filesystem. - * Warning: this must only be used for testing, as it bypasses core - * S3A operations. - * @param reason a justification for requesting access. - * @return AmazonS3Client + * S3AInternals method. + * {@inheritDoc}. */ - @VisibleForTesting - public AmazonS3 getAmazonS3ClientForTesting(String reason) { - LOG.warn("Access to S3A client requested, reason {}", reason); - V2Migration.v1S3ClientRequested(); - return s3; + @AuditEntryPoint + @Retries.RetryTranslated + public String getBucketLocation() throws IOException { + return s3aInternals.getBucketLocation(bucket); } /** - * Set the client -used in mocking tests to force in a different client. - * @param client client. + * Create the S3AInternals; left as something mocking + * subclasses may want to override. + * @return the internal implementation */ - protected void setAmazonS3Client(AmazonS3 client) { - Preconditions.checkNotNull(client, "client"); - LOG.debug("Setting S3 client to {}", client); - s3 = client; - - // Need to use a new TransferManager that uses the new client. - // Also, using a new TransferManager requires a new threadpool as the old - // TransferManager will shut the thread pool down when it is garbage - // collected. - initThreadPools(getConf()); - initTransferManager(); + protected S3AInternals createS3AInternals() { + return new S3AInternalsImpl(); } /** - * Get the region of a bucket. - * Invoked from StoreContext; consider an entry point. - * @return the region in which a bucket is located - * @throws AccessDeniedException if the caller lacks permission. - * @throws IOException on any failure. + * Get the S3AInternals. + * @return the internal implementation */ - @Retries.RetryTranslated - @InterfaceAudience.LimitedPrivate("diagnostics") - public String getBucketLocation() throws IOException { - return getBucketLocation(bucket); + public S3AInternals getS3AInternals() { + return s3aInternals; } /** - * Get the region of a bucket; fixing up the region so it can be used - * in the builders of other AWS clients. - * Requires the caller to have the AWS role permission - * {@code s3:GetBucketLocation}. - * Retry policy: retrying, translated. - * @param bucketName the name of the bucket - * @return the region in which a bucket is located - * @throws AccessDeniedException if the caller lacks permission. - * @throws IOException on any failure. + * Implementation of the S3A Internals operations; pulled out of S3AFileSystem to + * force code accessing it to call {@link #getS3AInternals()}. */ - @VisibleForTesting - @AuditEntryPoint - @Retries.RetryTranslated - public String getBucketLocation(String bucketName) throws IOException { - final String region = trackDurationAndSpan( - STORE_EXISTS_PROBE, bucketName, null, () -> - invoker.retry("getBucketLocation()", bucketName, true, () -> - // If accessPoint then region is known from Arn - accessPoint != null - ? accessPoint.getRegion() - : s3.getBucketLocation(bucketName))); - return fixBucketRegion(region); + private final class S3AInternalsImpl implements S3AInternals { + + @Override + public S3Client getAmazonS3Client(String reason) { + LOG.debug("Access to S3 client requested, reason {}", reason); + return s3Client; + } + + /** + * S3AInternals method. + * {@inheritDoc}. + */ + @Override + @AuditEntryPoint + @Retries.RetryTranslated + public String getBucketLocation() throws IOException { + return s3aInternals.getBucketLocation(bucket); + } + + /** + * S3AInternals method. + * {@inheritDoc}. + */ + @Override + @AuditEntryPoint + @Retries.RetryTranslated + public String getBucketLocation(String bucketName) throws IOException { + final String region = trackDurationAndSpan( + STORE_EXISTS_PROBE, bucketName, null, () -> + invoker.retry("getBucketLocation()", bucketName, true, () -> + // If accessPoint then region is known from Arn + accessPoint != null + ? accessPoint.getRegion() + : s3Client.getBucketLocation(GetBucketLocationRequest.builder() + .bucket(bucketName) + .build()) + .locationConstraintAsString())); + return fixBucketRegion(region); + } + + /** + * S3AInternals method. + * {@inheritDoc}. + */ + @Override + @AuditEntryPoint + @Retries.RetryTranslated + public HeadObjectResponse getObjectMetadata(Path path) throws IOException { + return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () -> + S3AFileSystem.this.getObjectMetadata(makeQualified(path), null, invoker, + "getObjectMetadata")); + } + + /** + * S3AInternals method. + * {@inheritDoc}. + */ + @Override + @AuditEntryPoint + @Retries.RetryTranslated + public HeadBucketResponse getBucketMetadata() throws IOException { + return S3AFileSystem.this.getBucketMetadata(); + } + + /** + * Get a shared copy of the AWS credentials, with its reference + * counter updated. + * Caller is required to call {@code close()} on this after + * they have finished using it. + * @param purpose what is this for? This is initially for logging + * @return a reference to shared credentials. + */ + public AWSCredentialProviderList shareCredentials(final String purpose) { + LOG.debug("Sharing credentials for: {}", purpose); + return credentials.share(); + } } /** @@ -1349,7 +1490,7 @@ public ChangeDetectionPolicy getChangeDetectionPolicy() { } /** - * Get the encryption algorithm of this endpoint. + * Get the encryption algorithm of this connector. * @return the encryption algorithm. */ public S3AEncryptionMethods getS3EncryptionAlgorithm() { @@ -1396,6 +1537,8 @@ private void initLocalDirAllocatorIfNotInitialized(Configuration conf) { * Get the bucket of this filesystem. * @return the bucket */ + @InterfaceAudience.Public + @InterfaceStability.Stable public String getBucket() { return bucket; } @@ -1413,7 +1556,7 @@ protected void setBucket(String bucket) { * Get the canned ACL of this FS. * @return an ACL, if any */ - CannedAccessControlList getCannedACL() { + String getCannedACL() { return cannedACL; } @@ -1644,18 +1787,18 @@ public void close() { } @Override - public GetObjectRequest newGetRequest(final String key) { + public GetObjectRequest.Builder newGetRequestBuilder(final String key) { // active the audit span used for the operation try (AuditSpan span = auditSpan.activate()) { - return getRequestFactory().newGetObjectRequest(key); + return getRequestFactory().newGetObjectRequestBuilder(key); } } @Override - public S3Object getObject(GetObjectRequest request) { + public ResponseInputStream getObject(GetObjectRequest request) { // active the audit span used for the operation try (AuditSpan span = auditSpan.activate()) { - return s3.getObject(request); + return s3Client.getObject(request); } } @@ -1682,18 +1825,19 @@ private final class WriteOperationHelperCallbacksImpl implements WriteOperationHelper.WriteOperationHelperCallbacks { @Override - public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { - return s3.selectObjectContent(request); + public CompletableFuture selectObjectContent( + SelectObjectContentRequest request, + SelectObjectContentResponseHandler responseHandler) { + return getS3AsyncClient().selectObjectContent(request, responseHandler); } @Override - public CompleteMultipartUploadResult completeMultipartUpload( + public CompleteMultipartUploadResponse completeMultipartUpload( CompleteMultipartUploadRequest request) { - return s3.completeMultipartUpload(request); + return s3Client.completeMultipartUpload(request); } } - /** * Create the read context for reading from the referenced file, * using FS state as well as the status. @@ -2058,7 +2202,7 @@ public boolean rename(Path src, Path dst) throws IOException { innerRename(src, dst)); LOG.debug("Copied {} bytes", bytesCopied); return true; - } catch (AmazonClientException e) { + } catch (SdkException e) { throw translateException("rename(" + src +", " + dst + ")", src, e); } catch (RenameFailedException e) { LOG.info("{}", e.getMessage()); @@ -2169,7 +2313,7 @@ private Pair initiateRename( * This operation throws an exception on any failure which needs to be * reported and downgraded to a failure. * Retries: retry translated, assuming all operations it is called do - * so. For safely, consider catch and handle AmazonClientException + * so. For safely, consider catch and handle SdkException * because this is such a complex method there's a risk it could surface. * @param source path to be renamed * @param dest new path after rename @@ -2180,12 +2324,12 @@ private Pair initiateRename( * @return the number of bytes copied. * @throws FileNotFoundException there's no source file. * @throws IOException on IO failure. - * @throws AmazonClientException on failures inside the AWS SDK + * @throws SdkException on failures inside the AWS SDK */ @Retries.RetryMixed private long innerRename(Path source, Path dest) throws RenameFailedException, FileNotFoundException, IOException, - AmazonClientException { + SdkException { Path src = qualify(source); Path dst = qualify(dest); @@ -2280,7 +2424,7 @@ public RemoteIterator listFilesAndDirectoryMarkers( } @Override - public CopyResult copyFile(final String srcKey, + public CopyObjectResponse copyFile(final String srcKey, final String destKey, final S3ObjectAttributes srcAttributes, final S3AReadOpContext readContext) throws IOException { @@ -2291,9 +2435,9 @@ public CopyResult copyFile(final String srcKey, @Override public void removeKeys( - final List keysToDelete, + final List keysToDelete, final boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, IOException { + throws MultiObjectDeleteException, SdkException, IOException { auditSpan.activate(); S3AFileSystem.this.removeKeys(keysToDelete, deleteFakeDir); } @@ -2392,21 +2536,17 @@ public int getMaxKeys() { * Low-level call to get at the object metadata. * This method is used in some external applications and so * must be viewed as a public entry point. - * Auditing: An audit entry point. + * @deprecated use S3AInternals API. * @param path path to the object. This will be qualified. * @return metadata * @throws IOException IO and object access problems. */ - @VisibleForTesting @AuditEntryPoint @InterfaceAudience.LimitedPrivate("utilities") @Retries.RetryTranslated - @InterfaceStability.Evolving - public ObjectMetadata getObjectMetadata(Path path) throws IOException { - V2Migration.v1GetObjectMetadataCalled(); - return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () -> - getObjectMetadata(makeQualified(path), null, invoker, - "getObjectMetadata")); + @Deprecated + public HeadObjectResponse getObjectMetadata(Path path) throws IOException { + return getS3AInternals().getObjectMetadata(path); } /** @@ -2419,7 +2559,7 @@ public ObjectMetadata getObjectMetadata(Path path) throws IOException { * @throws IOException IO and object access problems. */ @Retries.RetryTranslated - private ObjectMetadata getObjectMetadata(Path path, + private HeadObjectResponse getObjectMetadata(Path path, ChangeTracker changeTracker, Invoker changeInvoker, String operation) throws IOException { String key = pathToKey(path); @@ -2632,7 +2772,7 @@ protected DurationTrackerFactory nonNullDurationTrackerFactory( @Retries.RetryRaw @VisibleForTesting @InterfaceAudience.LimitedPrivate("external utilities") - ObjectMetadata getObjectMetadata(String key) throws IOException { + HeadObjectResponse getObjectMetadata(String key) throws IOException { return getObjectMetadata(key, null, invoker, "getObjectMetadata"); } @@ -2649,28 +2789,28 @@ ObjectMetadata getObjectMetadata(String key) throws IOException { * @throws RemoteFileChangedException if an unexpected version is detected */ @Retries.RetryRaw - protected ObjectMetadata getObjectMetadata(String key, + protected HeadObjectResponse getObjectMetadata(String key, ChangeTracker changeTracker, Invoker changeInvoker, String operation) throws IOException { - ObjectMetadata meta = changeInvoker.retryUntranslated("GET " + key, true, + HeadObjectResponse response = changeInvoker.retryUntranslated("GET " + key, true, () -> { - GetObjectMetadataRequest request - = getRequestFactory().newGetObjectMetadataRequest(key); + HeadObjectRequest.Builder requestBuilder = + getRequestFactory().newHeadObjectRequestBuilder(key); incrementStatistic(OBJECT_METADATA_REQUESTS); DurationTracker duration = getDurationTrackerFactory() .trackDuration(ACTION_HTTP_HEAD_REQUEST.getSymbol()); try { LOG.debug("HEAD {} with change tracker {}", key, changeTracker); if (changeTracker != null) { - changeTracker.maybeApplyConstraint(request); + changeTracker.maybeApplyConstraint(requestBuilder); } - ObjectMetadata objectMetadata = s3.getObjectMetadata(request); + HeadObjectResponse headObjectResponse = s3Client.headObject(requestBuilder.build()); if (changeTracker != null) { - changeTracker.processMetadata(objectMetadata, operation); + changeTracker.processMetadata(headObjectResponse, operation); } - return objectMetadata; - } catch(AmazonServiceException ase) { + return headObjectResponse; + } catch (AwsServiceException ase) { if (!isObjectNotFound(ase)) { // file not found is not considered a failure of the call, // so only switch the duration tracker to update failure @@ -2684,7 +2824,28 @@ protected ObjectMetadata getObjectMetadata(String key, } }); incrementReadOperations(); - return meta; + return response; + } + + /** + * Request bucket metadata. + * @return the metadata + * @throws UnknownStoreException the bucket is absent + * @throws IOException any other problem talking to S3 + */ + @AuditEntryPoint + @Retries.RetryTranslated + protected HeadBucketResponse getBucketMetadata() throws IOException { + final HeadBucketResponse response = trackDurationAndSpan(STORE_EXISTS_PROBE, bucket, null, + () -> invoker.retry("getBucketMetadata()", bucket, true, () -> { + try { + return s3Client.headBucket( + getRequestFactory().newHeadBucketRequestBuilder(bucket).build()); + } catch (NoSuchBucketException e) { + throw new UnknownStoreException("s3a://" + bucket + "/", " Bucket does " + "not exist"); + } + })); + return response; } /** @@ -2713,9 +2874,9 @@ protected S3ListResult listObjects(S3ListRequest request, OBJECT_LIST_REQUEST, () -> { if (useListV1) { - return S3ListResult.v1(s3.listObjects(request.getV1())); + return S3ListResult.v1(s3Client.listObjects(request.getV1())); } else { - return S3ListResult.v2(s3.listObjectsV2(request.getV2())); + return S3ListResult.v2(s3Client.listObjectsV2(request.getV2())); } })); } @@ -2758,15 +2919,21 @@ protected S3ListResult continueListObjects(S3ListRequest request, OBJECT_CONTINUE_LIST_REQUEST, () -> { if (useListV1) { - return S3ListResult.v1( - s3.listNextBatchOfObjects( - getRequestFactory() - .newListNextBatchOfObjectsRequest( - prevResult.getV1()))); + List prevListResult = prevResult.getV1().contents(); + + // Next markers are only present when a delimiter is specified. + String nextMarker; + if (prevResult.getV1().nextMarker() != null) { + nextMarker = prevResult.getV1().nextMarker(); + } else { + nextMarker = prevListResult.get(prevListResult.size() - 1).key(); + } + + return S3ListResult.v1(s3Client.listObjects( + request.getV1().toBuilder().marker(nextMarker).build())); } else { - request.getV2().setContinuationToken(prevResult.getV2() - .getNextContinuationToken()); - return S3ListResult.v2(s3.listObjectsV2(request.getV2())); + return S3ListResult.v2(s3Client.listObjectsV2(request.getV2().toBuilder() + .continuationToken(prevResult.getV2().nextContinuationToken()).build())); } })); } @@ -2796,14 +2963,14 @@ public void incrementWriteOperations() { * * Retry policy: retry untranslated; delete considered idempotent. * @param key key to blob to delete. - * @throws AmazonClientException problems working with S3 + * @throws SdkException problems working with S3 * @throws InvalidRequestException if the request was rejected due to * a mistaken attempt to delete the root directory. */ @VisibleForTesting @Retries.RetryRaw protected void deleteObject(String key) - throws AmazonClientException, IOException { + throws SdkException, IOException { blockRootDelete(key); incrementWriteOperations(); try (DurationInfo ignored = @@ -2815,8 +2982,9 @@ protected void deleteObject(String key) incrementStatistic(OBJECT_DELETE_OBJECTS); trackDurationOfInvocation(getDurationTrackerFactory(), OBJECT_DELETE_REQUEST.getSymbol(), - () -> s3.deleteObject(getRequestFactory() - .newDeleteObjectRequest(key))); + () -> s3Client.deleteObject(getRequestFactory() + .newDeleteObjectRequestBuilder(key) + .build())); return null; }); } @@ -2829,14 +2997,14 @@ protected void deleteObject(String key) * @param f path path to delete * @param key key of entry * @param isFile is the path a file (used for instrumentation only) - * @throws AmazonClientException problems working with S3 + * @throws SdkException problems working with S3 * @throws IOException from invoker signature only -should not be raised. */ @Retries.RetryRaw void deleteObjectAtPath(Path f, String key, boolean isFile) - throws AmazonClientException, IOException { + throws SdkException, IOException { if (isFile) { instrumentation.fileDeleted(1); } else { @@ -2878,66 +3046,58 @@ private void blockRootDelete(String key) throws InvalidRequestException { * @return the AWS response * @throws MultiObjectDeleteException one or more of the keys could not * be deleted. - * @throws AmazonClientException amazon-layer failure. + * @throws SdkException amazon-layer failure. */ @Retries.RetryRaw - private DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteRequest) - throws MultiObjectDeleteException, AmazonClientException, IOException { + private DeleteObjectsResponse deleteObjects(DeleteObjectsRequest deleteRequest) + throws MultiObjectDeleteException, SdkException, IOException { incrementWriteOperations(); BulkDeleteRetryHandler retryHandler = new BulkDeleteRetryHandler(createStoreContext()); - int keyCount = deleteRequest.getKeys().size(); - try(DurationInfo ignored = + int keyCount = deleteRequest.delete().objects().size(); + try (DurationInfo ignored = new DurationInfo(LOG, false, "DELETE %d keys", keyCount)) { - return invoker.retryUntranslated("delete", - DELETE_CONSIDERED_IDEMPOTENT, - (text, e, r, i) -> { - // handle the failure - retryHandler.bulkDeleteRetried(deleteRequest, e); - }, - // duration is tracked in the bulk delete counters - trackDurationOfOperation(getDurationTrackerFactory(), - OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> { - incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount); - return s3.deleteObjects(deleteRequest); - })); - } catch (MultiObjectDeleteException e) { - // one or more of the keys could not be deleted. - // log and rethrow - List errors = e.getErrors(); - LOG.debug("Partial failure of delete, {} errors", errors.size(), e); - for (MultiObjectDeleteException.DeleteError error : errors) { - LOG.debug("{}: \"{}\" - {}", - error.getKey(), error.getCode(), error.getMessage()); + DeleteObjectsResponse response = + invoker.retryUntranslated("delete", DELETE_CONSIDERED_IDEMPOTENT, + (text, e, r, i) -> { + // handle the failure + retryHandler.bulkDeleteRetried(deleteRequest, e); + }, + // duration is tracked in the bulk delete counters + trackDurationOfOperation(getDurationTrackerFactory(), + OBJECT_BULK_DELETE_REQUEST.getSymbol(), () -> { + incrementStatistic(OBJECT_DELETE_OBJECTS, keyCount); + return s3Client.deleteObjects(deleteRequest); + })); + + if (!response.errors().isEmpty()) { + // one or more of the keys could not be deleted. + // log and then throw + List errors = response.errors(); + LOG.debug("Partial failure of delete, {} errors", errors.size()); + for (S3Error error : errors) { + LOG.debug("{}: \"{}\" - {}", error.key(), error.code(), error.message()); + } + throw new MultiObjectDeleteException(errors); } - throw e; + + return response; } } /** - * Create a putObject request. + * Create a putObject request builder. * Adds the ACL and metadata * @param key key of object - * @param metadata metadata header - * @param srcfile source file + * @param length length of object to be uploaded + * @param isDirectoryMarker true if object to be uploaded is a directory marker * @return the request */ - public PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, File srcfile) { - return requestFactory.newPutObjectRequest(key, metadata, null, srcfile); - } - - /** - * Create a new object metadata instance. - * Any standard metadata headers are added here, for example: - * encryption. - * - * @param length length of data to set in header. - * @return a new metadata instance - */ - public ObjectMetadata newObjectMetadata(long length) { - return requestFactory.newObjectMetadata(length); + public PutObjectRequest.Builder newPutObjectRequestBuilder(String key, + long length, + boolean isDirectoryMarker) { + return requestFactory.newPutObjectRequestBuilder(key, null, length, isDirectoryMarker); } /** @@ -2954,15 +3114,24 @@ public ObjectMetadata newObjectMetadata(long length) { * Retry policy: N/A: the transfer manager is performing the upload. * Auditing: must be inside an audit span. * @param putObjectRequest the request + * @param file the file to be uploaded + * @param listener the progress listener for the request * @return the upload initiated */ @Retries.OnceRaw - public UploadInfo putObject(PutObjectRequest putObjectRequest) { + public UploadInfo putObject(PutObjectRequest putObjectRequest, File file, + ProgressableProgressListener listener) { long len = getPutRequestLength(putObjectRequest); - LOG.debug("PUT {} bytes to {} via transfer manager ", - len, putObjectRequest.getKey()); + LOG.debug("PUT {} bytes to {} via transfer manager ", len, putObjectRequest.key()); incrementPutStartStatistics(len); - Upload upload = transfers.upload(putObjectRequest); + + FileUpload upload = transferManager.uploadFile( + UploadFileRequest.builder() + .putObjectRequest(putObjectRequest) + .source(file) + .addTransferListener(listener) + .build()); + return new UploadInfo(upload, len); } @@ -2977,30 +3146,37 @@ public UploadInfo putObject(PutObjectRequest putObjectRequest) { * @param putObjectRequest the request * @param putOptions put object options * @param durationTrackerFactory factory for duration tracking + * @param uploadData data to be uploaded + * @param isFile represents if data to be uploaded is a file * @return the upload initiated - * @throws AmazonClientException on problems + * @throws SdkException on problems */ @VisibleForTesting @Retries.OnceRaw("For PUT; post-PUT actions are RetryExceptionsSwallowed") - PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest, + PutObjectResponse putObjectDirect(PutObjectRequest putObjectRequest, PutObjectOptions putOptions, + S3ADataBlocks.BlockUploadData uploadData, boolean isFile, DurationTrackerFactory durationTrackerFactory) - throws AmazonClientException { + throws SdkException { long len = getPutRequestLength(putObjectRequest); - LOG.debug("PUT {} bytes to {}", len, putObjectRequest.getKey()); + LOG.debug("PUT {} bytes to {}", len, putObjectRequest.key()); incrementPutStartStatistics(len); try { - PutObjectResult result = trackDurationOfSupplier( - nonNullDurationTrackerFactory(durationTrackerFactory), - OBJECT_PUT_REQUESTS.getSymbol(), () -> - s3.putObject(putObjectRequest)); + PutObjectResponse response = + trackDurationOfSupplier(nonNullDurationTrackerFactory(durationTrackerFactory), + OBJECT_PUT_REQUESTS.getSymbol(), + () -> isFile ? + s3Client.putObject(putObjectRequest, RequestBody.fromFile(uploadData.getFile())) : + s3Client.putObject(putObjectRequest, + RequestBody.fromInputStream(uploadData.getUploadStream(), + putObjectRequest.contentLength()))); incrementPutCompletedStatistics(true, len); // apply any post-write actions. - finishedWrite(putObjectRequest.getKey(), len, - result.getETag(), result.getVersionId(), + finishedWrite(putObjectRequest.key(), len, + response.eTag(), response.versionId(), putOptions); - return result; - } catch (SdkBaseException e) { + return response; + } catch (SdkException e) { incrementPutCompletedStatistics(false, len); throw e; } @@ -3013,12 +3189,8 @@ PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest, * @throws IllegalArgumentException if the length is negative */ private long getPutRequestLength(PutObjectRequest putObjectRequest) { - long len; - if (putObjectRequest.getFile() != null) { - len = putObjectRequest.getFile().length(); - } else { - len = putObjectRequest.getMetadata().getContentLength(); - } + long len = putObjectRequest.contentLength(); + Preconditions.checkState(len >= 0, "Cannot PUT object of unknown length"); return len; } @@ -3026,28 +3198,29 @@ private long getPutRequestLength(PutObjectRequest putObjectRequest) { /** * Upload part of a multi-partition file. * Increments the write and put counters. - * Important: this call does not close any input stream in the request. + * Important: this call does not close any input stream in the body. * * Retry Policy: none. - * @param request request * @param durationTrackerFactory duration tracker factory for operation + * @param request the upload part request. + * @param body the request body. * @return the result of the operation. - * @throws AmazonClientException on problems + * @throws AwsServiceException on problems */ @Retries.OnceRaw - UploadPartResult uploadPart(UploadPartRequest request, + UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body, final DurationTrackerFactory durationTrackerFactory) - throws AmazonClientException { - long len = request.getPartSize(); + throws AwsServiceException { + long len = request.contentLength(); incrementPutStartStatistics(len); try { - UploadPartResult uploadPartResult = trackDurationOfSupplier( + UploadPartResponse uploadPartResponse = trackDurationOfSupplier( nonNullDurationTrackerFactory(durationTrackerFactory), MULTIPART_UPLOAD_PART_PUT.getSymbol(), () -> - s3.uploadPart(request)); + s3Client.uploadPart(request, body)); incrementPutCompletedStatistics(true, len); - return uploadPartResult; - } catch (AmazonClientException e) { + return uploadPartResponse; + } catch (AwsServiceException e) { incrementPutCompletedStatistics(false, len); throw e; } @@ -3112,56 +3285,57 @@ public void incrementPutProgressStatistics(String key, long bytes) { * be deleted in a multiple object delete operation. * The number of rejected objects will be added to the metric * {@link Statistic#FILES_DELETE_REJECTED}. - * @throws AmazonClientException other amazon-layer failure. + * @throws AwsServiceException other amazon-layer failure. */ @Retries.RetryRaw private void removeKeysS3( - List keysToDelete, + List keysToDelete, boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, - IOException { + throws MultiObjectDeleteException, AwsServiceException, IOException { if (LOG.isDebugEnabled()) { LOG.debug("Initiating delete operation for {} objects", keysToDelete.size()); - for (DeleteObjectsRequest.KeyVersion key : keysToDelete) { - LOG.debug(" {} {}", key.getKey(), - key.getVersion() != null ? key.getVersion() : ""); + for (ObjectIdentifier objectIdentifier : keysToDelete) { + LOG.debug(" {} {}", objectIdentifier.key(), + objectIdentifier.versionId() != null ? objectIdentifier.versionId() : ""); } } if (keysToDelete.isEmpty()) { // exit fast if there are no keys to delete return; } - for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) { - blockRootDelete(keyVersion.getKey()); + for (ObjectIdentifier objectIdentifier : keysToDelete) { + blockRootDelete(objectIdentifier.key()); } try { if (enableMultiObjectsDelete) { if (keysToDelete.size() <= pageSize) { deleteObjects(getRequestFactory() - .newBulkDeleteRequest(keysToDelete)); + .newBulkDeleteRequestBuilder(keysToDelete) + .build()); } else { // Multi object deletion of more than 1000 keys is not supported // by s3. So we are paging the keys by page size. LOG.debug("Partitioning the keys to delete as it is more than " + "page size. Number of keys: {}, Page size: {}", keysToDelete.size(), pageSize); - for (List batchOfKeysToDelete : + for (List batchOfKeysToDelete : Lists.partition(keysToDelete, pageSize)) { deleteObjects(getRequestFactory() - .newBulkDeleteRequest(batchOfKeysToDelete)); + .newBulkDeleteRequestBuilder(batchOfKeysToDelete) + .build()); } } } else { - for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) { - deleteObject(keyVersion.getKey()); + for (ObjectIdentifier objectIdentifier : keysToDelete) { + deleteObject(objectIdentifier.key()); } } } catch (MultiObjectDeleteException ex) { // partial delete. // Update the stats with the count of the actual number of successful // deletions. - int rejected = ex.getErrors().size(); + int rejected = ex.errors().size(); noteDeleted(keysToDelete.size() - rejected, deleteFakeDir); incrementStatistic(FILES_DELETE_REJECTED, rejected); throw ex; @@ -3194,15 +3368,15 @@ private void noteDeleted(final int count, final boolean deleteFakeDir) { * a mistaken attempt to delete the root directory. * @throws MultiObjectDeleteException one or more of the keys could not * be deleted in a multiple object delete operation. - * @throws AmazonClientException amazon-layer failure. + * @throws AwsServiceException amazon-layer failure. * @throws IOException other IO Exception. */ @VisibleForTesting @Retries.RetryRaw public void removeKeys( - final List keysToDelete, + final List keysToDelete, final boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, + throws MultiObjectDeleteException, AwsServiceException, IOException { try (DurationInfo ignored = new DurationInfo(LOG, false, "Deleting %d keys", keysToDelete.size())) { @@ -3272,7 +3446,7 @@ protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOEx LOG.debug("Couldn't delete {} - does not exist: {}", path, e.toString()); instrumentation.errorIgnored(); return false; - } catch (AmazonClientException e) { + } catch (SdkException e) { throw translateException("delete", path, e); } } @@ -3286,7 +3460,7 @@ protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOEx */ @Retries.RetryTranslated private void createFakeDirectoryIfNecessary(Path f) - throws IOException, AmazonClientException { + throws IOException, SdkException { String key = pathToKey(f); // we only make the LIST call; the codepaths to get here should not // be reached if there is an empty dir marker -and if they do, it @@ -3306,7 +3480,7 @@ private void createFakeDirectoryIfNecessary(Path f) @Retries.RetryTranslated @VisibleForTesting protected void maybeCreateFakeParentDirectory(Path path) - throws IOException, AmazonClientException { + throws IOException, SdkException { Path parent = path.getParent(); if (parent != null && !parent.isRoot() && !isUnderMagicCommitPath(parent)) { createFakeDirectoryIfNecessary(parent); @@ -3360,11 +3534,11 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException, * @return the statuses of the files/directories in the given patch * @throws FileNotFoundException when the path does not exist; * @throws IOException due to an IO problem. - * @throws AmazonClientException on failures inside the AWS SDK + * @throws SdkException on failures inside the AWS SDK */ private RemoteIterator innerListStatus(Path f) throws FileNotFoundException, - IOException, AmazonClientException { + IOException, SdkException { Path path = qualify(f); LOG.debug("List status for path: {}", path); @@ -3428,15 +3602,15 @@ public S3ListRequest createListObjectsRequest(String key, private S3ListRequest createListObjectsRequest(String key, String delimiter, int limit) { if (!useListV1) { - ListObjectsV2Request request = - getRequestFactory().newListObjectsV2Request( + ListObjectsV2Request.Builder requestBuilder = + getRequestFactory().newListObjectsV2RequestBuilder( key, delimiter, limit); - return S3ListRequest.v2(request); + return S3ListRequest.v2(requestBuilder.build()); } else { - ListObjectsRequest request = - getRequestFactory().newListObjectsV1Request( + ListObjectsRequest.Builder requestBuilder = + getRequestFactory().newListObjectsV1RequestBuilder( key, delimiter, limit); - return S3ListRequest.v1(request); + return S3ListRequest.v1(requestBuilder.build()); } } @@ -3724,31 +3898,31 @@ S3AFileStatus s3GetFileStatus(final Path path, && probes.contains(StatusProbeEnum.Head)) { try { // look for the simple file - ObjectMetadata meta = getObjectMetadata(key); + HeadObjectResponse meta = getObjectMetadata(key); LOG.debug("Found exact file: normal file {}", key); - long contentLength = meta.getContentLength(); + long contentLength = meta.contentLength(); // check if CSE is enabled, then strip padded length. - if (isCSEEnabled - && meta.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null + if (isCSEEnabled && + meta.metadata().get(AWSHeaders.CRYPTO_CEK_ALGORITHM) != null && contentLength >= CSE_PADDING_LENGTH) { contentLength -= CSE_PADDING_LENGTH; } return new S3AFileStatus(contentLength, - dateToLong(meta.getLastModified()), + meta.lastModified().toEpochMilli(), path, getDefaultBlockSize(path), username, - meta.getETag(), - meta.getVersionId()); - } catch (AmazonServiceException e) { + meta.eTag(), + meta.versionId()); + } catch (AwsServiceException e) { // if the response is a 404 error, it just means that there is // no file at that path...the remaining checks will be needed. // But: an empty bucket is also a 404, so check for that // and fail. - if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) { + if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) { throw translateException("getFileStatus", path, e); } - } catch (AmazonClientException e) { + } catch (SdkException e) { throw translateException("getFileStatus", path, e); } } @@ -3791,11 +3965,11 @@ S3AFileStatus s3GetFileStatus(final Path path, LOG.debug("Found root directory"); return new S3AFileStatus(Tristate.TRUE, path, username); } - } catch (AmazonServiceException e) { - if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) { + } catch (AwsServiceException e) { + if (e.statusCode() != SC_404_NOT_FOUND || isUnknownBucket(e)) { throw translateException("getFileStatus", path, e); } - } catch (AmazonClientException e) { + } catch (SdkException e) { throw translateException("getFileStatus", path, e); } } @@ -3839,7 +4013,7 @@ private boolean s3Exists(final Path path, final Set probes) * @throws IOException IO problem * @throws FileAlreadyExistsException the destination file exists and * overwrite==false - * @throws AmazonClientException failure in the AWS SDK + * @throws SdkException failure in the AWS SDK */ @Override @AuditEntryPoint @@ -3894,13 +4068,12 @@ public void copyLocalFileFromTo(File file, Path from, Path to) throws IOExceptio to, () -> { final String key = pathToKey(to); - final ObjectMetadata om = newObjectMetadata(file.length()); Progressable progress = null; - PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, file); - S3AFileSystem.this.invoker.retry( - "putObject(" + "" + ")", to.toString(), - true, - () -> executePut(putObjectRequest, progress, putOptionsForPath(to))); + PutObjectRequest.Builder putObjectRequestBuilder = + newPutObjectRequestBuilder(key, file.length(), false); + S3AFileSystem.this.invoker.retry("putObject(" + "" + ")", to.toString(), true, + () -> executePut(putObjectRequestBuilder.build(), progress, putOptionsForPath(to), + file)); return null; }); @@ -3925,40 +4098,35 @@ public boolean createEmptyDir(Path path, StoreContext storeContext) /** * Execute a PUT via the transfer manager, blocking for completion. - * If the waiting for completion is interrupted, the upload will be - * aborted before an {@code InterruptedIOException} is thrown. * @param putObjectRequest request * @param progress optional progress callback * @param putOptions put object options * @return the upload result - * @throws InterruptedIOException if the blocking was interrupted. + * @throws IOException IO failure */ @Retries.OnceRaw("For PUT; post-PUT actions are RetrySwallowed") - UploadResult executePut( + PutObjectResponse executePut( final PutObjectRequest putObjectRequest, final Progressable progress, - final PutObjectOptions putOptions) - throws InterruptedIOException { - String key = putObjectRequest.getKey(); + final PutObjectOptions putOptions, + final File file) + throws IOException { + String key = putObjectRequest.key(); long len = getPutRequestLength(putObjectRequest); - UploadInfo info = putObject(putObjectRequest); - Upload upload = info.getUpload(); - ProgressableProgressListener listener = new ProgressableProgressListener( - this, key, upload, progress); - upload.addProgressListener(listener); - UploadResult result = waitForUploadCompletion(key, info); - listener.uploadCompleted(); + ProgressableProgressListener listener = + new ProgressableProgressListener(this, putObjectRequest.key(), progress); + UploadInfo info = putObject(putObjectRequest, file, listener); + PutObjectResponse result = waitForUploadCompletion(key, info).response(); + listener.uploadCompleted(info.getFileUpload()); // post-write actions finishedWrite(key, len, - result.getETag(), result.getVersionId(), putOptions); + result.eTag(), result.versionId(), putOptions); return result; } /** * Wait for an upload to complete. - * If the waiting for completion is interrupted, the upload will be - * aborted before an {@code InterruptedIOException} is thrown. * If the upload (or its result collection) failed, this is where * the failure is raised as an AWS exception. * Calls {@link #incrementPutCompletedStatistics(boolean, long)} @@ -3966,24 +4134,20 @@ UploadResult executePut( * @param key destination key * @param uploadInfo upload to wait for * @return the upload result - * @throws InterruptedIOException if the blocking was interrupted. + * @throws IOException IO failure */ @Retries.OnceRaw - UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo) - throws InterruptedIOException { - Upload upload = uploadInfo.getUpload(); + CompletedFileUpload waitForUploadCompletion(String key, UploadInfo uploadInfo) + throws IOException { + FileUpload upload = uploadInfo.getFileUpload(); try { - UploadResult result = upload.waitForUploadResult(); + CompletedFileUpload result = upload.completionFuture().join(); incrementPutCompletedStatistics(true, uploadInfo.getLength()); return result; - } catch (InterruptedException e) { + } catch (CompletionException e) { LOG.info("Interrupted: aborting upload"); incrementPutCompletedStatistics(false, uploadInfo.getLength()); - upload.abort(); - throw (InterruptedIOException) - new InterruptedIOException("Interrupted in PUT to " - + keyToQualifiedPath(key)) - .initCause(e); + throw extractException("upload", key, e); } } @@ -4076,17 +4240,13 @@ public void close() throws IOException { * both the expected state of this FS and of failures while being stopped. */ protected synchronized void stopAllServices() { - // shutting down the transfer manager also shuts - // down the S3 client it is bonded to. - if (transfers != null) { - try { - transfers.shutdownNow(true); - } catch (RuntimeException e) { - // catch and swallow for resilience. - LOG.debug("When shutting down", e); - } - transfers = null; - } + closeAutocloseables(LOG, transferManager, + s3Client, + getS3AsyncClient()); + transferManager = null; + s3Client = null; + s3AsyncClient = null; + // At this point the S3A client is shut down, // now the executor pools are closed HadoopExecutors.shutdown(boundedThreadPool, LOG, @@ -4241,21 +4401,11 @@ public List listAWSPolicyRules( * @throws IOException Other IO problems */ @Retries.RetryTranslated - private CopyResult copyFile(String srcKey, String dstKey, long size, + private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, S3ObjectAttributes srcAttributes, S3AReadOpContext readContext) - throws IOException, InterruptedIOException { + throws IOException { LOG.debug("copyFile {} -> {} ", srcKey, dstKey); - ProgressListener progressListener = progressEvent -> { - switch (progressEvent.getEventType()) { - case TRANSFER_PART_COMPLETED_EVENT: - incrementWriteOperations(); - break; - default: - break; - } - }; - ChangeTracker changeTracker = new ChangeTracker( keyToQualifiedPath(srcKey).toString(), changeDetectionPolicy, @@ -4267,7 +4417,7 @@ private CopyResult copyFile(String srcKey, String dstKey, long size, String action = "copyFile(" + srcKey + ", " + dstKey + ")"; Invoker readInvoker = readContext.getReadInvoker(); - ObjectMetadata srcom; + HeadObjectResponse srcom; try { srcom = once(action, srcKey, () -> @@ -4290,33 +4440,32 @@ private CopyResult copyFile(String srcKey, String dstKey, long size, action, srcKey, true, () -> { - CopyObjectRequest copyObjectRequest = - getRequestFactory().newCopyObjectRequest(srcKey, dstKey, srcom); - changeTracker.maybeApplyConstraint(copyObjectRequest); + CopyObjectRequest.Builder copyObjectRequestBuilder = + getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom); + changeTracker.maybeApplyConstraint(copyObjectRequestBuilder); incrementStatistic(OBJECT_COPY_REQUESTS); - Copy copy = transfers.copy(copyObjectRequest, - getAuditManager().createStateChangeListener()); - copy.addProgressListener(progressListener); - CopyOutcome copyOutcome = CopyOutcome.waitForCopy(copy); - InterruptedException interruptedException = - copyOutcome.getInterruptedException(); - if (interruptedException != null) { - // copy interrupted: convert to an IOException. - throw (IOException)new InterruptedIOException( - "Interrupted copying " + srcKey - + " to " + dstKey + ", cancelling") - .initCause(interruptedException); - } - SdkBaseException awsException = copyOutcome.getAwsException(); - if (awsException != null) { - changeTracker.processException(awsException, "copy"); - throw awsException; + + Copy copy = transferManager.copy( + CopyRequest.builder() + .copyObjectRequest(copyObjectRequestBuilder.build()) + .build()); + + try { + CompletedCopy completedCopy = copy.completionFuture().join(); + CopyObjectResponse result = completedCopy.response(); + changeTracker.processResponse(result); + incrementWriteOperations(); + instrumentation.filesCopied(1, size); + return result; + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SdkException) { + SdkException awsException = (SdkException)cause; + changeTracker.processException(awsException, "copy"); + throw awsException; + } + throw extractException(action, srcKey, e); } - CopyResult result = copyOutcome.getCopyResult(); - changeTracker.processResponse(result); - incrementWriteOperations(); - instrumentation.filesCopied(1, size); - return result; }); } @@ -4325,16 +4474,16 @@ private CopyResult copyFile(String srcKey, String dstKey, long size, * Retry policy: none + untranslated. * @param request request to initiate * @return the result of the call - * @throws AmazonClientException on failures inside the AWS SDK + * @throws SdkException on failures inside the AWS SDK * @throws IOException Other IO problems */ @Retries.OnceRaw - InitiateMultipartUploadResult initiateMultipartUpload( - InitiateMultipartUploadRequest request) throws IOException { - LOG.debug("Initiate multipart upload to {}", request.getKey()); + CreateMultipartUploadResponse initiateMultipartUpload( + CreateMultipartUploadRequest request) throws IOException { + LOG.debug("Initiate multipart upload to {}", request.key()); return trackDurationOfSupplier(getDurationTrackerFactory(), OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(), - () -> getAmazonS3Client().initiateMultipartUpload(request)); + () -> s3Client.createMultipartUpload(request)); } /** @@ -4407,22 +4556,22 @@ private PutObjectOptions putOptionsForPath(Path path) { */ @Retries.RetryExceptionsSwallowed private void deleteUnnecessaryFakeDirectories(Path path) { - List keysToRemove = new ArrayList<>(); + List keysToRemove = new ArrayList<>(); while (!path.isRoot()) { String key = pathToKey(path); key = (key.endsWith("/")) ? key : (key + "/"); LOG.trace("To delete unnecessary fake directory {} for {}", key, path); - keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key)); + keysToRemove.add(ObjectIdentifier.builder().key(key).build()); path = path.getParent(); } try { removeKeys(keysToRemove, true); - } catch(AmazonClientException | IOException e) { + } catch (AwsServiceException | IOException e) { instrumentation.errorIgnored(); if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); - for(DeleteObjectsRequest.KeyVersion kv : keysToRemove) { - sb.append(kv.getKey()).append(","); + for (ObjectIdentifier objectIdentifier : keysToRemove) { + sb.append(objectIdentifier.key()).append(","); } LOG.debug("While deleting keys {} ", sb.toString(), e); } @@ -4455,11 +4604,18 @@ private void createFakeDirectory(final String objectName, @Retries.RetryTranslated private void createEmptyObject(final String objectName, PutObjectOptions putOptions) throws IOException { - invoker.retry("PUT 0-byte object ", objectName, - true, () -> - putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName), - putOptions, - getDurationTrackerFactory())); + final InputStream im = new InputStream() { + @Override + public int read() throws IOException { + return -1; + } + }; + + S3ADataBlocks.BlockUploadData uploadData = new S3ADataBlocks.BlockUploadData(im); + + invoker.retry("PUT 0-byte object ", objectName, true, + () -> putObjectDirect(getRequestFactory().newDirectoryMarkerRequest(objectName).build(), + putOptions, uploadData, false, getDurationTrackerFactory())); incrementPutProgressStatistics(objectName, 0); instrumentation.directoryCreated(); } @@ -4716,10 +4872,10 @@ public EtagChecksum getFileChecksum(Path f, final long length) ETAG_CHECKSUM_ENABLED_DEFAULT)) { return trackDurationAndSpan(INVOCATION_GET_FILE_CHECKSUM, path, () -> { LOG.debug("getFileChecksum({})", path); - ObjectMetadata headers = getObjectMetadata(path, null, + HeadObjectResponse headers = getObjectMetadata(path, null, invoker, "getFileChecksum are"); - String eTag = headers.getETag(); + String eTag = headers.eTag(); return eTag != null ? new EtagChecksum(eTag) : null; }); } else { @@ -4801,11 +4957,18 @@ protected final class HeaderProcessingCallbacksImpl implements HeaderProcessing.HeaderProcessingCallbacks { @Override - public ObjectMetadata getObjectMetadata(final String key) + public HeadObjectResponse getObjectMetadata(final String key) throws IOException { return once("getObjectMetadata", key, () -> S3AFileSystem.this.getObjectMetadata(key)); } + + @Override + public HeadBucketResponse getBucketMetadata() + throws IOException { + return once("getBucketMetadata", bucket, () -> + S3AFileSystem.this.getBucketMetadata()); + } } /** * {@inheritDoc}. @@ -4918,7 +5081,7 @@ private RemoteIterator innerListFiles( // If we have reached here, it means either there are files // in this directory or it is empty. return listFilesAssumingDir; - } catch (AmazonClientException e) { + } catch (SdkException e) { throw translateException("listFiles", path, e); } } @@ -5016,8 +5179,7 @@ public MultipartUtils.UploadIterator listUploads(@Nullable String prefix) // span is picked up retained in the listing. return trackDurationAndSpan(MULTIPART_UPLOAD_LIST, prefix, null, () -> MultipartUtils.listMultipartUploads( - createStoreContext(), - s3, prefix, maxKeys + createStoreContext(), s3Client, prefix, maxKeys )); } @@ -5028,7 +5190,7 @@ public MultipartUtils.UploadIterator listUploads(@Nullable String prefix) * Retry policy: retry, translated. * @return a listing of multipart uploads. * @param prefix prefix to scan for, "" for none - * @throws IOException IO failure, including any uprated AmazonClientException + * @throws IOException IO failure, including any uprated SdkException */ @InterfaceAudience.Private @Retries.RetryTranslated @@ -5040,9 +5202,9 @@ public List listMultipartUploads(String prefix) } String p = prefix; return invoker.retry("listMultipartUploads", p, true, () -> { - ListMultipartUploadsRequest request = getRequestFactory() - .newListMultipartUploadsRequest(p); - return s3.listMultipartUploads(request).getMultipartUploads(); + ListMultipartUploadsRequest.Builder requestBuilder = getRequestFactory() + .newListMultipartUploadsRequestBuilder(p); + return s3Client.listMultipartUploads(requestBuilder.build()).uploads(); }); } @@ -5055,10 +5217,10 @@ public List listMultipartUploads(String prefix) @Retries.OnceRaw void abortMultipartUpload(String destKey, String uploadId) { LOG.info("Aborting multipart upload {} to {}", uploadId, destKey); - getAmazonS3Client().abortMultipartUpload( - getRequestFactory().newAbortMultipartUploadRequest( + s3Client.abortMultipartUpload( + getRequestFactory().newAbortMultipartUploadRequestBuilder( destKey, - uploadId)); + uploadId).build()); } /** @@ -5070,18 +5232,18 @@ void abortMultipartUpload(String destKey, String uploadId) { void abortMultipartUpload(MultipartUpload upload) { String destKey; String uploadId; - destKey = upload.getKey(); - uploadId = upload.getUploadId(); + destKey = upload.key(); + uploadId = upload.uploadId(); if (LOG.isInfoEnabled()) { DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}", - uploadId, destKey, upload.getInitiator(), - df.format(upload.getInitiated())); + uploadId, destKey, upload.initiator(), + df.format(Date.from(upload.initiated()))); } - getAmazonS3Client().abortMultipartUpload( - getRequestFactory().newAbortMultipartUploadRequest( + s3Client.abortMultipartUpload( + getRequestFactory().newAbortMultipartUploadRequestBuilder( destKey, - uploadId)); + uploadId).build()); } /** @@ -5471,4 +5633,5 @@ public boolean isCSEEnabled() { public boolean isMultipartUploadEnabled() { return isMultipartUploadEnabled; } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java index 4b50ab2c04bd9..2ed9083efcddd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java @@ -22,6 +22,7 @@ import java.io.Closeable; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.io.InterruptedIOException; import java.net.SocketTimeoutException; import java.nio.ByteBuffer; @@ -31,9 +32,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.IntFunction; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,6 @@ import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileRange; -import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.impl.CombinedFileRange; import org.apache.hadoop.fs.VectoredReadUtils; @@ -61,6 +61,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.functional.CallableRaisingIOE; + import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.hadoop.fs.VectoredReadUtils.isOrderedDisjoint; @@ -125,14 +126,9 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead, */ private volatile boolean closed; /** - * wrappedStream is associated with an object (instance of S3Object). When - * the object is garbage collected, the associated wrappedStream will be - * closed. Keep a reference to this object to prevent the wrapperStream - * still in use from being closed unexpectedly due to garbage collection. - * See HADOOP-17338 for details. + * Input stream returned by a getObject call. */ - private S3Object object; - private S3ObjectInputStream wrappedStream; + private ResponseInputStream wrappedStream; private final S3AReadOpContext context; private final InputStreamCallbacks client; @@ -271,28 +267,22 @@ private synchronized void reopen(String reason, long targetPos, long length, uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos, inputPolicy); + GetObjectRequest request = client.newGetRequestBuilder(key) + .range(S3AUtils.formatRange(targetPos, contentRangeFinish - 1)) + .applyMutation(changeTracker::maybeApplyConstraint) + .build(); long opencount = streamStatistics.streamOpened(); - GetObjectRequest request = client.newGetRequest(key) - .withRange(targetPos, contentRangeFinish - 1); String operation = opencount == 0 ? OPERATION_OPEN : OPERATION_REOPEN; String text = String.format("%s %s at %d", operation, uri, targetPos); - changeTracker.maybeApplyConstraint(request); - - object = onceTrackingDuration(text, uri, + wrappedStream = onceTrackingDuration(text, uri, streamStatistics.initiateGetRequest(), () -> client.getObject(request)); - - changeTracker.processResponse(object, operation, + changeTracker.processResponse(wrappedStream.response(), operation, targetPos); - wrappedStream = object.getObjectContent(); - contentRangeStart = targetPos; - if (wrappedStream == null) { - throw new PathIOException(uri, - "Null IO stream from " + operation + " of (" + reason + ") "); - } + contentRangeStart = targetPos; this.pos = targetPos; } @@ -505,14 +495,15 @@ public synchronized int read() throws IOException { */ @Retries.OnceTranslated private void onReadFailure(IOException ioe, boolean forceAbort) { + GetObjectResponse objectResponse = wrappedStream == null ? null : wrappedStream.response(); if (LOG.isDebugEnabled()) { LOG.debug("Got exception while trying to read from stream {}, " + "client: {} object: {}, trying to recover: ", - uri, client, object, ioe); + uri, client, objectResponse, ioe); } else { LOG.info("Got exception while trying to read from stream {}, " + "client: {} object: {}, trying to recover: " + ioe, - uri, client, object); + uri, client, objectResponse); } streamStatistics.readException(); closeStream("failure recovery", forceAbort, false); @@ -672,7 +663,6 @@ private CompletableFuture closeStream( CompletableFuture operation; SDKStreamDrainer drainer = new SDKStreamDrainer( uri, - object, wrappedStream, shouldAbort, (int) remaining, @@ -694,7 +684,6 @@ private CompletableFuture closeStream( // either the stream is closed in the blocking call or the async call is // submitted with its own copy of the references wrappedStream = null; - object = null; return operation; } @@ -910,23 +899,19 @@ public void readVectored(List ranges, private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange, IntFunction allocate) { LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr); - // This reference must be kept till all buffers are populated as this is a - // finalizable object which closes the internal stream when gc triggers. - S3Object objectRange = null; - S3ObjectInputStream objectContent = null; + ResponseInputStream rangeContent = null; try { - objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange", + rangeContent = getS3ObjectInputStream("readCombinedFileRange", combinedFileRange.getOffset(), combinedFileRange.getLength()); - objectContent = objectRange.getObjectContent(); - populateChildBuffers(combinedFileRange, objectContent, allocate); + populateChildBuffers(combinedFileRange, rangeContent, allocate); } catch (Exception ex) { LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex); for(FileRange child : combinedFileRange.getUnderlying()) { child.getData().completeExceptionally(ex); } } finally { - IOUtils.cleanupWithLogger(LOG, objectRange, objectContent); + IOUtils.cleanupWithLogger(LOG, rangeContent); } LOG.debug("Finished reading range {} from path {} ", combinedFileRange, pathStr); } @@ -939,7 +924,7 @@ private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRa * @throws IOException any IOE. */ private void populateChildBuffers(CombinedFileRange combinedFileRange, - S3ObjectInputStream objectContent, + InputStream objectContent, IntFunction allocate) throws IOException { // If the combined file range just contains a single child // range, we only have to fill that one child buffer else @@ -971,7 +956,7 @@ private void populateChildBuffers(CombinedFileRange combinedFileRange, * @param drainQuantity how many bytes to drain. * @throws IOException any IOE. */ - private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQuantity) + private void drainUnnecessaryData(InputStream objectContent, long drainQuantity) throws IOException { int drainBytes = 0; int readCount; @@ -1013,28 +998,24 @@ private void validateRangeRequest(FileRange range) throws EOFException { */ private void readSingleRange(FileRange range, ByteBuffer buffer) { LOG.debug("Start reading range {} from path {} ", range, pathStr); - // This reference must be kept till all buffers are populated as this is a - // finalizable object which closes the internal stream when gc triggers. - S3Object objectRange = null; - S3ObjectInputStream objectContent = null; + ResponseInputStream objectRange = null; try { long position = range.getOffset(); int length = range.getLength(); - objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length); - objectContent = objectRange.getObjectContent(); - populateBuffer(length, buffer, objectContent); + objectRange = getS3ObjectInputStream("readSingleRange", position, length); + populateBuffer(length, buffer, objectRange); range.getData().complete(buffer); } catch (Exception ex) { LOG.warn("Exception while reading a range {} from path {} ", range, pathStr, ex); range.getData().completeExceptionally(ex); } finally { - IOUtils.cleanupWithLogger(LOG, objectRange, objectContent); + IOUtils.cleanupWithLogger(LOG, objectRange); } LOG.debug("Finished reading range {} from path {} ", range, pathStr); } /** - * Get the s3 object for S3 server for a specified range. + * Get the s3 object input stream for S3 server for a specified range. * Also checks if the vectored io operation has been stopped before and after * the http get request such that we don't waste time populating the buffers. * @param operationName name of the operation for which get object on S3 is called. @@ -1043,15 +1024,11 @@ private void readSingleRange(FileRange range, ByteBuffer buffer) { * @return result s3 object. * @throws IOException exception if any. */ - private S3Object getS3ObjectAndValidateNotNull(final String operationName, - final long position, - final int length) throws IOException { + private ResponseInputStream getS3ObjectInputStream( + final String operationName, final long position, final int length) throws IOException { checkIfVectoredIOStopped(); - S3Object objectRange = getS3Object(operationName, position, length); - if (objectRange.getObjectContent() == null) { - throw new PathIOException(uri, - "Null IO stream received during " + operationName); - } + ResponseInputStream objectRange = + getS3Object(operationName, position, length); checkIfVectoredIOStopped(); return objectRange; } @@ -1066,7 +1043,7 @@ private S3Object getS3ObjectAndValidateNotNull(final String operationName, */ private void populateBuffer(int length, ByteBuffer buffer, - S3ObjectInputStream objectContent) throws IOException { + InputStream objectContent) throws IOException { if (buffer.isDirect()) { VectoredReadUtils.readInDirectBuffer(length, buffer, @@ -1091,7 +1068,7 @@ private void populateBuffer(int length, * @param length number of bytes to fill in dest. * @throws IOException any IOE. */ - private void readByteArray(S3ObjectInputStream objectContent, + private void readByteArray(InputStream objectContent, byte[] dest, int offset, int length) throws IOException { @@ -1118,13 +1095,16 @@ private void readByteArray(S3ObjectInputStream objectContent, * @return S3Object result s3 object. * @throws IOException exception if any. */ - private S3Object getS3Object(String operationName, long position, - int length) throws IOException { - final GetObjectRequest request = client.newGetRequest(key) - .withRange(position, position + length - 1); - changeTracker.maybeApplyConstraint(request); + private ResponseInputStream getS3Object(String operationName, + long position, + int length) + throws IOException { + final GetObjectRequest request = client.newGetRequestBuilder(key) + .range(S3AUtils.formatRange(position, position + length - 1)) + .applyMutation(changeTracker::maybeApplyConstraint) + .build(); DurationTracker tracker = streamStatistics.initiateGetRequest(); - S3Object objectRange; + ResponseInputStream objectRange; Invoker invoker = context.getReadInvoker(); try { objectRange = invoker.retry(operationName, pathStr, true, @@ -1139,7 +1119,7 @@ private S3Object getS3Object(String operationName, long position, } finally { tracker.close(); } - changeTracker.processResponse(objectRange, operationName, + changeTracker.processResponse(objectRange.response(), operationName, position); return objectRange; } @@ -1293,11 +1273,11 @@ public IOStatistics getIOStatistics() { public interface InputStreamCallbacks extends Closeable { /** - * Create a GET request. + * Create a GET request builder. * @param key object key - * @return the request + * @return the request builder */ - GetObjectRequest newGetRequest(String key); + GetObjectRequest.Builder newGetRequestBuilder(String key); /** * Execute the request. @@ -1305,7 +1285,7 @@ public interface InputStreamCallbacks extends Closeable { * @return the response */ @Retries.OnceRaw - S3Object getObject(GetObjectRequest request); + ResponseInputStream getObject(GetObjectRequest request); /** * Submit some asynchronous work, for example, draining a stream. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java new file mode 100644 index 0000000000000..23c4d3501206a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; +import java.nio.file.AccessDeniedException; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.store.audit.AuditEntryPoint; + +/** + * This is an unstable interface for access to S3A Internal state, S3 operations + * and the S3 client connector itself. + */ +@InterfaceStability.Unstable +@InterfaceAudience.LimitedPrivate("testing/diagnostics") +public interface S3AInternals { + + /** + * Returns the S3 client used by this filesystem. + * Will log at debug. + *

+ * Warning + * This bypasses core S3A operations, including auditing. + * It is intended for use in testing, diagnostics and for accessing + * operations not available through the S3A connector itself. + *

+ * Unless audit spans are created through the S3AFileSystem, make + * sure that {@code fs.s3a.audit.reject.out.of.span.operations} is + * set to false. + *

+ * Mocking note: this is the same S3Client as is used by the owning + * filesystem; changes to this client will be reflected by changes + * in the behavior of that filesystem. + * @param reason a justification for requesting access. + * @return S3Client + */ + S3Client getAmazonS3Client(String reason); + + /** + * Get the region of a bucket. + * Invoked from StoreContext; consider an entry point. + * @return the region in which a bucket is located + * @throws AccessDeniedException if the caller lacks permission. + * @throws IOException on any failure. + */ + @Retries.RetryTranslated + @AuditEntryPoint + String getBucketLocation() throws IOException; + + /** + * Get the region of a bucket; fixing up the region so it can be used + * in the builders of other AWS clients. + * Requires the caller to have the AWS role permission + * {@code s3:GetBucketLocation}. + * Retry policy: retrying, translated. + * @param bucketName the name of the bucket + * @return the region in which a bucket is located + * @throws AccessDeniedException if the caller lacks permission. + * @throws IOException on any failure. + */ + @AuditEntryPoint + @Retries.RetryTranslated + String getBucketLocation(String bucketName) throws IOException; + + /** + * Low-level call to get at the object metadata. + * Auditing: An audit entry point. + * @param path path to the object. This will be qualified. + * @return metadata + * @throws IOException IO and object access problems. + */ + @AuditEntryPoint + @Retries.RetryTranslated + HeadObjectResponse getObjectMetadata(Path path) throws IOException; + + /** + * Get a shared copy of the AWS credentials, with its reference + * counter updated. + * Caller is required to call {@code close()} on this after + * they have finished using it. + * @param purpose what is this for? This is for logging + * @return a reference to shared credentials. + */ + AWSCredentialProviderList shareCredentials(String purpose); + + /** + * Request bucket metadata. + * @return the metadata + * @throws UnknownStoreException the bucket is absent + * @throws IOException any other problem talking to S3 + */ + @AuditEntryPoint + @Retries.RetryTranslated + HeadBucketResponse getBucketMetadata() throws IOException; +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java index 528a99f5e0966..fdb4591476c56 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java @@ -30,7 +30,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import com.amazonaws.AmazonClientException; +import software.amazon.awssdk.core.exception.SdkException; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +43,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.net.ConnectTimeoutException; -import org.apache.hadoop.util.Preconditions; + import static org.apache.hadoop.io.retry.RetryPolicies.*; @@ -68,9 +69,9 @@ * * The retry policy is all built around that of the normal IO exceptions, * particularly those extracted from - * {@link S3AUtils#translateException(String, Path, AmazonClientException)}. + * {@link S3AUtils#translateException(String, Path, SdkException)}. * Because the {@link #shouldRetry(Exception, int, int, boolean)} method - * does this translation if an {@code AmazonClientException} is processed, + * does this translation if an {@code SdkException} is processed, * the policy defined for the IOEs also applies to the original exceptions. * * Put differently: this retry policy aims to work for handlers of the @@ -111,6 +112,11 @@ public class S3ARetryPolicy implements RetryPolicy { */ protected final RetryPolicy connectivityFailure; + /** + * Handling of AWSClientIOException and subclasses. + */ + protected final RetryPolicy retryAwsClientExceptions; + /** * Instantiate. * @param conf configuration to read. @@ -138,6 +144,11 @@ public S3ARetryPolicy(Configuration conf) { retryIdempotentCalls = new FailNonIOEs( new IdempotencyRetryFilter(baseExponentialRetry)); + // retry on AWSClientIOException and possibly subclasses; + // See: HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid + // currently the normal retryIdempotentCalls policy is used. + retryAwsClientExceptions = retryIdempotentCalls; + // and a separate policy for throttle requests, which are considered // repeatable, even for non-idempotent calls, as the service // rejected the call entirely @@ -223,9 +234,11 @@ protected Map, RetryPolicy> createExceptionMap() { // server didn't respond. policyMap.put(AWSNoResponseException.class, retryIdempotentCalls); + // use specific retry policy for aws client exceptions + policyMap.put(AWSClientIOException.class, retryAwsClientExceptions); + policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions); + // other operations - policyMap.put(AWSClientIOException.class, retryIdempotentCalls); - policyMap.put(AWSServiceIOException.class, retryIdempotentCalls); policyMap.put(AWSS3IOException.class, retryIdempotentCalls); policyMap.put(SocketTimeoutException.class, retryIdempotentCalls); @@ -242,11 +255,10 @@ public RetryAction shouldRetry(Exception exception, boolean idempotent) throws Exception { Preconditions.checkArgument(exception != null, "Null exception"); Exception ex = exception; - if (exception instanceof AmazonClientException) { - // uprate the amazon client exception for the purpose of exception + if (exception instanceof SdkException) { + // update the sdk exception for the purpose of exception // processing. - ex = S3AUtils.translateException("", "", - (AmazonClientException) exception); + ex = S3AUtils.translateException("", "", (SdkException) exception); } return retryPolicy.shouldRetry(ex, retries, failovers, idempotent); } @@ -315,4 +327,28 @@ public RetryAction shouldRetry(Exception e, } } + /** + * Policy where AWS SDK exceptions are retried if they state that they are retryable. + * See HADOOP-18871. S3ARetryPolicy to use sdk exception retryable() if it is valid. + */ + private static final class RetryFromAWSClientExceptionPolicy implements RetryPolicy { + + private final RetryPolicy next; + + private RetryFromAWSClientExceptionPolicy(RetryPolicy next) { + this.next = next; + } + + @Override + public RetryAction shouldRetry(Exception e, + int retries, + int failovers, + boolean isIdempotentOrAtMostOnce) throws Exception { + return + e instanceof AWSClientIOException ? + next.shouldRetry(e, retries, failovers, ((AWSClientIOException)e).retryable()) + : RetryAction.FAIL; + } + } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 27f061482ca98..093608fb528c7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -18,20 +18,12 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AbortedException; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.Protocol; -import com.amazonaws.SdkBaseException; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.retry.RetryUtils; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.util.Preconditions; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.AbortedException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.retry.RetryUtils; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -44,16 +36,12 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.functional.RemoteIterators; -import org.apache.hadoop.fs.s3a.audit.AuditFailureException; -import org.apache.hadoop.fs.s3a.audit.AuditIntegration; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; -import org.apache.hadoop.fs.s3a.impl.NetworkBinding; -import org.apache.hadoop.fs.s3a.impl.V2Migration; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.security.ProviderUtils; -import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Lists; import org.slf4j.Logger; @@ -74,23 +62,24 @@ import java.net.URI; import java.nio.file.AccessDeniedException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.hadoop.fs.s3a.AWSCredentialProviderList.maybeTranslateCredentialException; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; -import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.instantiationException; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -104,13 +93,7 @@ public final class S3AUtils { private static final Logger LOG = LoggerFactory.getLogger(S3AUtils.class); - static final String CONSTRUCTOR_EXCEPTION = "constructor exception"; - static final String INSTANTIATION_EXCEPTION - = "instantiation exception"; - static final String NOT_AWS_PROVIDER = - "does not implement AWSCredentialsProvider"; - static final String ABSTRACT_PROVIDER = - "is abstract and therefore cannot be created"; + static final String ENDPOINT_KEY = "Endpoint"; /** Filesystem is closed; kept here to keep the errors close. */ @@ -145,21 +128,13 @@ public final class S3AUtils { private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s"; - /** - * Error message when the AWS provider list built up contains a forbidden - * entry. - */ - @VisibleForTesting - public static final String E_FORBIDDEN_AWS_PROVIDER - = "AWS provider class cannot be used"; - private S3AUtils() { } /** * Translate an exception raised in an operation into an IOException. * The specific type of IOException depends on the class of - * {@link AmazonClientException} passed in, and any status codes included + * {@link SdkException} passed in, and any status codes included * in the operation. That is: HTTP error codes are examined and can be * used to build a more specific response. * @@ -172,14 +147,14 @@ private S3AUtils() { */ public static IOException translateException(String operation, Path path, - AmazonClientException exception) { + SdkException exception) { return translateException(operation, path.toString(), exception); } /** * Translate an exception raised in an operation into an IOException. * The specific type of IOException depends on the class of - * {@link AmazonClientException} passed in, and any status codes included + * {@link SdkException} passed in, and any status codes included * in the operation. That is: HTTP error codes are examined and can be * used to build a more specific response. * @param operation operation @@ -190,12 +165,14 @@ public static IOException translateException(String operation, @SuppressWarnings("ThrowableInstanceNeverThrown") public static IOException translateException(@Nullable String operation, String path, - SdkBaseException exception) { + SdkException exception) { String message = String.format("%s%s: %s", operation, StringUtils.isNotEmpty(path)? (" on " + path) : "", exception); - if (!(exception instanceof AmazonServiceException)) { + if (!(exception instanceof AwsServiceException)) { + // exceptions raised client-side: connectivity, auth, network problems... + Exception innerCause = containsInterruptedException(exception); if (innerCause != null) { // interrupted IO, or a socket exception underneath that class @@ -207,57 +184,60 @@ public static IOException translateException(@Nullable String operation, } // if the exception came from the auditor, hand off translation // to it. - if (exception instanceof AuditFailureException) { - return AuditIntegration.translateAuditException(path, (AuditFailureException) exception); + IOException ioe = maybeTranslateAuditException(path, exception); + if (ioe != null) { + return ioe; } - if (exception instanceof CredentialInitializationException) { - // the exception raised by AWSCredentialProvider list if the - // credentials were not accepted, - return (AccessDeniedException)new AccessDeniedException(path, null, - exception.toString()).initCause(exception); + ioe = maybeTranslateCredentialException(path, exception); + if (ioe != null) { + return ioe; + } else { + // no custom handling. + return new AWSClientIOException(message, exception); } - return new AWSClientIOException(message, exception); } else { + // "error response returned by an S3 or other service." + // These contain more details and should be translated based + // on the HTTP status code and other details. IOException ioe; - AmazonServiceException ase = (AmazonServiceException) exception; + AwsServiceException ase = (AwsServiceException) exception; // this exception is non-null if the service exception is an s3 one - AmazonS3Exception s3Exception = ase instanceof AmazonS3Exception - ? (AmazonS3Exception) ase + S3Exception s3Exception = ase instanceof S3Exception + ? (S3Exception) ase : null; - int status = ase.getStatusCode(); - message = message + ":" + ase.getErrorCode(); + int status = ase.statusCode(); + if (ase.awsErrorDetails() != null) { + message = message + ":" + ase.awsErrorDetails().errorCode(); + } switch (status) { - case 301: - case 307: + case SC_301_MOVED_PERMANENTLY: + case SC_307_TEMPORARY_REDIRECT: if (s3Exception != null) { - if (s3Exception.getAdditionalDetails() != null && - s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) { - message = String.format("Received permanent redirect response to " - + "endpoint %s. This likely indicates that the S3 endpoint " - + "configured in %s does not match the AWS region containing " - + "the bucket.", - s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT); - } + message = String.format("Received permanent redirect response to " + + "region %s. This likely indicates that the S3 region " + + "configured in %s does not match the AWS region containing " + "the bucket.", + s3Exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER), + AWS_REGION); ioe = new AWSRedirectException(message, s3Exception); } else { ioe = new AWSRedirectException(message, ase); } break; - case 400: + case SC_400_BAD_REQUEST: ioe = new AWSBadRequestException(message, ase); break; // permissions - case 401: - case 403: + case SC_401_UNAUTHORIZED: + case SC_403_FORBIDDEN: ioe = new AccessDeniedException(path, null, message); ioe.initCause(ase); break; // the object isn't there - case 404: + case SC_404_NOT_FOUND: if (isUnknownBucket(ase)) { // this is a missing bucket ioe = new UnknownStoreException(path, message, ase); @@ -270,20 +250,20 @@ public static IOException translateException(@Nullable String operation, // this also surfaces sometimes and is considered to // be ~ a not found exception. - case 410: + case SC_410_GONE: ioe = new FileNotFoundException(message); ioe.initCause(ase); break; // method not allowed; seen on S3 Select. // treated as a bad request - case 405: + case SC_405_METHOD_NOT_ALLOWED: ioe = new AWSBadRequestException(message, s3Exception); break; // out of range. This may happen if an object is overwritten with // a shorter one while it is being read. - case 416: + case SC_416_RANGE_NOT_SATISFIABLE: ioe = new EOFException(message); ioe.initCause(ase); break; @@ -291,26 +271,26 @@ public static IOException translateException(@Nullable String operation, // this has surfaced as a "no response from server" message. // so rare we haven't replicated it. // Treating as an idempotent proxy error. - case 443: - case 444: + case SC_443_NO_RESPONSE: + case SC_444_NO_RESPONSE: ioe = new AWSNoResponseException(message, ase); break; // throttling - case 503: + case SC_503_SERVICE_UNAVAILABLE: ioe = new AWSServiceThrottledException(message, ase); break; // internal error - case 500: + case SC_500_INTERNAL_SERVER_ERROR: ioe = new AWSStatus500Exception(message, ase); break; - case 200: + case SC_200_OK: if (exception instanceof MultiObjectDeleteException) { // failure during a bulk delete - return translateDeleteException(message, - (MultiObjectDeleteException) exception); + return ((MultiObjectDeleteException) exception) + .translateException(message); } // other 200: FALL THROUGH @@ -336,10 +316,35 @@ public static IOException translateException(@Nullable String operation, public static IOException extractException(String operation, String path, ExecutionException ee) { + return convertExceptionCause(operation, path, ee.getCause()); + } + + /** + * Extract an exception from a failed future, and convert to an IOE. + * @param operation operation which failed + * @param path path operated on (may be null) + * @param ce completion exception + * @return an IOE which can be thrown + */ + public static IOException extractException(String operation, + String path, + CompletionException ce) { + return convertExceptionCause(operation, path, ce.getCause()); + } + + /** + * Convert the cause of a concurrent exception to an IOE. + * @param operation operation which failed + * @param path path operated on (may be null) + * @param cause cause of a concurrent exception + * @return an IOE which can be thrown + */ + private static IOException convertExceptionCause(String operation, + String path, + Throwable cause) { IOException ioe; - Throwable cause = ee.getCause(); - if (cause instanceof AmazonClientException) { - ioe = translateException(operation, path, (AmazonClientException) cause); + if (cause instanceof SdkException) { + ioe = translateException(operation, path, (SdkException) cause); } else if (cause instanceof IOException) { ioe = (IOException) cause; } else { @@ -377,7 +382,7 @@ static Exception containsInterruptedException(Throwable thrown) { * @return an IOE which can be rethrown */ private static InterruptedIOException translateInterruptedException( - SdkBaseException exception, + SdkException exception, final Exception innerCause, String message) { InterruptedIOException ioe; @@ -388,6 +393,7 @@ private static InterruptedIOException translateInterruptedException( if (name.endsWith(".ConnectTimeoutException") || name.endsWith(".ConnectionPoolTimeoutException") || name.endsWith("$ConnectTimeoutException")) { + // TODO: review in v2 // TCP connection http timeout from the shaded or unshaded filenames // com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException ioe = new ConnectTimeoutException(message); @@ -411,10 +417,10 @@ private static InterruptedIOException translateInterruptedException( */ public static boolean isThrottleException(Exception ex) { return ex instanceof AWSServiceThrottledException - || (ex instanceof AmazonServiceException - && 503 == ((AmazonServiceException)ex).getStatusCode()) - || (ex instanceof SdkBaseException - && RetryUtils.isThrottlingException((SdkBaseException) ex)); + || (ex instanceof AwsServiceException + && 503 == ((AwsServiceException)ex).statusCode()) + || (ex instanceof SdkException + && RetryUtils.isThrottlingException((SdkException) ex)); } /** @@ -424,7 +430,8 @@ public static boolean isThrottleException(Exception ex) { * @param ex exception * @return true if this is believed to be a sign the connection was broken. */ - public static boolean isMessageTranslatableToEOF(SdkBaseException ex) { + public static boolean isMessageTranslatableToEOF(SdkException ex) { + // TODO: review in v2 return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER) || ex.toString().contains(EOF_READ_DIFFERENT_LENGTH); } @@ -434,47 +441,26 @@ public static boolean isMessageTranslatableToEOF(SdkBaseException ex) { * @param e exception * @return string details */ - public static String stringify(AmazonServiceException e) { + public static String stringify(AwsServiceException e) { StringBuilder builder = new StringBuilder( - String.format("%s: %s error %d: %s; %s%s%n", - e.getErrorType(), - e.getServiceName(), - e.getStatusCode(), - e.getErrorCode(), - e.getErrorMessage(), - (e.isRetryable() ? " (retryable)": "") + String.format("%s error %d: %s; %s%s%n", + e.awsErrorDetails().serviceName(), + e.statusCode(), + e.awsErrorDetails().errorCode(), + e.awsErrorDetails().errorMessage(), + (e.retryable() ? " (retryable)": "") )); - String rawResponseContent = e.getRawResponseContent(); + String rawResponseContent = e.awsErrorDetails().rawResponse().asUtf8String(); if (rawResponseContent != null) { builder.append(rawResponseContent); } return builder.toString(); } - /** - * Get low level details of an amazon exception for logging; multi-line. - * @param e exception - * @return string details - */ - public static String stringify(AmazonS3Exception e) { - // get the low level details of an exception, - StringBuilder builder = new StringBuilder( - stringify((AmazonServiceException) e)); - Map details = e.getAdditionalDetails(); - if (details != null) { - builder.append('\n'); - for (Map.Entry d : details.entrySet()) { - builder.append(d.getKey()).append('=') - .append(d.getValue()).append('\n'); - } - } - return builder.toString(); - } - /** * Create a files status instance from a listing. * @param keyPath path to entry - * @param summary summary from AWS + * @param s3Object s3Object entry * @param blockSize block size to declare. * @param owner owner of the file * @param eTag S3 object eTag or null if unavailable @@ -483,20 +469,20 @@ public static String stringify(AmazonS3Exception e) { * @return a status entry */ public static S3AFileStatus createFileStatus(Path keyPath, - S3ObjectSummary summary, + S3Object s3Object, long blockSize, String owner, String eTag, String versionId, boolean isCSEEnabled) { - long size = summary.getSize(); + long size = s3Object.size(); // check if cse is enabled; strip out constant padding length. if (isCSEEnabled && size >= CSE_PADDING_LENGTH) { size -= CSE_PADDING_LENGTH; } return createFileStatus(keyPath, - objectRepresentsDirectory(summary.getKey()), - size, summary.getLastModified(), blockSize, owner, eTag, versionId); + objectRepresentsDirectory(s3Object.key()), + size, Date.from(s3Object.lastModified()), blockSize, owner, eTag, versionId); } /** @@ -558,114 +544,7 @@ public static long dateToLong(final Date date) { } /** - * The standard AWS provider list for AWS connections. - */ - @SuppressWarnings("deprecation") - public static final List> - STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( - Arrays.asList( - TemporaryAWSCredentialsProvider.class, - SimpleAWSCredentialsProvider.class, - EnvironmentVariableCredentialsProvider.class, - IAMInstanceCredentialsProvider.class)); - - /** - * Create the AWS credentials from the providers, the URI and - * the key {@link Constants#AWS_CREDENTIALS_PROVIDER} in the configuration. - * @param binding Binding URI -may be null - * @param conf filesystem configuration - * @return a credentials provider list - * @throws IOException Problems loading the providers (including reading - * secrets from credential files). - */ - public static AWSCredentialProviderList createAWSCredentialProviderSet( - @Nullable URI binding, - Configuration conf) throws IOException { - // this will reject any user:secret entries in the URI - S3xLoginHelper.rejectSecretsInURIs(binding); - AWSCredentialProviderList credentials = - buildAWSProviderList(binding, - conf, - AWS_CREDENTIALS_PROVIDER, - STANDARD_AWS_PROVIDERS, - new HashSet<>()); - // make sure the logging message strips out any auth details - LOG.debug("For URI {}, using credentials {}", - binding, credentials); - return credentials; - } - - /** - * Load list of AWS credential provider/credential provider factory classes. - * @param conf configuration - * @param key key - * @param defaultValue list of default values - * @return the list of classes, possibly empty - * @throws IOException on a failure to load the list. - */ - public static List> loadAWSProviderClasses(Configuration conf, - String key, - Class... defaultValue) throws IOException { - try { - return Arrays.asList(conf.getClasses(key, defaultValue)); - } catch (RuntimeException e) { - Throwable c = e.getCause() != null ? e.getCause() : e; - throw new IOException("From option " + key + ' ' + c, c); - } - } - - /** - * Load list of AWS credential provider/credential provider factory classes; - * support a forbidden list to prevent loops, mandate full secrets, etc. - * @param binding Binding URI -may be null - * @param conf configuration - * @param key key - * @param forbidden a possibly empty set of forbidden classes. - * @param defaultValues list of default providers. - * @return the list of classes, possibly empty - * @throws IOException on a failure to load the list. - */ - public static AWSCredentialProviderList buildAWSProviderList( - @Nullable final URI binding, - final Configuration conf, - final String key, - final List> defaultValues, - final Set> forbidden) throws IOException { - - // build up the base provider - List> awsClasses = loadAWSProviderClasses(conf, - key, - defaultValues.toArray(new Class[defaultValues.size()])); - // and if the list is empty, switch back to the defaults. - // this is to address the issue that configuration.getClasses() - // doesn't return the default if the config value is just whitespace. - if (awsClasses.isEmpty()) { - awsClasses = defaultValues; - } - // iterate through, checking for blacklists and then instantiating - // each provider - AWSCredentialProviderList providers = new AWSCredentialProviderList(); - for (Class aClass : awsClasses) { - - // List of V1 credential providers that will be migrated with V2 upgrade - if (!Arrays.asList("EnvironmentVariableCredentialsProvider", - "EC2ContainerCredentialsProviderWrapper", "InstanceProfileCredentialsProvider") - .contains(aClass.getSimpleName()) && aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) { - V2Migration.v1ProviderReferenced(aClass.getName()); - } - - if (forbidden.contains(aClass)) { - throw new IOException(E_FORBIDDEN_AWS_PROVIDER - + " in option " + key + ": " + aClass); - } - providers.add(createAWSCredentialProvider(conf, - aClass, binding)); - } - return providers; - } - - /** - * Create an AWS credential provider from its class by using reflection. The + * Creates an instance of a class using reflection. The * class must implement one of the following means of construction, which are * attempted in order: * @@ -674,92 +553,87 @@ public static AWSCredentialProviderList buildAWSProviderList( * org.apache.hadoop.conf.Configuration *

  • a public constructor accepting * org.apache.hadoop.conf.Configuration
    • - *
  • a public static method named getInstance that accepts no + *
  • a public static method named as per methodName, that accepts no * arguments and returns an instance of - * com.amazonaws.auth.AWSCredentialsProvider, or
    • + * specified type, or *
  • a public default constructor.
    • * * + * @param className name of class for which instance is to be created * @param conf configuration - * @param credClass credential class * @param uri URI of the FS - * @return the instantiated class - * @throws IOException on any instantiation failure. + * @param interfaceImplemented interface that this class implements + * @param methodName name of factory method to be invoked + * @param configKey config key under which this class is specified + * @param Instance of class + * @return instance of the specified class + * @throws IOException on any problem */ - private static AWSCredentialsProvider createAWSCredentialProvider( + @SuppressWarnings("unchecked") + public static InstanceT getInstanceFromReflection(String className, Configuration conf, - Class credClass, - @Nullable URI uri) throws IOException { - AWSCredentialsProvider credentials = null; - String className = credClass.getName(); - if (!AWSCredentialsProvider.class.isAssignableFrom(credClass)) { - throw new IOException("Class " + credClass + " " + NOT_AWS_PROVIDER); - } - if (Modifier.isAbstract(credClass.getModifiers())) { - throw new IOException("Class " + credClass + " " + ABSTRACT_PROVIDER); - } - LOG.debug("Credential provider class is {}", className); - + @Nullable URI uri, + Class interfaceImplemented, + String methodName, + String configKey) throws IOException { try { - // new X(uri, conf) - Constructor cons = getConstructor(credClass, URI.class, - Configuration.class); - if (cons != null) { - credentials = (AWSCredentialsProvider)cons.newInstance(uri, conf); - return credentials; + Class instanceClass = S3AUtils.class.getClassLoader().loadClass(className); + if (Modifier.isAbstract(instanceClass.getModifiers())) { + throw isAbstract(uri, className, configKey); } - // new X(conf) - cons = getConstructor(credClass, Configuration.class); - if (cons != null) { - credentials = (AWSCredentialsProvider)cons.newInstance(conf); - return credentials; + if (!interfaceImplemented.isAssignableFrom(instanceClass)) { + throw isNotInstanceOf(uri, className, interfaceImplemented.getName(), configKey); + } + Constructor cons; + if (conf != null) { + // new X(uri, conf) + cons = getConstructor(instanceClass, URI.class, Configuration.class); - // X.getInstance() - Method factory = getFactoryMethod(credClass, AWSCredentialsProvider.class, - "getInstance"); + if (cons != null) { + return (InstanceT) cons.newInstance(uri, conf); + } + // new X(conf) + cons = getConstructor(instanceClass, Configuration.class); + if (cons != null) { + return (InstanceT) cons.newInstance(conf); + } + } + + // X.methodName() + Method factory = getFactoryMethod(instanceClass, interfaceImplemented, methodName); if (factory != null) { - credentials = (AWSCredentialsProvider)factory.invoke(null); - return credentials; + return (InstanceT) factory.invoke(null); } // new X() - cons = getConstructor(credClass); + cons = getConstructor(instanceClass); if (cons != null) { - credentials = (AWSCredentialsProvider)cons.newInstance(); - return credentials; + return (InstanceT) cons.newInstance(); } // no supported constructor or factory method found - throw new IOException(String.format("%s " + CONSTRUCTOR_EXCEPTION - + ". A class specified in %s must provide a public constructor " - + "of a supported signature, or a public factory method named " - + "getInstance that accepts no arguments.", - className, AWS_CREDENTIALS_PROVIDER)); + throw unsupportedConstructor(uri, className, configKey); } catch (InvocationTargetException e) { Throwable targetException = e.getTargetException(); if (targetException == null) { - targetException = e; + targetException = e; } if (targetException instanceof IOException) { throw (IOException) targetException; - } else if (targetException instanceof SdkBaseException) { - throw translateException("Instantiate " + className, "", - (SdkBaseException) targetException); + } else if (targetException instanceof SdkException) { + throw translateException("Instantiate " + className, "", (SdkException) targetException); } else { // supported constructor or factory method found, but the call failed - throw new IOException(className + " " + INSTANTIATION_EXCEPTION - + ": " + targetException, - targetException); + throw instantiationException(uri, className, configKey, targetException); } } catch (ReflectiveOperationException | IllegalArgumentException e) { // supported constructor or factory method found, but the call failed - throw new IOException(className + " " + INSTANTIATION_EXCEPTION - + ": " + e, - e); + throw instantiationException(uri, className, configKey, e); } } + /** * Set a key if the value is non-empty. * @param config config to patch @@ -946,13 +820,13 @@ static String lookupPassword(Configuration conf, String key, String defVal) /** * String information about a summary entry for debug messages. - * @param summary summary object + * @param s3Object s3Object entry * @return string value */ - public static String stringify(S3ObjectSummary summary) { - StringBuilder builder = new StringBuilder(summary.getKey().length() + 100); - builder.append(summary.getKey()).append(' '); - builder.append("size=").append(summary.getSize()); + public static String stringify(S3Object s3Object) { + StringBuilder builder = new StringBuilder(s3Object.key().length() + 100); + builder.append(s3Object.key()).append(' '); + builder.append("size=").append(s3Object.size()); return builder.toString(); } @@ -1225,218 +1099,6 @@ public static void deleteWithWarning(FileSystem fs, } } - /** - * Create a new AWS {@code ClientConfiguration}. - * All clients to AWS services MUST use this for consistent setup - * of connectivity, UA, proxy settings. - * @param conf The Hadoop configuration - * @param bucket Optional bucket to use to look up per-bucket proxy secrets - * @return new AWS client configuration - * @throws IOException problem creating AWS client configuration - * - * @deprecated use {@link #createAwsConf(Configuration, String, String)} - */ - @Deprecated - public static ClientConfiguration createAwsConf(Configuration conf, - String bucket) - throws IOException { - return createAwsConf(conf, bucket, null); - } - - /** - * Create a new AWS {@code ClientConfiguration}. All clients to AWS services - * MUST use this or the equivalents for the specific service for - * consistent setup of connectivity, UA, proxy settings. - * - * @param conf The Hadoop configuration - * @param bucket Optional bucket to use to look up per-bucket proxy secrets - * @param awsServiceIdentifier a string representing the AWS service (S3, - * etc) for which the ClientConfiguration is being created. - * @return new AWS client configuration - * @throws IOException problem creating AWS client configuration - */ - public static ClientConfiguration createAwsConf(Configuration conf, - String bucket, String awsServiceIdentifier) - throws IOException { - final ClientConfiguration awsConf = new ClientConfiguration(); - initConnectionSettings(conf, awsConf); - initProxySupport(conf, bucket, awsConf); - initUserAgent(conf, awsConf); - if (StringUtils.isNotEmpty(awsServiceIdentifier)) { - String configKey = null; - switch (awsServiceIdentifier) { - case AWS_SERVICE_IDENTIFIER_S3: - configKey = SIGNING_ALGORITHM_S3; - break; - case AWS_SERVICE_IDENTIFIER_STS: - configKey = SIGNING_ALGORITHM_STS; - break; - default: - // Nothing to do. The original signer override is already setup - } - if (configKey != null) { - String signerOverride = conf.getTrimmed(configKey, ""); - if (!signerOverride.isEmpty()) { - LOG.debug("Signer override for {}} = {}", awsServiceIdentifier, - signerOverride); - awsConf.setSignerOverride(signerOverride); - } - } - } - return awsConf; - } - - /** - * Initializes all AWS SDK settings related to connection management. - * - * @param conf Hadoop configuration - * @param awsConf AWS SDK configuration - * - * @throws IOException if there was an error initializing the protocol - * settings - */ - public static void initConnectionSettings(Configuration conf, - ClientConfiguration awsConf) throws IOException { - awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS, - DEFAULT_MAXIMUM_CONNECTIONS, 1)); - awsConf.setConnectionTTL(longOption(conf, CONNECTION_TTL, - DEFAULT_CONNECTION_TTL, -1)); - initProtocolSettings(conf, awsConf); - awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES, - DEFAULT_MAX_ERROR_RETRIES, 0)); - awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT, - DEFAULT_ESTABLISH_TIMEOUT, 0)); - awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT, - DEFAULT_SOCKET_TIMEOUT, 0)); - int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER, - DEFAULT_SOCKET_SEND_BUFFER, 2048); - int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER, - DEFAULT_SOCKET_RECV_BUFFER, 2048); - long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT, - DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS); - - if (requestTimeoutMillis > Integer.MAX_VALUE) { - LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead", - requestTimeoutMillis, Integer.MAX_VALUE); - requestTimeoutMillis = Integer.MAX_VALUE; - } - awsConf.setRequestTimeout((int) requestTimeoutMillis); - awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer); - String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, ""); - if (!signerOverride.isEmpty()) { - LOG.debug("Signer override = {}", signerOverride); - awsConf.setSignerOverride(signerOverride); - } - } - - /** - * Initializes the connection protocol settings when connecting to S3 (e.g. - * either HTTP or HTTPS). If secure connections are enabled, this method - * will load the configured SSL providers. - * - * @param conf Hadoop configuration - * @param awsConf AWS SDK configuration - * - * @throws IOException if there is an error initializing the configured - * {@link javax.net.ssl.SSLSocketFactory} - */ - private static void initProtocolSettings(Configuration conf, - ClientConfiguration awsConf) throws IOException { - boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, - DEFAULT_SECURE_CONNECTIONS); - awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); - if (secureConnections) { - NetworkBinding.bindSSLChannelMode(conf, awsConf); - } - } - - /** - * Initializes AWS SDK proxy support in the AWS client configuration - * if the S3A settings enable it. - *
    - * Note: LimitedPrivate to provide proxy support in ranger repository. - * - * @param conf Hadoop configuration - * @param bucket Optional bucket to use to look up per-bucket proxy secrets - * @param awsConf AWS SDK configuration to update - * @throws IllegalArgumentException if misconfigured - * @throws IOException problem getting username/secret from password source. - */ - @InterfaceAudience.LimitedPrivate("Ranger") - public static void initProxySupport(Configuration conf, - String bucket, - ClientConfiguration awsConf) throws IllegalArgumentException, - IOException { - String proxyHost = conf.getTrimmed(PROXY_HOST, ""); - int proxyPort = conf.getInt(PROXY_PORT, -1); - if (!proxyHost.isEmpty()) { - awsConf.setProxyHost(proxyHost); - if (proxyPort >= 0) { - awsConf.setProxyPort(proxyPort); - } else { - if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) { - LOG.warn("Proxy host set without port. Using HTTPS default 443"); - awsConf.setProxyPort(443); - } else { - LOG.warn("Proxy host set without port. Using HTTP default 80"); - awsConf.setProxyPort(80); - } - } - final String proxyUsername = lookupPassword(bucket, conf, PROXY_USERNAME, - null, null); - final String proxyPassword = lookupPassword(bucket, conf, PROXY_PASSWORD, - null, null); - if ((proxyUsername == null) != (proxyPassword == null)) { - String msg = "Proxy error: " + PROXY_USERNAME + " or " + - PROXY_PASSWORD + " set without the other."; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false); - awsConf.setProxyUsername(proxyUsername); - awsConf.setProxyPassword(proxyPassword); - awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); - awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); - awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP); - if (LOG.isDebugEnabled()) { - LOG.debug("Using proxy server {}://{}:{} as user {} with password {} " - + "on domain {} as workstation {}", - awsConf.getProxyProtocol(), - awsConf.getProxyHost(), - awsConf.getProxyPort(), - String.valueOf(awsConf.getProxyUsername()), - awsConf.getProxyPassword(), awsConf.getProxyDomain(), - awsConf.getProxyWorkstation()); - } - } else if (proxyPort >= 0) { - String msg = - "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - } - - /** - * Initializes the User-Agent header to send in HTTP requests to AWS - * services. We always include the Hadoop version number. The user also - * may set an optional custom prefix to put in front of the Hadoop version - * number. The AWS SDK internally appends its own information, which seems - * to include the AWS SDK version, OS and JVM version. - * - * @param conf Hadoop configuration - * @param awsConf AWS SDK configuration to update - */ - private static void initUserAgent(Configuration conf, - ClientConfiguration awsConf) { - String userAgent = "Hadoop " + VersionInfo.getVersion(); - String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, ""); - if (!userAgentPrefix.isEmpty()) { - userAgent = userAgentPrefix + ", " + userAgent; - } - LOG.debug("Using User-Agent: {}", userAgent); - awsConf.setUserAgentPrefix(userAgent); - } - /** * Convert the data of an iterator of {@link S3AFileStatus} to * an array. @@ -1930,4 +1592,15 @@ public String toString() { } }; + /** + * Format a byte range for a request header. + * See https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2 + * + * @param rangeStart the start byte offset + * @param rangeEnd the end byte offset (inclusive) + * @return a formatted byte range + */ + public static String formatRange(long rangeStart, long rangeEnd) { + return String.format("bytes=%d-%d", rangeStart, rangeEnd); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 9010f34dc259c..d4504cd08d74c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -24,11 +24,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.monitoring.MonitoringListener; -import com.amazonaws.services.s3.AmazonS3; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.transfer.s3.S3TransferManager; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -37,7 +40,7 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT; /** - * Factory for creation of {@link AmazonS3} client instances. + * Factory for creation of {@link S3Client} client instances. * Important: HBase's HBoss module implements this interface in its * tests. * Take care when updating this interface to ensure that a client @@ -49,20 +52,44 @@ */ @InterfaceAudience.LimitedPrivate("HBoss") @InterfaceStability.Evolving -@Deprecated public interface S3ClientFactory { /** - * Creates a new {@link AmazonS3} client. + * Creates a new {@link S3Client}. + * The client returned supports synchronous operations. For + * asynchronous operations, use + * {@link #createS3AsyncClient(URI, S3ClientCreationParameters)}. * * @param uri S3A file system URI * @param parameters parameter object * @return S3 client - * @throws IOException IO problem + * @throws IOException on any IO problem */ - AmazonS3 createS3Client(URI uri, + S3Client createS3Client(URI uri, S3ClientCreationParameters parameters) throws IOException; + /** + * Creates a new {@link S3AsyncClient}. + * The client returned supports asynchronous operations. For + * synchronous operations, use + * {@link #createS3Client(URI, S3ClientCreationParameters)}. + * + * @param uri S3A file system URI + * @param parameters parameter object + * @return Async S3 client + * @throws IOException on any IO problem + */ + S3AsyncClient createS3AsyncClient(URI uri, + S3ClientCreationParameters parameters) throws IOException; + + /** + * Creates a new {@link S3TransferManager}. + * + * @param s3AsyncClient the async client to be used by the TM. + * @return S3 transfer manager + */ + S3TransferManager createS3TransferManager(S3AsyncClient s3AsyncClient); + /** * Settings for the S3 Client. * Implemented as a class to pass in so that adding @@ -74,7 +101,7 @@ final class S3ClientCreationParameters { /** * Credentials. */ - private AWSCredentialsProvider credentialSet; + private AwsCredentialsProvider credentialSet; /** * Endpoint. @@ -86,11 +113,6 @@ final class S3ClientCreationParameters { */ private final Map headers = new HashMap<>(); - /** - * Monitoring listener. - */ - private MonitoringListener monitoringListener; - /** * RequestMetricCollector metrics...if not-null will be wrapped * with an {@code AwsStatisticsCollector} and passed to @@ -109,9 +131,9 @@ final class S3ClientCreationParameters { private boolean requesterPays; /** - * Request handlers; used for auditing, X-Ray etc. - */ - private List requestHandlers; + * Execution interceptors; used for auditing, X-Ray etc. + * */ + private List executionInterceptors; /** * Suffix to UA. @@ -125,37 +147,43 @@ final class S3ClientCreationParameters { private URI pathUri; /** - * List of request handlers to include in the chain - * of request execution in the SDK. - * @return the handler list + * Minimum part size for transfer parts. */ - public List getRequestHandlers() { - return requestHandlers; - } + private long minimumPartSize; /** - * List of request handlers. - * @param handlers handler list. - * @return this object + * Threshold for multipart operations. */ - public S3ClientCreationParameters withRequestHandlers( - @Nullable final List handlers) { - requestHandlers = handlers; - return this; - } + private long multiPartThreshold; - public MonitoringListener getMonitoringListener() { - return monitoringListener; + /** + * Executor that the transfer manager will use to execute background tasks. + */ + private Executor transferManagerExecutor; + + /** + * Region of the S3 bucket. + */ + private Region region; + + + /** + * List of execution interceptors to include in the chain + * of interceptors in the SDK. + * @return the interceptors list + */ + public List getExecutionInterceptors() { + return executionInterceptors; } /** - * listener for AWS monitoring events. - * @param listener listener + * List of execution interceptors. + * @param interceptors interceptors list. * @return this object */ - public S3ClientCreationParameters withMonitoringListener( - @Nullable final MonitoringListener listener) { - monitoringListener = listener; + public S3ClientCreationParameters withExecutionInterceptors( + @Nullable final List interceptors) { + executionInterceptors = interceptors; return this; } @@ -191,7 +219,7 @@ public boolean isRequesterPays() { return requesterPays; } - public AWSCredentialsProvider getCredentialSet() { + public AwsCredentialsProvider getCredentialSet() { return credentialSet; } @@ -202,7 +230,7 @@ public AWSCredentialsProvider getCredentialSet() { */ public S3ClientCreationParameters withCredentialSet( - final AWSCredentialsProvider value) { + final AwsCredentialsProvider value) { credentialSet = value; return this; } @@ -294,5 +322,82 @@ public S3ClientCreationParameters withPathUri( pathUri = value; return this; } + + /** + * Get the minimum part size for transfer parts. + * @return part size + */ + public long getMinimumPartSize() { + return minimumPartSize; + } + + /** + * Set the minimum part size for transfer parts. + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withMinimumPartSize( + final long value) { + minimumPartSize = value; + return this; + } + + /** + * Get the threshold for multipart operations. + * @return multipart threshold + */ + public long getMultiPartThreshold() { + return multiPartThreshold; + } + + /** + * Set the threshold for multipart operations. + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withMultipartThreshold( + final long value) { + multiPartThreshold = value; + return this; + } + + /** + * Get the executor that the transfer manager will use to execute background tasks. + * @return part size + */ + public Executor getTransferManagerExecutor() { + return transferManagerExecutor; + } + + /** + * Set the executor that the transfer manager will use to execute background tasks. + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withTransferManagerExecutor( + final Executor value) { + transferManagerExecutor = value; + return this; + } + + /** + * Set region. + * + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withRegion( + final Region value) { + region = value; + return this; + } + + /** + * Get the region. + * @return invoker + */ + public Region getRegion() { + return region; + } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java index d51211516f251..c729f3de15f08 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java @@ -18,8 +18,8 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; /** * API version-independent container for S3 List requests. @@ -78,14 +78,14 @@ public ListObjectsV2Request getV2() { public String toString() { if (isV1()) { return String.format(DESCRIPTION, - v1Request.getBucketName(), v1Request.getPrefix(), - v1Request.getDelimiter(), v1Request.getMaxKeys(), - v1Request.isRequesterPays()); + v1Request.bucket(), v1Request.prefix(), + v1Request.delimiter(), v1Request.maxKeys(), + v1Request.requestPayerAsString()); } else { return String.format(DESCRIPTION, - v2Request.getBucketName(), v2Request.getPrefix(), - v2Request.getDelimiter(), v2Request.getMaxKeys(), - v2Request.isRequesterPays()); + v2Request.bucket(), v2Request.prefix(), + v2Request.delimiter(), v2Request.maxKeys(), + v2Request.requestPayerAsString()); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java index 69c42bfe1471a..c77311211abcb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java @@ -22,19 +22,21 @@ import java.util.List; import java.util.stream.Collectors; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.services.s3.model.CommonPrefix; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; + import org.slf4j.Logger; /** * API version-independent container for S3 List responses. */ public class S3ListResult { - private ObjectListing v1Result; - private ListObjectsV2Result v2Result; + private ListObjectsResponse v1Result; + private ListObjectsV2Response v2Result; - protected S3ListResult(ObjectListing v1, ListObjectsV2Result v2) { + protected S3ListResult(ListObjectsResponse v1, ListObjectsV2Response v2) { v1Result = v1; v2Result = v2; } @@ -44,7 +46,7 @@ protected S3ListResult(ObjectListing v1, ListObjectsV2Result v2) { * @param result v1 result * @return new list result container */ - public static S3ListResult v1(ObjectListing result) { + public static S3ListResult v1(ListObjectsResponse result) { return new S3ListResult(result, null); } @@ -53,7 +55,7 @@ public static S3ListResult v1(ObjectListing result) { * @param result v2 result * @return new list result container */ - public static S3ListResult v2(ListObjectsV2Result result) { + public static S3ListResult v2(ListObjectsV2Response result) { return new S3ListResult(null, result); } @@ -65,19 +67,19 @@ public boolean isV1() { return v1Result != null; } - public ObjectListing getV1() { + public ListObjectsResponse getV1() { return v1Result; } - public ListObjectsV2Result getV2() { + public ListObjectsV2Response getV2() { return v2Result; } - public List getObjectSummaries() { + public List getS3Objects() { if (isV1()) { - return v1Result.getObjectSummaries(); + return v1Result.contents(); } else { - return v2Result.getObjectSummaries(); + return v2Result.contents(); } } @@ -89,21 +91,21 @@ public boolean isTruncated() { } } - public List getCommonPrefixes() { + public List getCommonPrefixes() { if (isV1()) { - return v1Result.getCommonPrefixes(); + return v1Result.commonPrefixes(); } else { - return v2Result.getCommonPrefixes(); + return v2Result.commonPrefixes(); } } /** - * Get the list of keys in the object summary. + * Get the list of keys in the list result. * @return a possibly empty list */ - private List objectSummaryKeys() { - return getObjectSummaries().stream() - .map(S3ObjectSummary::getKey) + private List objectKeys() { + return getS3Objects().stream() + .map(S3Object::key) .collect(Collectors.toList()); } @@ -112,9 +114,8 @@ private List objectSummaryKeys() { * @return true if the result is non-empty */ public boolean hasPrefixesOrObjects() { - return !(getCommonPrefixes()).isEmpty() - || !getObjectSummaries().isEmpty(); + || !getS3Objects().isEmpty(); } /** @@ -128,7 +129,7 @@ public boolean representsEmptyDirectory( // no children. // So the listing must contain the marker entry only as an object, // and prefixes is null - List keys = objectSummaryKeys(); + List keys = objectKeys(); return keys.size() == 1 && keys.contains(dirKey) && getCommonPrefixes().isEmpty(); } @@ -138,15 +139,15 @@ public boolean representsEmptyDirectory( * @param log log to use */ public void logAtDebug(Logger log) { - Collection prefixes = getCommonPrefixes(); - Collection summaries = getObjectSummaries(); + Collection prefixes = getCommonPrefixes(); + Collection s3Objects = getS3Objects(); log.debug("Prefix count = {}; object count={}", - prefixes.size(), summaries.size()); - for (S3ObjectSummary summary : summaries) { - log.debug("Summary: {} {}", summary.getKey(), summary.getSize()); + prefixes.size(), s3Objects.size()); + for (S3Object s3Object : s3Objects) { + log.debug("Summary: {} {}", s3Object.key(), s3Object.size()); } - for (String prefix : prefixes) { - log.debug("Prefix: {}", prefix); + for (CommonPrefix prefix : prefixes) { + log.debug("Prefix: {}", prefix.prefix()); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java index 0a0454854b2ac..4fc5b8658b605 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.transfer.model.CopyResult; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; @@ -74,31 +72,6 @@ public S3ObjectAttributes( this.len = len; } - /** - * Construct from the result of a copy and those parameters - * which aren't included in an AWS SDK response. - * @param path path - * @param copyResult copy result. - * @param serverSideEncryptionAlgorithm current encryption algorithm - * @param serverSideEncryptionKey any server side encryption key? - * @param len object length - */ - public S3ObjectAttributes( - final Path path, - final CopyResult copyResult, - final S3AEncryptionMethods serverSideEncryptionAlgorithm, - final String serverSideEncryptionKey, - final long len) { - this.bucket = copyResult.getDestinationBucketName(); - this.key = copyResult.getDestinationKey(); - this.path = path; - this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; - this.serverSideEncryptionKey = serverSideEncryptionKey; - this.eTag = copyResult.getETag(); - this.versionId = copyResult.getVersionId(); - this.len = len; - } - public String getBucket() { return bucket; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java index 6579a2bc3e7d2..b7de937f5137a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java @@ -39,6 +39,5 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving -@SuppressWarnings("deprecation") public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java index 50a2dd5fb3fc2..f7eaf825b9c94 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java @@ -18,9 +18,10 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + import org.apache.hadoop.classification.VisibleForTesting; import org.apache.commons.lang3.StringUtils; @@ -42,13 +43,10 @@ * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable -@Deprecated -public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider { +public class SimpleAWSCredentialsProvider implements AwsCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"; @@ -63,7 +61,7 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider { */ public SimpleAWSCredentialsProvider(final URI uri, final Configuration conf) throws IOException { - this(getAWSAccessKeys(uri, conf)); + this(getAWSAccessKeys(uri, conf)); } /** @@ -75,25 +73,25 @@ public SimpleAWSCredentialsProvider(final URI uri, final Configuration conf) @VisibleForTesting SimpleAWSCredentialsProvider(final S3xLoginHelper.Login login) throws IOException { - this.accessKey = login.getUser(); - this.secretKey = login.getPassword(); + this.accessKey = login.getUser(); + this.secretKey = login.getPassword(); } @Override - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) { - return new BasicAWSCredentials(accessKey, secretKey); + return AwsBasicCredentials.create(accessKey, secretKey); } throw new NoAwsCredentialsException("SimpleAWSCredentialsProvider", "No AWS credentials in the Hadoop configuration"); } - @Override - public void refresh() {} - @Override public String toString() { - return getClass().getSimpleName(); + return "SimpleAWSCredentialsProvider{" + + "accessKey.empty=" + accessKey.isEmpty() + + ", secretKey.empty'" + secretKey.isEmpty() + + '}'; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index 651769ff283bd..ae761fe270f46 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -538,6 +538,11 @@ public enum Statistic { StoreStatisticNames.STORE_IO_THROTTLE_RATE, "Rate of S3 request throttling", TYPE_QUANTILE), + STORE_REGION_PROBE( + StoreStatisticNames.STORE_REGION_PROBE, + "Store Region Probe", + TYPE_DURATION + ), /* * Delegation Token Operations. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index db3d0bb13297c..5587b11a36f88 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -21,7 +21,7 @@ import javax.annotation.Nullable; import java.io.IOException; -import com.amazonaws.auth.AWSCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; import java.net.URI; @@ -44,12 +44,9 @@ * This credential provider must not fail in creation because that will * break a chain of credential providers. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable -@Deprecated public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider { public static final String NAME @@ -92,7 +89,7 @@ public TemporaryAWSCredentialsProvider( * @throws NoAwsCredentialsException the credentials are actually empty. */ @Override - protected AWSCredentials createCredentials(Configuration config) + protected AwsCredentials createCredentials(Configuration config) throws IOException { MarshalledCredentials creds = MarshalledCredentialBinding.fromFileSystem( getUri(), config); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java index 0462ccfd7cbbd..d7123787735a1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Tristate.java @@ -18,15 +18,58 @@ package org.apache.hadoop.fs.s3a; +import java.util.Optional; + +import static java.util.Optional.empty; +import static java.util.Optional.of; + /** * Simple enum to express {true, false, don't know}. */ public enum Tristate { + // Do not add additional values here. Logic will assume there are exactly // three possibilities. - TRUE, FALSE, UNKNOWN; + TRUE(of(Boolean.TRUE)), + FALSE(of(Boolean.FALSE)), + UNKNOWN(empty()); + + /** + * Mapping to an optional boolean. + */ + @SuppressWarnings("NonSerializableFieldInSerializableClass") + private final Optional mapping; + + Tristate(final Optional t) { + mapping = t; + } + + /** + * Get the boolean mapping, if present. + * @return the boolean value, if present. + */ + public Optional getMapping() { + return mapping; + } + + /** + * Does this value map to a boolean. + * @return true if the state is one of true or false. + */ + public boolean isBoolean() { + return mapping.isPresent(); + } public static Tristate fromBool(boolean v) { return v ? TRUE : FALSE; } + + /** + * Build a tristate from a boolean. + * @param b source optional + * @return a tristate derived from the argument. + */ + public static Tristate from(Optional b) { + return b.map(Tristate::fromBool).orElse(UNKNOWN); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java index 238cd97ed8f48..85ed69e14bc79 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UploadInfo.java @@ -18,22 +18,22 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.transfer.Upload; +import software.amazon.awssdk.transfer.s3.model.FileUpload; /** * Simple struct that contains information about a S3 upload. */ public class UploadInfo { - private final Upload upload; + private final FileUpload fileUpload; private final long length; - public UploadInfo(Upload upload, long length) { - this.upload = upload; + public UploadInfo(FileUpload upload, long length) { + this.fileUpload = upload; this.length = length; } - public Upload getUpload() { - return upload; + public FileUpload getFileUpload() { + return fileUpload; } public long getLength() { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index 8e15a10944c38..f2ece63a854fa 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -19,29 +19,28 @@ package org.apache.hadoop.fs.s3a; import javax.annotation.Nullable; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.SelectObjectContentResult; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -50,6 +49,8 @@ import org.apache.hadoop.fs.s3a.api.RequestFactory; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.impl.StoreContext; +import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher; +import org.apache.hadoop.fs.s3a.select.SelectObjectContentHelper; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.s3a.select.SelectBinding; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; @@ -239,49 +240,25 @@ private void deactivateAuditSpan() { /** * Create a {@link PutObjectRequest} request against the specific key. * @param destKey destination key - * @param inputStream source data. * @param length size, if known. Use -1 for not known * @param options options for the request + * @param isFile is data to be uploaded a file * @return the request */ @Retries.OnceRaw public PutObjectRequest createPutObjectRequest(String destKey, - InputStream inputStream, long length, - final PutObjectOptions options) { - activateAuditSpan(); - ObjectMetadata objectMetadata = newObjectMetadata(length); - return getRequestFactory().newPutObjectRequest( - destKey, - objectMetadata, - options, - inputStream); - } + final PutObjectOptions options, + boolean isFile) { - /** - * Create a {@link PutObjectRequest} request to upload a file. - * @param dest key to PUT to. - * @param sourceFile source file - * @param options options for the request - * @return the request - */ - @Retries.OnceRaw - public PutObjectRequest createPutObjectRequest( - String dest, - File sourceFile, - final PutObjectOptions options) { activateAuditSpan(); - final ObjectMetadata objectMetadata = - newObjectMetadata((int) sourceFile.length()); - - PutObjectRequest putObjectRequest = getRequestFactory(). - newPutObjectRequest(dest, - objectMetadata, - options, - sourceFile); - return putObjectRequest; + + return getRequestFactory() + .newPutObjectRequestBuilder(destKey, options, length, false) + .build(); } + /** * Callback on a successful write. * @param length length of the write @@ -297,17 +274,6 @@ public void writeFailed(Exception ex) { LOG.debug("Write to {} failed", this, ex); } - /** - * Create a new object metadata instance. - * Any standard metadata headers are added here, for example: - * encryption. - * @param length size, if known. Use -1 for not known - * @return a new metadata instance - */ - public ObjectMetadata newObjectMetadata(long length) { - return getRequestFactory().newObjectMetadata(length); - } - /** * {@inheritDoc} */ @@ -320,11 +286,11 @@ public String initiateMultiPartUpload( try (AuditSpan span = activateAuditSpan()) { return retry("initiate MultiPartUpload", destKey, true, () -> { - final InitiateMultipartUploadRequest initiateMPURequest = - getRequestFactory().newMultipartUploadRequest( + final CreateMultipartUploadRequest.Builder initiateMPURequestBuilder = + getRequestFactory().newMultipartUploadRequestBuilder( destKey, options); - return owner.initiateMultipartUpload(initiateMPURequest) - .getUploadId(); + return owner.initiateMultipartUpload(initiateMPURequestBuilder.build()) + .uploadId(); }); } } @@ -345,10 +311,10 @@ public String initiateMultiPartUpload( * @throws IOException on problems. */ @Retries.RetryTranslated - private CompleteMultipartUploadResult finalizeMultipartUpload( + private CompleteMultipartUploadResponse finalizeMultipartUpload( String destKey, String uploadId, - List partETags, + List partETags, long length, PutObjectOptions putOptions, Retried retrying) throws IOException { @@ -357,18 +323,18 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( "No upload parts in multipart upload"); } try (AuditSpan span = activateAuditSpan()) { - CompleteMultipartUploadResult uploadResult; + CompleteMultipartUploadResponse uploadResult; uploadResult = invoker.retry("Completing multipart upload", destKey, true, retrying, () -> { - final CompleteMultipartUploadRequest request = - getRequestFactory().newCompleteMultipartUploadRequest( + final CompleteMultipartUploadRequest.Builder requestBuilder = + getRequestFactory().newCompleteMultipartUploadRequestBuilder( destKey, uploadId, partETags); - return writeOperationHelperCallbacks.completeMultipartUpload(request); + return writeOperationHelperCallbacks.completeMultipartUpload(requestBuilder.build()); }); - owner.finishedWrite(destKey, length, uploadResult.getETag(), - uploadResult.getVersionId(), + owner.finishedWrite(destKey, length, uploadResult.eTag(), + uploadResult.versionId(), putOptions); return uploadResult; } @@ -391,10 +357,10 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( * the retry count was exceeded */ @Retries.RetryTranslated - public CompleteMultipartUploadResult completeMPUwithRetries( + public CompleteMultipartUploadResponse completeMPUwithRetries( String destKey, String uploadId, - List partETags, + List partETags, long length, AtomicInteger errorCount, PutObjectOptions putOptions) @@ -452,7 +418,7 @@ public void abortMultipartUpload(String destKey, String uploadId, @Retries.RetryTranslated public void abortMultipartUpload(MultipartUpload upload) throws IOException { - invoker.retry("Aborting multipart commit", upload.getKey(), true, + invoker.retry("Aborting multipart commit", upload.key(), true, withinAuditSpan(getAuditSpan(), () -> owner.abortMultipartUpload(upload))); } @@ -477,7 +443,7 @@ public int abortMultipartUploadsUnderPath(String prefix) abortMultipartUpload(upload); count++; } catch (FileNotFoundException e) { - LOG.debug("Already aborted: {}", upload.getKey(), e); + LOG.debug("Already aborted: {}", upload.key(), e); } } return count; @@ -506,45 +472,31 @@ public void abortMultipartCommit(String destKey, String uploadId) } /** - * Create and initialize a part request of a multipart upload. - * Exactly one of: {@code uploadStream} or {@code sourceFile} - * must be specified. - * A subset of the file may be posted, by providing the starting point - * in {@code offset} and a length of block in {@code size} equal to - * or less than the remaining bytes. + * Create and initialize a part request builder of a multipart upload. * The part number must be less than 10000. * Retry policy is once-translated; to much effort * @param destKey destination key of ongoing operation * @param uploadId ID of ongoing upload * @param partNumber current part number of the upload * @param size amount of data - * @param uploadStream source of data to upload - * @param sourceFile optional source file. - * @param offset offset in file to start reading. - * @return the request. + * @return the request builder. * @throws IllegalArgumentException if the parameters are invalid. * @throws PathIOException if the part number is out of range. */ @Override @Retries.OnceTranslated - public UploadPartRequest newUploadPartRequest( + public UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, - long size, - InputStream uploadStream, - File sourceFile, - Long offset) throws IOException { + long size) throws IOException { return once("upload part request", destKey, withinAuditSpan(getAuditSpan(), () -> - getRequestFactory().newUploadPartRequest( + getRequestFactory().newUploadPartRequestBuilder( destKey, uploadId, partNumber, - size, - uploadStream, - sourceFile, - offset))); + size))); } /** @@ -566,18 +518,20 @@ public String toString() { * @param putObjectRequest the request * @param putOptions put object options * @param durationTrackerFactory factory for duration tracking + * @param uploadData data to be uploaded + * @param isFile is data to be uploaded a file + * * @return the upload initiated * @throws IOException on problems */ @Retries.RetryTranslated - public PutObjectResult putObject(PutObjectRequest putObjectRequest, - PutObjectOptions putOptions, + public PutObjectResponse putObject(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile, DurationTrackerFactory durationTrackerFactory) throws IOException { - return retry("Writing Object", - putObjectRequest.getKey(), true, - withinAuditSpan(getAuditSpan(), () -> - owner.putObjectDirect(putObjectRequest, putOptions, durationTrackerFactory))); + return retry("Writing Object", putObjectRequest.key(), true, withinAuditSpan(getAuditSpan(), + () -> owner.putObjectDirect(putObjectRequest, putOptions, uploadData, isFile, + durationTrackerFactory))); } /** @@ -613,10 +567,10 @@ public void revertCommit(String destKey) throws IOException { * the retry count was exceeded */ @Retries.RetryTranslated - public CompleteMultipartUploadResult commitUpload( + public CompleteMultipartUploadResponse commitUpload( String destKey, String uploadId, - List partETags, + List partETags, long length) throws IOException { checkNotNull(uploadId); @@ -635,19 +589,21 @@ public CompleteMultipartUploadResult commitUpload( * Upload part of a multi-partition file. * @param request request * @param durationTrackerFactory duration tracker factory for operation + * @param request the upload part request. + * @param body the request body. * @return the result of the operation. * @throws IOException on problems */ @Retries.RetryTranslated - public UploadPartResult uploadPart(UploadPartRequest request, + public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body, final DurationTrackerFactory durationTrackerFactory) throws IOException { - return retry("upload part #" + request.getPartNumber() - + " upload ID " + request.getUploadId(), - request.getKey(), + return retry("upload part #" + request.partNumber() + + " upload ID " + request.uploadId(), + request.key(), true, withinAuditSpan(getAuditSpan(), - () -> owner.uploadPart(request, durationTrackerFactory))); + () -> owner.uploadPart(request, body, durationTrackerFactory))); } /** @@ -659,15 +615,9 @@ public Configuration getConf() { return conf; } - /** - * Create a S3 Select request for the destination path. - * This does not build the query. - * @param path pre-qualified path for query - * @return the request - */ - public SelectObjectContentRequest newSelectRequest(Path path) { + public SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path) { try (AuditSpan span = getAuditSpan()) { - return getRequestFactory().newSelectRequest( + return getRequestFactory().newSelectRequestBuilder( storeContext.pathToKey(path)); } } @@ -676,26 +626,27 @@ public SelectObjectContentRequest newSelectRequest(Path path) { * Execute an S3 Select operation. * On a failure, the request is only logged at debug to avoid the * select exception being printed. - * @param source source for selection + * + * @param source source for selection * @param request Select request to issue. - * @param action the action for use in exception creation + * @param action the action for use in exception creation * @return response * @throws IOException failure */ @Retries.RetryTranslated - public SelectObjectContentResult select( + public SelectEventStreamPublisher select( final Path source, final SelectObjectContentRequest request, final String action) throws IOException { // no setting of span here as the select binding is (statically) created // without any span. - String bucketName = request.getBucketName(); + String bucketName = request.bucket(); Preconditions.checkArgument(bucket.equals(bucketName), "wrong bucket: %s", bucketName); if (LOG.isDebugEnabled()) { LOG.debug("Initiating select call {} {}", - source, request.getExpression()); + source, request.expression()); LOG.debug(SelectBinding.toString(request)); } return invoker.retry( @@ -706,8 +657,9 @@ public SelectObjectContentResult select( try (DurationInfo ignored = new DurationInfo(LOG, "S3 Select operation")) { try { - return writeOperationHelperCallbacks.selectObjectContent(request); - } catch (AmazonS3Exception e) { + return SelectObjectContentHelper.select( + writeOperationHelperCallbacks, source, request, action); + } catch (Throwable e) { LOG.error("Failure of S3 Select request against {}", source); LOG.debug("S3 Select request against {}:\n{}", @@ -756,16 +708,18 @@ public interface WriteOperationHelperCallbacks { /** * Initiates a select request. * @param request selectObjectContent request + * @param t selectObjectContent request handler * @return selectObjectContentResult */ - SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request); + CompletableFuture selectObjectContent(SelectObjectContentRequest request, + SelectObjectContentResponseHandler t); /** * Initiates a complete multi-part upload request. * @param request Complete multi-part upload request * @return completeMultipartUploadResult */ - CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request); + CompleteMultipartUploadResponse completeMultipartUpload(CompleteMultipartUploadRequest request); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java index 1c3d368857583..0fda4921a30da 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java @@ -20,29 +20,27 @@ import javax.annotation.Nullable; import java.io.Closeable; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.SelectObjectContentResult; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; +import org.apache.hadoop.fs.s3a.select.SelectEventStreamPublisher; import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.util.functional.CallableRaisingIOE; @@ -77,27 +75,15 @@ T retry(String action, /** * Create a {@link PutObjectRequest} request against the specific key. * @param destKey destination key - * @param inputStream source data. * @param length size, if known. Use -1 for not known * @param options options for the request + * @param isFile is data to be uploaded a file * @return the request */ PutObjectRequest createPutObjectRequest(String destKey, - InputStream inputStream, long length, - @Nullable PutObjectOptions options); - - /** - * Create a {@link PutObjectRequest} request to upload a file. - * @param dest key to PUT to. - * @param sourceFile source file - * @param options options for the request - * @return the request - */ - PutObjectRequest createPutObjectRequest( - String dest, - File sourceFile, - @Nullable PutObjectOptions options); + @Nullable PutObjectOptions options, + boolean isFile); /** * Callback on a successful write. @@ -111,15 +97,6 @@ PutObjectRequest createPutObjectRequest( */ void writeFailed(Exception ex); - /** - * Create a new object metadata instance. - * Any standard metadata headers are added here, for example: - * encryption. - * @param length size, if known. Use -1 for not known - * @return a new metadata instance - */ - ObjectMetadata newObjectMetadata(long length); - /** * Start the multipart upload process. * Retry policy: retrying, translated. @@ -148,10 +125,10 @@ PutObjectRequest createPutObjectRequest( * the retry count was exceeded */ @Retries.RetryTranslated - CompleteMultipartUploadResult completeMPUwithRetries( + CompleteMultipartUploadResponse completeMPUwithRetries( String destKey, String uploadId, - List partETags, + List partETags, long length, AtomicInteger errorCount, PutObjectOptions putOptions) @@ -213,31 +190,20 @@ void abortMultipartCommit(String destKey, String uploadId) throws IOException; /** - * Create and initialize a part request of a multipart upload. - * Exactly one of: {@code uploadStream} or {@code sourceFile} - * must be specified. - * A subset of the file may be posted, by providing the starting point - * in {@code offset} and a length of block in {@code size} equal to - * or less than the remaining bytes. + * Create and initialize a part request builder of a multipart upload. * @param destKey destination key of ongoing operation * @param uploadId ID of ongoing upload * @param partNumber current part number of the upload * @param size amount of data - * @param uploadStream source of data to upload - * @param sourceFile optional source file. - * @param offset offset in file to start reading. - * @return the request. + * @return the request builder. * @throws IllegalArgumentException if the parameters are invalid * @throws PathIOException if the part number is out of range. */ - UploadPartRequest newUploadPartRequest( + UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, - long size, - InputStream uploadStream, - File sourceFile, - Long offset) throws IOException; + long size) throws IOException; /** * PUT an object directly (i.e. not via the transfer manager). @@ -246,12 +212,14 @@ UploadPartRequest newUploadPartRequest( * @param putObjectRequest the request * @param putOptions put object options * @param durationTrackerFactory factory for duration tracking + * @param uploadData data to be uploaded + * @param isFile is data to be uploaded a file * @return the upload initiated * @throws IOException on problems */ @Retries.RetryTranslated - PutObjectResult putObject(PutObjectRequest putObjectRequest, - PutObjectOptions putOptions, + PutObjectResponse putObject(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions, S3ADataBlocks.BlockUploadData uploadData, boolean isFile, DurationTrackerFactory durationTrackerFactory) throws IOException; @@ -279,22 +247,23 @@ PutObjectResult putObject(PutObjectRequest putObjectRequest, * the retry count was exceeded */ @Retries.RetryTranslated - CompleteMultipartUploadResult commitUpload( + CompleteMultipartUploadResponse commitUpload( String destKey, String uploadId, - List partETags, + List partETags, long length) throws IOException; /** * Upload part of a multi-partition file. - * @param request request + * @param request the upload part request. + * @param body the request body. * @param durationTrackerFactory factory for duration tracking * @return the result of the operation. * @throws IOException on problems */ @Retries.RetryTranslated - UploadPartResult uploadPart(UploadPartRequest request, + UploadPartResponse uploadPart(UploadPartRequest request, RequestBody body, DurationTrackerFactory durationTrackerFactory) throws IOException; @@ -306,25 +275,26 @@ UploadPartResult uploadPart(UploadPartRequest request, Configuration getConf(); /** - * Create a S3 Select request for the destination path. + * Create a S3 Select request builder for the destination path. * This does not build the query. * @param path pre-qualified path for query - * @return the request + * @return the request builder */ - SelectObjectContentRequest newSelectRequest(Path path); + SelectObjectContentRequest.Builder newSelectRequestBuilder(Path path); /** * Execute an S3 Select operation. * On a failure, the request is only logged at debug to avoid the * select exception being printed. - * @param source source for selection + * + * @param source source for selection * @param request Select request to issue. - * @param action the action for use in exception creation + * @param action the action for use in exception creation * @return response * @throws IOException failure */ @Retries.RetryTranslated - SelectObjectContentResult select( + SelectEventStreamPublisher select( Path source, SelectObjectContentRequest request, String action) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java new file mode 100644 index 0000000000000..177952cb9d22b --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/AwsV1BindingSupport.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.adapter; + +import java.io.IOException; +import java.net.URI; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; + +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable; + +/** + * Binding support; the sole way which the rest of the code should instantiate v1 SDK libraries. + * Uses this class's Classloader for its analysis/loading. + */ +@SuppressWarnings("StaticNonFinalField") +public final class AwsV1BindingSupport { + + private static final Logger LOG = LoggerFactory.getLogger( + AwsV1BindingSupport.class); + + /** + * V1 credential provider classname: {@code}. + */ + public static final String CREDENTIAL_PROVIDER_CLASSNAME = + "com.amazonaws.auth.AWSCredentialsProvider"; + + /** + * SDK availability. + */ + private static final boolean SDK_V1_FOUND = checkForAwsV1Sdk(); + + private AwsV1BindingSupport() { + } + + /** + * Probe for the AWS v1 SDK being available by looking for + * the class {@link #CREDENTIAL_PROVIDER_CLASSNAME}. + * @return true if it was found in the classloader + */ + private static boolean checkForAwsV1Sdk() { + + try { + ClassLoader cl = AwsV1BindingSupport.class.getClassLoader(); + cl.loadClass(CREDENTIAL_PROVIDER_CLASSNAME); + LOG.debug("v1 SDK class {} found", CREDENTIAL_PROVIDER_CLASSNAME); + return true; + } catch (Exception e) { + LOG.debug("v1 SDK class {} not found", CREDENTIAL_PROVIDER_CLASSNAME, e); + return false; + } + } + + /** + * Is the AWS v1 SDK available? + * @return true if it was found in the classloader + */ + public static synchronized boolean isAwsV1SdkAvailable() { + return SDK_V1_FOUND; + } + + + /** + * Create an AWS credential provider from its class by using reflection. The + * class must implement one of the following means of construction, which are + * attempted in order: + * + *
      + *
    1. a public constructor accepting java.net.URI and + * org.apache.hadoop.conf.Configuration
      2. + *
    2. a public constructor accepting + * org.apache.hadoop.conf.Configuration
      3. + *
    3. a public static method named getInstance that accepts no + * arguments and returns an instance of + * com.amazonaws.auth.AWSCredentialsProvider, or
      4. + *
    4. a public default constructor.
      5. + *
    + * @param conf configuration + * @param className credential classname + * @param uri URI of the FS + * @param key configuration key to use + * @return the instantiated class + * @throws InstantiationIOException on any instantiation failure, including v1 SDK not found + * @throws IOException anything else. + */ + public static AwsCredentialsProvider createAWSV1CredentialProvider( + Configuration conf, + String className, + @Nullable URI uri, + final String key) throws IOException { + if (!isAwsV1SdkAvailable()) { + throw unavailable(uri, className, key, "No AWS v1 SDK available"); + } + return V1ToV2AwsCredentialProviderAdapter.create(conf, className, uri); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java new file mode 100644 index 0000000000000..2cbd9836d3a43 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/V1ToV2AwsCredentialProviderAdapter.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.adapter; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import javax.annotation.Nullable; + +import com.amazonaws.SdkClientException; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSSessionCredentials; +import com.amazonaws.auth.AnonymousAWSCredentials; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.CredentialInitializationException; +import org.apache.hadoop.fs.s3a.S3AUtils; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; + +/** + * Adapts a V1 {@link AWSCredentialsProvider} to the V2 {@link AwsCredentialsProvider} interface. + */ +public final class V1ToV2AwsCredentialProviderAdapter + implements AwsCredentialsProvider, Closeable { + + private static final Logger LOG = LoggerFactory.getLogger( + V1ToV2AwsCredentialProviderAdapter.class); + + /** + * The V1 credential provider constructed. + */ + private final AWSCredentialsProvider v1CredentialsProvider; + + + private V1ToV2AwsCredentialProviderAdapter(AWSCredentialsProvider v1CredentialsProvider) { + this.v1CredentialsProvider = requireNonNull(v1CredentialsProvider); + } + + + /** + * Collect v1 credentials and convert to v2. + * @return v2 credentials + * @throws CredentialInitializationException if the inner retrieval raised an exception + */ + @Override + public AwsCredentials resolveCredentials() { + try { + // get the wrapped credentials + AWSCredentials toAdapt = v1CredentialsProvider.getCredentials(); + return convertToV2Credentials(toAdapt); + } catch (SdkClientException e) { + // wrap with a v2 exception so that code which adds a try/catch for v2 sdk exceptions + // gets a compatible exception. + throw new CredentialInitializationException(e.toString(), e); + } + } + + /** + * Close the wrapped provider if it implements Closeable/AutoCloseable. + * @throws IOException failure + */ + @Override + public void close() throws IOException { + if (v1CredentialsProvider instanceof Closeable) { + ((Closeable) v1CredentialsProvider).close(); + } else if (v1CredentialsProvider instanceof AutoCloseable) { + S3AUtils.closeAutocloseables(LOG, (AutoCloseable) v1CredentialsProvider); + } + } + + /** + * Convert v1 credentials to v2, including support for session and anonymous + * credentials. + * @param toAdapt credentials to adapt. + * @return v2 credentials. + */ + static AwsCredentials convertToV2Credentials(final AWSCredentials toAdapt) { + if (toAdapt instanceof AWSSessionCredentials) { + return AwsSessionCredentials.create(toAdapt.getAWSAccessKeyId(), + toAdapt.getAWSSecretKey(), + ((AWSSessionCredentials) toAdapt).getSessionToken()); + } else if (toAdapt instanceof AnonymousAWSCredentials) { + return AnonymousCredentialsProvider.create().resolveCredentials(); + } else { + return AwsBasicCredentials.create(toAdapt.getAWSAccessKeyId(), toAdapt.getAWSSecretKey()); + } + } + + @Override + public String toString() { + return "V1ToV2AwsCredentialProviderAdapter{" + + "v1CredentialsProvider=" + v1CredentialsProvider + + '}'; + } + + /** + * @param v1CredentialsProvider V1 credential provider to adapt. + * @return A new instance of the credentials provider adapter. + */ + static AwsCredentialsProvider create(AWSCredentialsProvider v1CredentialsProvider) { + return new V1ToV2AwsCredentialProviderAdapter(v1CredentialsProvider); + } + + /** + * Create an AWS credential provider from its class by using reflection. The + * class must implement one of the following means of construction, which are + * attempted in order: + * + *
      + *
    1. a public constructor accepting java.net.URI and + * org.apache.hadoop.conf.Configuration
      2. + *
    2. a public constructor accepting + * org.apache.hadoop.conf.Configuration
      3. + *
    3. a public static method named getInstance that accepts no + * arguments and returns an instance of + * com.amazonaws.auth.AWSCredentialsProvider, or
      4. + *
    4. a public default constructor.
      5. + *
    + * @param conf configuration + * @param className classname + * @param uri URI of the FS + * @return the instantiated class + * @throws InstantiationIOException on construction and instantiation failures, + * including v1 SDK exceptions. + * @throws IOException if raised by a constructor/factory method. + */ + static AwsCredentialsProvider create( + Configuration conf, + String className, + @Nullable URI uri) throws InstantiationIOException, IOException { + + final AWSCredentialsProvider instance = + S3AUtils.getInstanceFromReflection(className, conf, uri, AWSCredentialsProvider.class, + "getInstance", AWS_CREDENTIALS_PROVIDER); + return create(instance); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java new file mode 100644 index 0000000000000..124534188bccc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/adapter/package-info.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Adapter classes for allowing V1 credential providers to be used with SDKV2. + * This is the only package where use of aws v1 classes are permitted; + * all instantiations of objects here must use reflection to probe for + * availability or be prepared to catch exceptions which may be raised + * if the v1 SDK isn't found on the classpath + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.s3a.adapter; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java index 2a4771925f086..99a898f728166 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java @@ -19,41 +19,37 @@ package org.apache.hadoop.fs.s3a.api; import javax.annotation.Nullable; -import java.io.File; -import java.io.InputStream; import java.util.List; -import java.util.Optional; - -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.CannedAccessControlList; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams; -import com.amazonaws.services.s3.model.SSECustomerKey; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.StorageClass; -import com.amazonaws.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; /** - * Factory for S3 objects. + * Factory for S3 request objects. * * This is where the owner FS's {@code prepareRequest()} * callback is invoked to mark up a request for this span. @@ -67,6 +63,8 @@ * as there are no guarantees how they are processed. * That is: no guarantees of retry or translation. */ +@InterfaceStability.Unstable +@InterfaceAudience.LimitedPrivate("testing/diagnostics") public interface RequestFactory { /** @@ -79,22 +77,7 @@ public interface RequestFactory { * Get the canned ACL of this FS. * @return an ACL, if any */ - CannedAccessControlList getCannedACL(); - - /** - * Create the AWS SDK structure used to configure SSE, - * if the encryption secrets contain the information/settings for this. - * @return an optional set of KMS Key settings - */ - Optional generateSSEAwsKeyParams(); - - /** - * Create the SSE-C structure for the AWS SDK, if the encryption secrets - * contain the information/settings for this. - * This will contain a secret extracted from the bucket/configuration. - * @return an optional customer key. - */ - Optional generateSSECustomerKey(); + String getCannedACL(); /** * Get the encryption algorithm of this endpoint. @@ -115,79 +98,58 @@ public interface RequestFactory { StorageClass getStorageClass(); /** - * Create a new object metadata instance. - * Any standard metadata headers are added here, for example: - * encryption. - * - * @param length length of data to set in header; Ignored if negative - * @return a new metadata instance - */ - ObjectMetadata newObjectMetadata(long length); - - /** - * Create a copy request. + * Create a copy request builder. * This includes the work of copying the relevant parts * of the metadata from the source * @param srcKey source * @param dstKey destination * @param srcom source object metadata. - * @return the request + * @return the request builder */ - CopyObjectRequest newCopyObjectRequest(String srcKey, + CopyObjectRequest.Builder newCopyObjectRequestBuilder(String srcKey, String dstKey, - ObjectMetadata srcom); + HeadObjectResponse srcom); - /** - * Create a putObject request. - * Adds the ACL and metadata - * @param key key of object - * @param metadata metadata header - * @param options options for the request - * @param srcfile source file - * @return the request - */ - PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, PutObjectOptions options, File srcfile); /** - * Create a {@link PutObjectRequest} request. + * Create a {@link PutObjectRequest} request builder. * The metadata is assumed to have been configured with the size of the * operation. * @param key key of object - * @param metadata metadata header * @param options options for the request - * @param inputStream source data. - * @return the request + * @param length length of object to be uploaded + * @param isDirectoryMarker true if object to be uploaded is a directory marker + * @return the request builder */ - PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, + PutObjectRequest.Builder newPutObjectRequestBuilder(String key, PutObjectOptions options, - InputStream inputStream); + long length, + boolean isDirectoryMarker); /** * Create a {@link PutObjectRequest} request for creating * an empty directory. * * @param directory destination directory. - * @return request for a zero byte upload. + * @return request builder for a zero byte upload. */ - PutObjectRequest newDirectoryMarkerRequest(String directory); + PutObjectRequest.Builder newDirectoryMarkerRequest(String directory); /** * List all multipart uploads under a prefix. * @param prefix prefix to list under - * @return the request. + * @return the request builder. */ - ListMultipartUploadsRequest newListMultipartUploadsRequest( + ListMultipartUploadsRequest.Builder newListMultipartUploadsRequestBuilder( @Nullable String prefix); /** * Abort a multipart upload. * @param destKey destination object key * @param uploadId ID of initiated upload - * @return the request. + * @return the request builder. */ - AbortMultipartUploadRequest newAbortMultipartUploadRequest( + AbortMultipartUploadRequest.Builder newAbortMultipartUploadRequestBuilder( String destKey, String uploadId); @@ -195,10 +157,10 @@ AbortMultipartUploadRequest newAbortMultipartUploadRequest( * Start a multipart upload. * @param destKey destination object key * @param options options for the request - * @return the request. + * @return the request builder. * @throws PathIOException if multipart uploads are disabled */ - InitiateMultipartUploadRequest newMultipartUploadRequest( + CreateMultipartUploadRequest.Builder newMultipartUploadRequestBuilder( String destKey, @Nullable PutObjectOptions options) throws PathIOException; @@ -207,107 +169,95 @@ InitiateMultipartUploadRequest newMultipartUploadRequest( * @param destKey destination object key * @param uploadId ID of initiated upload * @param partETags ordered list of etags - * @return the request. + * @return the request builder. */ - CompleteMultipartUploadRequest newCompleteMultipartUploadRequest( + CompleteMultipartUploadRequest.Builder newCompleteMultipartUploadRequestBuilder( String destKey, String uploadId, - List partETags); + List partETags); /** - * Create a HEAD request. + * Create a HEAD object request builder. * @param key key, may have trailing / - * @return the request. + * @return the request builder. + */ + HeadObjectRequest.Builder newHeadObjectRequestBuilder(String key); + + /** + * Create a HEAD bucket request builder. + * @param bucket bucket to get metadata for + * @return the request builder. */ - GetObjectMetadataRequest newGetObjectMetadataRequest(String key); + HeadBucketRequest.Builder newHeadBucketRequestBuilder(String bucket); + /** - * Create a GET request. + * Create a GET request builder. * @param key object key - * @return the request. + * @return the request builder. */ - GetObjectRequest newGetObjectRequest(String key); + GetObjectRequest.Builder newGetObjectRequestBuilder(String key); /** - * Create and initialize a part request of a multipart upload. - * Exactly one of: {@code uploadStream} or {@code sourceFile} - * must be specified. - * A subset of the file may be posted, by providing the starting point - * in {@code offset} and a length of block in {@code size} equal to - * or less than the remaining bytes. - * @param destKey destination key of ongoing operation - * @param uploadId ID of ongoing upload - * @param partNumber current part number of the upload - * @param size amount of data - * @param uploadStream source of data to upload - * @param sourceFile optional source file. - * @param offset offset in file to start reading. - * @return the request. + * Create and initialize a part request builder of a multipart upload. + * + * @param destKey destination key of ongoing operation + * @param uploadId ID of ongoing upload + * @param partNumber current part number of the upload + * @param size amount of data + * @return the request builder. * @throws PathIOException if the part number is out of range. */ - UploadPartRequest newUploadPartRequest( + UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, - long size, - InputStream uploadStream, - File sourceFile, - long offset) throws PathIOException; + long size) throws PathIOException; /** - * Create a S3 Select request for the destination object. + * Create a S3 Select request builder for the destination object. * This does not build the query. * @param key object key - * @return the request + * @return the request builder */ - SelectObjectContentRequest newSelectRequest(String key); + SelectObjectContentRequest.Builder newSelectRequestBuilder(String key); /** - * Create the (legacy) V1 list request. + * Create the (legacy) V1 list request builder. * @param key key to list under * @param delimiter delimiter for keys * @param maxKeys maximum number in a list page. - * @return the request + * @return the request builder. */ - ListObjectsRequest newListObjectsV1Request(String key, + ListObjectsRequest.Builder newListObjectsV1RequestBuilder(String key, String delimiter, int maxKeys); /** - * Create the next V1 page list request, following - * on from the previous response. - * @param prev previous response - * @return the request - */ - - ListNextBatchOfObjectsRequest newListNextBatchOfObjectsRequest( - ObjectListing prev); - - /** - * Create a V2 list request. + * Create a V2 list request builder. * This will be recycled for any subsequent requests. * @param key key to list under * @param delimiter delimiter for keys * @param maxKeys maximum number in a list page. - * @return the request + * @return the request builder. */ - ListObjectsV2Request newListObjectsV2Request(String key, + ListObjectsV2Request.Builder newListObjectsV2RequestBuilder(String key, String delimiter, int maxKeys); /** - * Create a request to delete a single object. + * Create a request builder to delete a single object. * @param key object to delete - * @return the request + * @return the request builder. */ - DeleteObjectRequest newDeleteObjectRequest(String key); + DeleteObjectRequest.Builder newDeleteObjectRequestBuilder(String key); /** - * Bulk delete request. + * Create a request builder to delete objects in bulk. * @param keysToDelete list of keys to delete. - * @return the request + * @return the request builder. */ - DeleteObjectsRequest newBulkDeleteRequest( - List keysToDelete); + DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder( + List keysToDelete); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java index 8134d5cea942e..712246c21961c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java @@ -18,15 +18,9 @@ package org.apache.hadoop.fs.s3a.audit; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.Request; -import com.amazonaws.Response; -import com.amazonaws.SdkBaseException; -import com.amazonaws.handlers.HandlerAfterAttemptContext; -import com.amazonaws.handlers.HandlerBeforeAttemptContext; -import com.amazonaws.http.HttpResponse; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; -import org.apache.hadoop.fs.s3a.Retries; /** * Callbacks for audit spans. This is implemented @@ -37,10 +31,10 @@ * detect this and raise an exception. * * Look at the documentation for - * {@code com.amazonaws.handlers.IRequestHandler2} for details + * {@code ExecutionInterceptor} for details * on the callbacks. */ -public interface AWSAuditEventCallbacks { +public interface AWSAuditEventCallbacks extends ExecutionInterceptor { /** * Return a span ID which must be unique for all spans within @@ -66,95 +60,8 @@ public interface AWSAuditEventCallbacks { * It is not invoked on any AWS requests created in the SDK. * Avoid raising exceptions or talking to any remote service; * this callback is for annotation rather than validation. - * @param request request request. - * @param type of request - * @return the request, possibly modified. + * @param builder the request builder. */ - default T requestCreated(T request) { - return request; - } + default void requestCreated(SdkRequest.Builder builder) {} - /** - * Preflight preparation of AWS request. - * @param request request - * @param type of request - * @return an updated request. - * @throws AuditFailureException for generic audit failures - * @throws SdkBaseException for other reasons. - */ - @Retries.OnceRaw - default T beforeExecution(T request) - throws AuditFailureException, SdkBaseException { - return request; - } - - /** - * Callback after S3 responded to a request. - * @param request request - * @param response response. - * @throws AuditFailureException for generic audit failures - * @throws SdkBaseException for other reasons. - */ - default void afterResponse(Request request, - Response response) - throws AuditFailureException, SdkBaseException { - } - - /** - * Callback after a request resulted in an error. - * @param request request - * @param response response. - * @param exception exception raised. - * @throws AuditFailureException for generic audit failures - * @throws SdkBaseException for other reasons. - */ - default void afterError(Request request, - Response response, - Exception exception) - throws AuditFailureException, SdkBaseException { - } - - /** - * Request before marshalling. - * @param request request - * @return possibly modified request. - */ - default AmazonWebServiceRequest beforeMarshalling( - AmazonWebServiceRequest request) { - return request; - } - - /** - * Request before marshalling. - * @param request request - */ - default void beforeRequest(Request request) { - } - - /** - * Before any attempt is made. - * @param context full context, including the request. - */ - default void beforeAttempt(HandlerBeforeAttemptContext context) { - } - - /** - * After any attempt is made. - * @param context full context, including the request. - */ - default void afterAttempt( - HandlerAfterAttemptContext context) { - } - - /** - * Before unmarshalling the response. - * @param request request made. - * @param httpResponse response received - * @return updated response. - */ - default HttpResponse beforeUnmarshalling( - final Request request, - final HttpResponse httpResponse) { - return httpResponse; - } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java index b4be341c912e0..8a24a4e14db1c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java @@ -20,24 +20,23 @@ import java.util.List; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CopyPartRequest; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.GetBucketLocationRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_HEAD_REQUEST; @@ -64,102 +63,85 @@ public class AWSRequestAnalyzer { * read/write and path. * @param request request. * @return information about the request. - * @param type of request. */ - public RequestInfo analyze(T request) { + public RequestInfo analyze(SdkRequest request) { // this is where Scala's case statement would massively // simplify life. // Please Keep in Alphabetical Order. if (request instanceof AbortMultipartUploadRequest) { return writing(MULTIPART_UPLOAD_ABORTED, - ((AbortMultipartUploadRequest) request).getKey(), + ((AbortMultipartUploadRequest) request).key(), 0); } else if (request instanceof CompleteMultipartUploadRequest) { CompleteMultipartUploadRequest r = (CompleteMultipartUploadRequest) request; return writing(MULTIPART_UPLOAD_COMPLETED, - r.getKey(), - r.getPartETags().size()); + r.key(), + r.multipartUpload().parts().size()); + } else if (request instanceof CreateMultipartUploadRequest) { + return writing(MULTIPART_UPLOAD_STARTED, + ((CreateMultipartUploadRequest) request).key(), + 0); } else if (request instanceof DeleteObjectRequest) { // DeleteObject: single object return writing(OBJECT_DELETE_REQUEST, - ((DeleteObjectRequest) request).getKey(), + ((DeleteObjectRequest) request).key(), 1); } else if (request instanceof DeleteObjectsRequest) { // DeleteObjects: bulk delete // use first key as the path DeleteObjectsRequest r = (DeleteObjectsRequest) request; - List keys - = r.getKeys(); + List objectIdentifiers + = r.delete().objects(); return writing(OBJECT_BULK_DELETE_REQUEST, - keys.isEmpty() ? null : keys.get(0).getKey(), - keys.size()); + objectIdentifiers.isEmpty() ? null : objectIdentifiers.get(0).key(), + objectIdentifiers.size()); } else if (request instanceof GetBucketLocationRequest) { GetBucketLocationRequest r = (GetBucketLocationRequest) request; return reading(STORE_EXISTS_PROBE, - r.getBucketName(), + r.bucket(), 0); - } else if (request instanceof GetObjectMetadataRequest) { - return reading(ACTION_HTTP_HEAD_REQUEST, - ((GetObjectMetadataRequest) request).getKey(), 0); } else if (request instanceof GetObjectRequest) { GetObjectRequest r = (GetObjectRequest) request; - long[] range = r.getRange(); - long size = range == null - ? -1 - : range[1] - range[0]; return reading(ACTION_HTTP_GET_REQUEST, - r.getKey(), - size); - } else if (request instanceof InitiateMultipartUploadRequest) { - return writing(MULTIPART_UPLOAD_STARTED, - ((InitiateMultipartUploadRequest) request).getKey(), - 0); + r.key(), + sizeFromRangeHeader(r.range())); + } else if (request instanceof HeadObjectRequest) { + return reading(ACTION_HTTP_HEAD_REQUEST, + ((HeadObjectRequest) request).key(), 0); } else if (request instanceof ListMultipartUploadsRequest) { ListMultipartUploadsRequest r = (ListMultipartUploadsRequest) request; return reading(MULTIPART_UPLOAD_LIST, - r.getPrefix(), - r.getMaxUploads()); + r.prefix(), + r.maxUploads()); } else if (request instanceof ListObjectsRequest) { ListObjectsRequest r = (ListObjectsRequest) request; return reading(OBJECT_LIST_REQUEST, - r.getPrefix(), - r.getMaxKeys()); - } else if (request instanceof ListNextBatchOfObjectsRequest) { - ListNextBatchOfObjectsRequest r = (ListNextBatchOfObjectsRequest) request; - ObjectListing l = r.getPreviousObjectListing(); - String prefix = ""; - int size = 0; - if (l != null) { - prefix = l.getPrefix(); - size = l.getMaxKeys(); - } - return reading(OBJECT_LIST_REQUEST, - prefix, - size); + r.prefix(), + r.maxKeys()); } else if (request instanceof ListObjectsV2Request) { ListObjectsV2Request r = (ListObjectsV2Request) request; return reading(OBJECT_LIST_REQUEST, - r.getPrefix(), - r.getMaxKeys()); + r.prefix(), + r.maxKeys()); } else if (request instanceof PutObjectRequest) { PutObjectRequest r = (PutObjectRequest) request; return writing(OBJECT_PUT_REQUEST, - r.getKey(), + r.key(), 0); } else if (request instanceof SelectObjectContentRequest) { SelectObjectContentRequest r = (SelectObjectContentRequest) request; return reading(OBJECT_SELECT_REQUESTS, - r.getKey(), + r.key(), 1); } else if (request instanceof UploadPartRequest) { UploadPartRequest r = (UploadPartRequest) request; return writing(MULTIPART_UPLOAD_PART_PUT, - r.getKey(), - r.getPartSize()); + r.key(), + r.contentLength()); } // no explicit support, return classname return writing(request.getClass().getName(), null, 0); @@ -212,7 +194,7 @@ private RequestInfo writing(final String verb, */ public static boolean isRequestNotAlwaysInSpan(final Object request) { - return request instanceof CopyPartRequest + return request instanceof UploadPartCopyRequest || request instanceof CompleteMultipartUploadRequest || request instanceof GetBucketLocationRequest; } @@ -225,9 +207,9 @@ private RequestInfo writing(final String verb, * @return true if the transfer manager creates them. */ public static boolean isRequestMultipartIO(final Object request) { - return request instanceof CopyPartRequest + return request instanceof UploadPartCopyRequest || request instanceof CompleteMultipartUploadRequest - || request instanceof InitiateMultipartUploadRequest + || request instanceof CreateMultipartUploadRequest || request instanceof UploadPartRequest; } @@ -307,4 +289,23 @@ public String toString() { private static long toSafeLong(final Number size) { return size != null ? size.longValue() : 0; } + + private static final String BYTES_PREFIX = "bytes="; + + private static Number sizeFromRangeHeader(String rangeHeader) { + if (rangeHeader != null && rangeHeader.startsWith(BYTES_PREFIX)) { + String[] values = rangeHeader + .substring(BYTES_PREFIX.length()) + .split("-"); + if (values.length == 2) { + try { + long start = Long.parseUnsignedLong(values[0]); + long end = Long.parseUnsignedLong(values[0]); + return end - start; + } catch(NumberFormatException e) { + } + } + } + return -1; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java index c66f45eb309ff..dfbe8e1a13502 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java @@ -23,7 +23,7 @@ import java.lang.reflect.InvocationTargetException; import java.nio.file.AccessDeniedException; -import com.amazonaws.HandlerContextAware; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,10 +34,11 @@ import org.apache.hadoop.fs.s3a.audit.impl.NoopAuditManagerS3A; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; + import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED_DEFAULT; -import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_HANDLER_CONTEXT; +import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_EXECUTION_ATTRIBUTE; /** * Support for integrating auditing within the S3A code. @@ -123,25 +124,24 @@ public static OperationAuditor createAndInitAuditor( } /** - * Get the span from a handler context. - * @param request request - * @param type of request. + * Get the span from the execution attributes. + * @param executionAttributes the execution attributes * @return the span callbacks or null */ - public static AWSAuditEventCallbacks - retrieveAttachedSpan(final T request) { - return request.getHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT); + public static AuditSpanS3A + retrieveAttachedSpan(final ExecutionAttributes executionAttributes) { + return executionAttributes.getAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE); } /** - * Attach a span to a handler context. - * @param request request + * Attach a span to the execution attributes. + * @param executionAttributes the execution attributes * @param span span to attach - * @param type of request. */ - public static void attachSpanToRequest( - final T request, final AWSAuditEventCallbacks span) { - request.addHandlerContext(AUDIT_SPAN_HANDLER_CONTEXT, span); + public static void attachSpanToRequest( + final ExecutionAttributes executionAttributes, + final AuditSpanS3A span) { + executionAttributes.putAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE, span); } /** @@ -160,4 +160,30 @@ public static IOException translateAuditException(String path, return (AccessDeniedException)new AccessDeniedException(path, null, exception.toString()).initCause(exception); } + + /** + * Translate an exception if it or its inner exception is an + * {@link AuditFailureException}. + * If this condition is not met, null is returned. + * @param path path of operation. + * @param exception exception + * @return a translated exception or null. + */ + public static IOException maybeTranslateAuditException(String path, + Exception exception) { + if (exception instanceof AuditFailureException) { + return translateAuditException(path, (AuditFailureException) exception); + } else if (exception.getCause() instanceof AuditFailureException) { + return translateAuditException(path, + (AuditFailureException) exception.getCause()); + } else { + return null; + } + } + + public static boolean containsAuditException(Exception exception) { + return exception instanceof AuditFailureException + || exception.getCause() instanceof AuditFailureException; + } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java index c1302d57454b1..bb7f94cfc206d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.List; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.service.Service; + /** * Interface for Audit Managers auditing operations through the * AWS libraries. @@ -56,24 +57,24 @@ public interface AuditManagerS3A extends Service, OperationAuditor getAuditor(); /** - * Create the request handler(s) for this audit service. - * The list returned is mutable; new handlers may be added. - * @return list of handlers for the SDK. + * Create the execution interceptor(s) for this audit service. + * The list returned is mutable; new interceptors may be added. + * @return list of interceptors for the SDK. * @throws IOException failure. */ - List createRequestHandlers() throws IOException; + List createExecutionInterceptors() throws IOException; /** - * Return a transfer state change callback which + * Return a transfer callback which * fixes the active span context to be that in which - * the state change listener was created. + * the transfer listener was created. * This can be used to audit the creation of the multipart * upload initiation request which the transfer manager * makes when a file to be copied is split up. * This must be invoked/used within the active span. - * @return a state change listener. + * @return a transfer listener. */ - TransferStateChangeListener createStateChangeListener(); + TransferListener createTransferListener(); /** * Check for permission to access a path. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java index 1d76833f8ceab..55deb0a1989de 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java @@ -66,13 +66,20 @@ private S3AAuditConstants() { "org.apache.hadoop.fs.s3a.audit.impl.NoopAuditor"; /** - * List of extra AWS SDK request handlers: {@value}. - * These are added to the SDK request chain after - * any audit service. + * Deprecated list of extra AWS SDK request handlers: {@value}. + * Use {@link #AUDIT_EXECUTION_INTERCEPTORS} instead. */ public static final String AUDIT_REQUEST_HANDLERS = "fs.s3a.audit.request.handlers"; + /** + * List of extra AWS SDK execution interceptors: {@value}. + * These are added to the SDK request chain after + * any audit service. + */ + public static final String AUDIT_EXECUTION_INTERCEPTORS = + "fs.s3a.audit.execution.interceptors"; + /** * Should operations outside spans be rejected? * This is for testing coverage of the span code; if used diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java index 3d2102d305c7d..9dd04af68e8a9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java @@ -25,16 +25,16 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.HandlerContextAware; -import com.amazonaws.Request; -import com.amazonaws.Response; -import com.amazonaws.SdkBaseException; -import com.amazonaws.handlers.HandlerAfterAttemptContext; -import com.amazonaws.handlers.HandlerBeforeAttemptContext; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.http.HttpResponse; -import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.SdkResponse; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; + +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +56,7 @@ import org.apache.hadoop.fs.s3a.audit.OperationAuditor; import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions; import org.apache.hadoop.fs.s3a.audit.S3AAuditConstants; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.service.CompositeService; @@ -66,6 +67,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION; import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.attachSpanToRequest; import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.retrieveAttachedSpan; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS; /** @@ -82,10 +84,11 @@ * will deactivate the wrapped span and then * switch the active span to the unbounded span. * - * The inner class {@link AWSAuditEventCallbacks} is returned - * as a request handler in {@link #createRequestHandlers()}; - * this forwards all requests to the outer {@code ActiveAuditManagerS3A}, - * which then locates the active span and forwards the request. + * This class also implements {@link ExecutionInterceptor} and + * returns itself in {@link #createExecutionInterceptors()}; + * once registered with the S3 client, the implemented methods + * will be called during different parts of an SDK request lifecycle, + * which then locate the active span and forward the request. * If any such invocation raises an {@link AuditFailureException} * then the IOStatistics counter for {@code AUDIT_FAILURE} * is incremented. @@ -390,25 +393,39 @@ public AuditSpanS3A createSpan(final String operation, } /** - * Return a request handler for the AWS SDK which + * Return a list of execution interceptors for the AWS SDK which * relays to this class. - * @return a request handler. + * @return a list of execution interceptors. */ @Override - public List createRequestHandlers() + public List createExecutionInterceptors() throws IOException { // wire up the AWS SDK To call back into this class when // preparing to make S3 calls. - List requestHandlers = new ArrayList<>(); - requestHandlers.add(new SdkRequestHandler()); - // now look for any more handlers - final Class[] handlers = getConfig().getClasses(AUDIT_REQUEST_HANDLERS); - if (handlers != null) { - for (Class handler : handlers) { + List executionInterceptors = new ArrayList<>(); + executionInterceptors.add(this); + + final String handlers = getConfig().getTrimmed(AUDIT_REQUEST_HANDLERS, ""); + if (!handlers.isEmpty()) { + // warn and ignore v1 handlers. + V2Migration.v1RequestHandlersUsed(handlers); + } + + // V2 SDK supports global/service interceptors, but they need to be configured on the + // classpath and don't get the filesystem/job configuration passed down. + final Class[] interceptors = getConfig().getClasses(AUDIT_EXECUTION_INTERCEPTORS); + if (interceptors != null) { + for (Class handler : interceptors) { try { + LOG.debug("Adding intercept of class {}", handler); Constructor ctor = handler.getConstructor(); - requestHandlers.add((RequestHandler2)ctor.newInstance()); + final ExecutionInterceptor interceptor = (ExecutionInterceptor) ctor.newInstance(); + if (interceptor instanceof Configurable) { + // pass in the configuration. + ((Configurable) interceptor).setConf(getConfig()); + } + executionInterceptors.add(interceptor); } catch (ExceptionInInitializerError e) { throw FutureIO.unwrapInnerException(e); } catch (Exception e) { @@ -416,13 +433,18 @@ public List createRequestHandlers() } } } - return requestHandlers; + return executionInterceptors; } @Override - public TransferStateChangeListener createStateChangeListener() { + public TransferListener createTransferListener() { final WrappingAuditSpan span = activeSpan(); - return (transfer, state) -> switchToActiveSpan(span); + return new TransferListener() { + @Override + public void transferInitiated(Context.TransferInitiated context) { + switchToActiveSpan(span); + } + }; } @Override @@ -434,20 +456,18 @@ public boolean checkAccess(final Path path, } /** - * Attach a reference to the active thread span, then - * invoke the same callback on that active thread. + * Audit the creation of a request and retrieve + * a reference to the active thread span. */ @Override - public T requestCreated( - final T request) { + public void requestCreated(final SdkRequest.Builder builder) { AuditSpanS3A span = getActiveAuditSpan(); if (LOG.isTraceEnabled()) { LOG.trace("Created Request {} in span {}", - analyzer.analyze(request), span); + analyzer.analyze(builder.build()), span); } - attachSpanToRequest(request, span); try { - return span.requestCreated(request); + span.requestCreated(builder); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -463,14 +483,13 @@ public T requestCreated( * {@inheritDoc} */ @Override - public T beforeExecution( - final T request) { + public void beforeExecution(Context.BeforeExecution context, + ExecutionAttributes executionAttributes) { ioStatisticsStore.incrementCounter(AUDIT_REQUEST_EXECUTION.getSymbol()); - - // identify the span and invoke the callback + AuditSpanS3A span = getActiveAuditSpan(); + attachSpanToRequest(executionAttributes, span); try { - return extractAndActivateSpanFromRequest(request) - .beforeExecution(request); + span.beforeExecution(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -479,16 +498,14 @@ public T beforeExecution( /** * Forward to active span. - * @param request request - * @param response response. + * {@inheritDoc} */ @Override - public void afterResponse(final Request request, - final Response response) - throws AuditFailureException, SdkBaseException { + public void afterExecution(Context.AfterExecution context, + ExecutionAttributes executionAttributes) { try { - extractAndActivateSpanFromRequest(request) - .afterResponse(request, response); + extractAndActivateSpanFromRequest(context.request(), executionAttributes) + .afterExecution(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -496,18 +513,19 @@ public void afterResponse(final Request request, } /** - * Get the active span from the handler context, + * Get the active span from the execution attributes, * falling back to the active thread span if there - * is nothing in the context. - * Provided the span is a wrapped span, the + * is nothing in the attributes. + * Provided the span is a wrapped span, the span is + * activated. * @param request request - * @param type of request. - * @return the callbacks + * @param executionAttributes the execution attributes + * @return the active span */ - private AWSAuditEventCallbacks - extractAndActivateSpanFromRequest(final T request) { - AWSAuditEventCallbacks span; - span = retrieveAttachedSpan(request); + private AuditSpanS3A extractAndActivateSpanFromRequest( + final SdkRequest request, + final ExecutionAttributes executionAttributes) { + AuditSpanS3A span = retrieveAttachedSpan(executionAttributes); if (span == null) { // no span is attached. Not unusual for the copy operations, // or for calls to GetBucketLocation made by the AWS client @@ -530,18 +548,16 @@ public void afterResponse(final Request request, /** * Forward to active span. - * @param request request - * @param response response. - * @param exception exception raised. + * @param context execution context + * @param executionAttributes the execution attributes + * {@inheritDoc} */ @Override - public void afterError(final Request request, - final Response response, - final Exception exception) - throws AuditFailureException, SdkBaseException { + public void onExecutionFailure(Context.FailedExecution context, + ExecutionAttributes executionAttributes) { try { - extractAndActivateSpanFromRequest(request) - .afterError(request, response, exception); + extractAndActivateSpanFromRequest(context.request(), executionAttributes).onExecutionFailure( + context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -549,11 +565,12 @@ public void afterError(final Request request, } @Override - public AmazonWebServiceRequest beforeMarshalling( - final AmazonWebServiceRequest request) { + public SdkRequest modifyRequest(Context.ModifyRequest context, + ExecutionAttributes executionAttributes) { try { - return extractAndActivateSpanFromRequest(request) - .beforeMarshalling(request); + return extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .modifyRequest(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -561,10 +578,12 @@ public AmazonWebServiceRequest beforeMarshalling( } @Override - public void beforeRequest(final Request request) { + public void beforeMarshalling(Context.BeforeMarshalling context, + ExecutionAttributes executionAttributes) { try { - extractAndActivateSpanFromRequest(request) - .beforeRequest(request); + extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .beforeMarshalling(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -572,10 +591,12 @@ public void beforeRequest(final Request request) { } @Override - public void beforeAttempt(final HandlerBeforeAttemptContext context) { + public void afterMarshalling(Context.AfterMarshalling context, + ExecutionAttributes executionAttributes) { try { - extractAndActivateSpanFromRequest(context.getRequest()) - .beforeAttempt(context); + extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .afterMarshalling(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -583,10 +604,12 @@ public void beforeAttempt(final HandlerBeforeAttemptContext context) { } @Override - public void afterAttempt(final HandlerAfterAttemptContext context) { + public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, + ExecutionAttributes executionAttributes) { try { - extractAndActivateSpanFromRequest(context.getRequest()) - .afterAttempt(context); + return extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .modifyHttpRequest(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; @@ -594,73 +617,80 @@ public void afterAttempt(final HandlerAfterAttemptContext context) { } @Override - public HttpResponse beforeUnmarshalling(final Request request, - final HttpResponse httpResponse) { + public void beforeTransmission(Context.BeforeTransmission context, + ExecutionAttributes executionAttributes) { try { - extractAndActivateSpanFromRequest(request.getOriginalRequest()) - .beforeUnmarshalling(request, httpResponse); + extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .beforeTransmission(context, executionAttributes); } catch (AuditFailureException e) { ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); throw e; } - return httpResponse; } - /** - * Callbacks from the AWS SDK; all forward to the ActiveAuditManagerS3A. - * We need a separate class because the SDK requires the handler list - * to be list of {@code RequestHandler2} instances. - */ - private class SdkRequestHandler extends RequestHandler2 { - - @Override - public AmazonWebServiceRequest beforeExecution( - final AmazonWebServiceRequest request) { - return ActiveAuditManagerS3A.this.beforeExecution(request); - } - - @Override - public void afterResponse(final Request request, - final Response response) { - ActiveAuditManagerS3A.this.afterResponse(request, response); - } - - @Override - public void afterError(final Request request, - final Response response, - final Exception e) { - ActiveAuditManagerS3A.this.afterError(request, response, e); - } - - @Override - public AmazonWebServiceRequest beforeMarshalling( - final AmazonWebServiceRequest request) { - return ActiveAuditManagerS3A.this.beforeMarshalling(request); + @Override + public void afterTransmission(Context.AfterTransmission context, + ExecutionAttributes executionAttributes) { + try { + extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .afterTransmission(context, executionAttributes); + } catch (AuditFailureException e) { + ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); + throw e; } + } - @Override - public void beforeRequest(final Request request) { - ActiveAuditManagerS3A.this.beforeRequest(request); + @Override + public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context, + ExecutionAttributes executionAttributes) { + try { + return extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .modifyHttpResponse(context, executionAttributes); + } catch (AuditFailureException e) { + ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); + throw e; } + } - @Override - public void beforeAttempt( - final HandlerBeforeAttemptContext context) { - ActiveAuditManagerS3A.this.beforeAttempt(context); + @Override + public void beforeUnmarshalling(Context.BeforeUnmarshalling context, + ExecutionAttributes executionAttributes) { + try { + extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .beforeUnmarshalling(context, executionAttributes); + } catch (AuditFailureException e) { + ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); + throw e; } + } - @Override - public HttpResponse beforeUnmarshalling( - final Request request, - final HttpResponse httpResponse) { - return ActiveAuditManagerS3A.this.beforeUnmarshalling(request, - httpResponse); + @Override + public void afterUnmarshalling(Context.AfterUnmarshalling context, + ExecutionAttributes executionAttributes) { + try { + extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .afterUnmarshalling(context, executionAttributes); + } catch (AuditFailureException e) { + ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); + throw e; } + } - @Override - public void afterAttempt( - final HandlerAfterAttemptContext context) { - ActiveAuditManagerS3A.this.afterAttempt(context); + @Override + public SdkResponse modifyResponse(Context.ModifyResponse context, + ExecutionAttributes executionAttributes) { + try { + return extractAndActivateSpanFromRequest(context.request(), + executionAttributes) + .modifyResponse(context, executionAttributes); + } catch (AuditFailureException e) { + ioStatisticsStore.incrementCounter(AUDIT_FAILURE.getSymbol()); + throw e; } } @@ -748,9 +778,8 @@ public void deactivate() { * {@inheritDoc} */ @Override - public T requestCreated( - final T request) { - return span.requestCreated(request); + public void requestCreated(final SdkRequest.Builder builder) { + span.requestCreated(builder); } /** @@ -774,79 +803,132 @@ public void set(final String key, final String value) { /** * Forward to the inner span. - * @param request request - * @param type of request - * @return an updated request. + * {@inheritDoc} */ @Override - public T beforeExecution( - final T request) { - return span.beforeExecution(request); + public void beforeExecution(Context.BeforeExecution context, + ExecutionAttributes executionAttributes) { + span.beforeExecution(context, executionAttributes); } /** * Forward to the inner span. - * @param request request - * @param response response. + * {@inheritDoc} */ @Override - public void afterResponse(final Request request, - final Response response) { - span.afterResponse(request, response); + public void afterExecution(Context.AfterExecution context, + ExecutionAttributes executionAttributes) { + span.afterExecution(context, executionAttributes); } /** * Forward to the inner span. - * @param request request - * @param response response. - * @param exception exception raised. + * {@inheritDoc} */ @Override - public void afterError(final Request request, - final Response response, - final Exception exception) { - span.afterError(request, response, exception); + public void onExecutionFailure(Context.FailedExecution context, + ExecutionAttributes executionAttributes) { + span.onExecutionFailure(context, executionAttributes); } /** * Forward to the inner span. - * @param request request - * @return request to marshall + * {@inheritDoc} */ @Override - public AmazonWebServiceRequest beforeMarshalling( - final AmazonWebServiceRequest request) { - return span.beforeMarshalling(request); + public void beforeMarshalling(Context.BeforeMarshalling context, + ExecutionAttributes executionAttributes) { + span.beforeMarshalling(context, executionAttributes); } /** * Forward to the inner span. - * @param request request + * {@inheritDoc} */ @Override - public void beforeRequest(final Request request) { - span.beforeRequest(request); + public SdkRequest modifyRequest(Context.ModifyRequest context, + ExecutionAttributes executionAttributes) { + return span.modifyRequest(context, executionAttributes); } /** * Forward to the inner span. - * @param context full context, including the request. + * {@inheritDoc} */ @Override - public void beforeAttempt( - final HandlerBeforeAttemptContext context) { - span.beforeAttempt(context); + public void afterMarshalling(Context.AfterMarshalling context, + ExecutionAttributes executionAttributes) { + span.afterMarshalling(context, executionAttributes); } /** * Forward to the inner span. - * - * @param context full context, including the request. + * {@inheritDoc} */ @Override - public void afterAttempt( - final HandlerAfterAttemptContext context) { - span.afterAttempt(context); + public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, + ExecutionAttributes executionAttributes) { + return span.modifyHttpRequest(context, executionAttributes); + } + + /** + * Forward to the inner span. + * {@inheritDoc} + */ + @Override + public void beforeTransmission(Context.BeforeTransmission context, + ExecutionAttributes executionAttributes) { + span.beforeTransmission(context, executionAttributes); + } + + /** + * Forward to the inner span. + * {@inheritDoc} + */ + @Override + public void afterTransmission(Context.AfterTransmission context, + ExecutionAttributes executionAttributes) { + span.afterTransmission(context, executionAttributes); + } + + /** + * Forward to the inner span. + * {@inheritDoc} + */ + @Override + public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context, + ExecutionAttributes executionAttributes) { + return span.modifyHttpResponse(context, executionAttributes); + } + + /** + * Forward to the inner span. + * {@inheritDoc} + */ + @Override + public void beforeUnmarshalling(Context.BeforeUnmarshalling context, + ExecutionAttributes executionAttributes) { + span.beforeUnmarshalling(context, executionAttributes); + } + + /** + * Forward to the inner span. + * {@inheritDoc} + */ + @Override + public void afterUnmarshalling(Context.AfterUnmarshalling context, + ExecutionAttributes executionAttributes) { + span.afterUnmarshalling(context, executionAttributes); + } + + /** + * Forward to the inner span. + * {@inheritDoc} + */ + @Override + public SdkResponse modifyResponse(Context.ModifyResponse context, + ExecutionAttributes executionAttributes) { + return span.modifyResponse(context, executionAttributes); } @Override @@ -859,5 +941,4 @@ public String toString() { return sb.toString(); } } - } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index fcf2015487c48..3a2d9d7f823ee 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -24,10 +24,14 @@ import java.util.HashMap; import java.util.Map; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.awscore.AwsExecutionAttribute; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -258,21 +262,22 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { /** * Attach Range of data for GetObject Request. - * @param request given get object request + * @param request the sdk request to be modified + * @param executionAttributes execution attributes for this request */ - private void attachRangeFromRequest(AmazonWebServiceRequest request) { - if (request instanceof GetObjectRequest) { - long[] rangeValue = ((GetObjectRequest) request).getRange(); - if (rangeValue == null || rangeValue.length == 0) { - return; - } - if (rangeValue.length != 2) { - WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements." - + " Got {} elements. Ignoring.", rangeValue.length); - return; + private void attachRangeFromRequest(SdkHttpRequest request, + ExecutionAttributes executionAttributes) { + + String operationName = executionAttributes.getAttribute(AwsExecutionAttribute.OPERATION_NAME); + + if (operationName != null && operationName.equals("GetObject")) { + if (request.headers() != null && request.headers().get("Range") != null) { + String[] rangeHeader = request.headers().get("Range").get(0).split("="); + // only set header if range unit is bytes + if (rangeHeader[0].equals("bytes")) { + referrer.set(AuditConstants.PARAM_RANGE, rangeHeader[1]); + } } - String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]); - referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); } } @@ -346,64 +351,78 @@ public void set(final String key, final String value) { referrer.set(key, value); } + + /** - * Before execution, the logging auditor always builds - * the referrer header, saves to the outer class - * (where {@link #getLastHeader()} can retrieve it, + * Before transmitting a request, the logging auditor + * always builds the referrer header, saves to the outer + * class (where {@link #getLastHeader()} can retrieve it, * and logs at debug. * If configured to add the header to the S3 logs, it will * be set as the HTTP referrer. - * @param request request - * @param type of request. - * @return the request with any extra headers. + * @param context The current state of the execution, + * including the SDK and current HTTP request. + * @param executionAttributes A mutable set of attributes scoped + * to one specific request/response + * cycle that can be used to give data + * to future lifecycle methods. + * @return The potentially-modified HTTP request that should be + * sent to the service. Must not be null. */ @Override - public T beforeExecution( - final T request) { + public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, + ExecutionAttributes executionAttributes) { + SdkHttpRequest httpRequest = context.httpRequest(); + SdkRequest sdkRequest = context.request(); + // attach range for GetObject requests - attachRangeFromRequest(request); + attachRangeFromRequest(httpRequest, executionAttributes); + // for delete op, attach the number of files to delete - attachDeleteKeySizeAttribute(request); + attachDeleteKeySizeAttribute(sdkRequest); + // build the referrer header final String header = referrer.buildHttpReferrer(); // update the outer class's field. setLastHeader(header); if (headerEnabled) { // add the referrer header - request.putCustomRequestHeader(HEADER_REFERRER, - header); + httpRequest = httpRequest.toBuilder() + .appendHeader(HEADER_REFERRER, header) + .build(); } if (LOG.isDebugEnabled()) { LOG.debug("[{}] {} Executing {} with {}; {}", currentThreadID(), getSpanId(), getOperationName(), - analyzer.analyze(request), + analyzer.analyze(context.request()), header); } + // now see if the request is actually a blocked multipart request - if (!isMultipartUploadEnabled && isRequestMultipartIO(request)) { + if (!isMultipartUploadEnabled && isRequestMultipartIO(sdkRequest)) { throw new AuditOperationRejectedException("Multipart IO request " - + request + " rejected " + header); + + sdkRequest + " rejected " + header); } - return request; + return httpRequest; } /** * For delete requests, attach delete key size as a referrer attribute. * * @param request the request object. - * @param type of the request. */ - private void attachDeleteKeySizeAttribute(T request) { + private void attachDeleteKeySizeAttribute(SdkRequest request) { + if (request instanceof DeleteObjectsRequest) { - int keySize = ((DeleteObjectsRequest) request).getKeys().size(); - this.set(DELETE_KEYS_SIZE, String.valueOf(keySize)); + int keySize = ((DeleteObjectsRequest) request).delete().objects().size(); + referrer.set(DELETE_KEYS_SIZE, String.valueOf(keySize)); } else if (request instanceof DeleteObjectRequest) { - String key = ((DeleteObjectRequest) request).getKey(); + String key = ((DeleteObjectRequest) request).key(); if (key != null && key.length() > 0) { - this.set(DELETE_KEYS_SIZE, "1"); + referrer.set(DELETE_KEYS_SIZE, "1"); } } } @@ -460,15 +479,13 @@ public boolean isValidSpan() { } @Override - public T requestCreated( - final T request) { + public void requestCreated(final SdkRequest.Builder builder) { String error = "Creating a request outside an audit span " - + analyzer.analyze(request); + + analyzer.analyze(builder.build()); LOG.info(error); if (LOG.isDebugEnabled()) { LOG.debug(error, new AuditFailureException("unaudited")); } - return request; } /** @@ -476,20 +493,22 @@ public T requestCreated( * increment the failure count. * Some requests (e.g. copy part) are not expected in spans due * to how they are executed; these do not trigger failures. - * @param request request - * @param type of request - * @return an updated request. - * @throws AuditFailureException if failure is enabled. + * @param context The current state of the execution, including + * the unmodified SDK request from the service + * client call. + * @param executionAttributes A mutable set of attributes scoped + * to one specific request/response + * cycle that can be used to give data + * to future lifecycle methods. */ @Override - public T beforeExecution( - final T request) { - + public void beforeExecution(Context.BeforeExecution context, + ExecutionAttributes executionAttributes) { String error = "executing a request outside an audit span " - + analyzer.analyze(request); + + analyzer.analyze(context.request()); final String unaudited = getSpanId() + " " + UNAUDITED_OPERATION + " " + error; - if (isRequestNotAlwaysInSpan(request)) { + if (isRequestNotAlwaysInSpan(context.request())) { // can get by auditing during a copy, so don't overreact LOG.debug(unaudited); } else { @@ -500,7 +519,7 @@ public T beforeExecution( } } // now hand off to the superclass for its normal preparation - return super.beforeExecution(request); + super.beforeExecution(context, executionAttributes); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java index d1ebd922e073d..e58c906460daa 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java @@ -24,9 +24,8 @@ import java.util.List; import java.util.UUID; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.services.s3.transfer.Transfer; -import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -39,6 +38,7 @@ import org.apache.hadoop.fs.s3a.audit.OperationAuditorOptions; import org.apache.hadoop.service.CompositeService; + import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; /** @@ -121,17 +121,13 @@ public AuditSpanS3A createSpan(final String operation, } @Override - public List createRequestHandlers() throws IOException { + public List createExecutionInterceptors() throws IOException { return new ArrayList<>(); } @Override - public TransferStateChangeListener createStateChangeListener() { - return new TransferStateChangeListener() { - public void transferStateChanged(final Transfer transfer, - final Transfer.TransferState state) { - } - }; + public TransferListener createTransferListener() { + return new TransferListener() {}; } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java index f82e3d7f1e5e8..c170a2be6611d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java @@ -18,10 +18,10 @@ package org.apache.hadoop.fs.s3a.audit.impl; -import com.amazonaws.handlers.HandlerContextKey; +import software.amazon.awssdk.core.interceptor.ExecutionAttribute; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks; +import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; /** * Internal constants; not intended for public use, or @@ -34,11 +34,11 @@ private S3AInternalAuditConstants() { } /** - * Handler key for audit span callbacks. - * This is used to bind the handler in the AWS code. + * Exceution attribute for audit span callbacks. + * This is used to retrieve the span in the AWS code. */ - public static final HandlerContextKey - AUDIT_SPAN_HANDLER_CONTEXT = - new HandlerContextKey<>( - "org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks"); + public static final ExecutionAttribute + AUDIT_SPAN_EXECUTION_ATTRIBUTE = + new ExecutionAttribute<>( + "org.apache.hadoop.fs.s3a.audit.AuditSpanS3A"); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java index 1815285738b0e..4754427a4b118 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java @@ -21,7 +21,7 @@ import javax.annotation.Nullable; import java.net.URI; -import com.amazonaws.auth.AWSCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import org.apache.hadoop.conf.Configuration; @@ -29,12 +29,9 @@ * Base class for AWS credential providers which * take a URI and config in their constructor. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ -@Deprecated public abstract class AbstractAWSCredentialProvider - implements AWSCredentialsProvider { + implements AwsCredentialsProvider { private final URI binding; @@ -65,10 +62,4 @@ public URI getUri() { return binding; } - /** - * Refresh is a no-op by default. - */ - @Override - public void refresh() { - } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java index 5b1829e096123..c88a0128f8ec5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java @@ -23,29 +23,26 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; -import com.amazonaws.SdkBaseException; -import com.amazonaws.auth.AWSCredentials; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.auth.credentials.AwsCredentials; import org.apache.hadoop.classification.VisibleForTesting; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.CredentialInitializationException; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; + /** * Base class for session credential support. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private -@Deprecated public abstract class AbstractSessionCredentialsProvider extends AbstractAWSCredentialProvider { /** Credentials, created in {@link #init()}. */ - private volatile AWSCredentials awsCredentials; + private volatile AwsCredentials awsCredentials; /** Atomic flag for on-demand initialization. */ private final AtomicBoolean initialized = new AtomicBoolean(false); @@ -105,7 +102,7 @@ public boolean isInitialized() { * @return the credentials * @throws IOException on any failure. */ - protected abstract AWSCredentials createCredentials(Configuration config) + protected abstract AwsCredentials createCredentials(Configuration config) throws IOException; /** @@ -115,10 +112,10 @@ protected abstract AWSCredentials createCredentials(Configuration config) * is thrown here before any attempt to return the credentials * is made. * @return credentials, if set. - * @throws SdkBaseException if one was raised during init + * @throws SdkException if one was raised during init * @throws CredentialInitializationException on other failures. */ - public AWSCredentials getCredentials() throws SdkBaseException { + public AwsCredentials resolveCredentials() throws SdkException { // do an on-demand init then raise an AWS SDK exception if // there was a failure. try { @@ -126,8 +123,8 @@ public AWSCredentials getCredentials() throws SdkBaseException { init(); } } catch (IOException e) { - if (e.getCause() instanceof SdkBaseException) { - throw (SdkBaseException) e.getCause(); + if (e.getCause() instanceof SdkException) { + throw (SdkException) e.getCause(); } else { throw new CredentialInitializationException(e.getMessage(), e); } @@ -165,15 +162,16 @@ public IOException getInitializationException() { * This will be interpreted as "this provider has no credentials to offer", * rather than an explicit error or anonymous access. */ - protected static final class NoCredentials implements AWSCredentials { + protected static final class NoCredentials implements AwsCredentials { @Override - public String getAWSAccessKeyId() { + public String accessKeyId() { return null; } @Override - public String getAWSSecretKey() { + public String secretAccessKey() { return null; } } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index 1e2ac16075aeb..c2ac8fe4c8197 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -26,21 +26,23 @@ import java.util.Locale; import java.util.concurrent.TimeUnit; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; -import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; -import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.util.Sets; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.StsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.CredentialInitializationException; import org.apache.hadoop.fs.s3a.Retries; @@ -49,9 +51,10 @@ import org.apache.hadoop.fs.s3a.S3ARetryPolicy; import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Sets; import static org.apache.hadoop.fs.s3a.Constants.*; -import static org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList; /** * Support IAM Assumed roles by instantiating an instance of @@ -61,13 +64,10 @@ * * Classname is used in configuration files; do not move. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving -@Deprecated -public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, +public final class AssumedRoleCredentialProvider implements AwsCredentialsProvider, Closeable { private static final Logger LOG = @@ -78,7 +78,7 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, public static final String E_NO_ROLE = "Unset property " + ASSUMED_ROLE_ARN; - private final STSAssumeRoleSessionCredentialsProvider stsProvider; + private final StsAssumeRoleCredentialsProvider stsProvider; private final String sessionName; @@ -90,22 +90,24 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, private final Invoker invoker; + private final StsClient stsClient; + /** * Instantiate. - * This calls {@link #getCredentials()} to fail fast on the inner + * This calls {@link #resolveCredentials()} to fail fast on the inner * role credential retrieval. * @param fsUri possibly null URI of the filesystem. * @param conf configuration * @throws IOException on IO problems and some parameter checking * @throws IllegalArgumentException invalid parameters - * @throws AWSSecurityTokenServiceException problems getting credentials + * @throws StsException problems getting credentials */ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) throws IOException { arn = conf.getTrimmed(ASSUMED_ROLE_ARN, ""); if (StringUtils.isEmpty(arn)) { - throw new IOException(E_NO_ROLE); + throw new PathIOException(String.valueOf(fsUri), E_NO_ROLE); } // build up the base provider @@ -114,8 +116,8 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) Arrays.asList( SimpleAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class), - Sets.newHashSet(this.getClass())); - LOG.debug("Credentials to obtain role credentials: {}", credentialsToSTS); + Sets.newHashSet(getClass())); + LOG.debug("Credentials used to obtain role credentials: {}", credentialsToSTS); // then the STS binding sessionName = conf.getTrimmed(ASSUMED_ROLE_SESSION_NAME, @@ -125,29 +127,31 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) String policy = conf.getTrimmed(ASSUMED_ROLE_POLICY, ""); LOG.debug("{}", this); - STSAssumeRoleSessionCredentialsProvider.Builder builder - = new STSAssumeRoleSessionCredentialsProvider.Builder(arn, sessionName); - builder.withRoleSessionDurationSeconds((int) duration); + + AssumeRoleRequest.Builder requestBuilder = + AssumeRoleRequest.builder().roleArn(arn).roleSessionName(sessionName) + .durationSeconds((int) duration); + if (StringUtils.isNotEmpty(policy)) { LOG.debug("Scope down policy {}", policy); - builder.withScopeDownPolicy(policy); + requestBuilder.policy(policy); } + String endpoint = conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT, ""); String region = conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT_REGION, ASSUMED_ROLE_STS_ENDPOINT_REGION_DEFAULT); - AWSSecurityTokenServiceClientBuilder stsbuilder = + stsClient = STSClientFactory.builder( conf, fsUri != null ? fsUri.getHost() : "", credentialsToSTS, endpoint, - region); - // the STS client is not tracked for a shutdown in close(), because it - // (currently) throws an UnsupportedOperationException in shutdown(). - builder.withStsClient(stsbuilder.build()); + region).build(); //now build the provider - stsProvider = builder.build(); + stsProvider = StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(requestBuilder.build()) + .stsClient(stsClient).build(); // to handle STS throttling by the AWS account, we // need to retry @@ -155,21 +159,21 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) // and force in a fail-fast check just to keep the stack traces less // convoluted - getCredentials(); + resolveCredentials(); } /** * Get credentials. * @return the credentials - * @throws AWSSecurityTokenServiceException if none could be obtained. + * @throws StsException if none could be obtained. */ @Override @Retries.RetryRaw - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { try { - return invoker.retryUntranslated("getCredentials", + return invoker.retryUntranslated("resolveCredentials", true, - stsProvider::getCredentials); + stsProvider::resolveCredentials); } catch (IOException e) { // this is in the signature of retryUntranslated; // its hard to see how this could be raised, but for @@ -178,35 +182,28 @@ public AWSCredentials getCredentials() { throw new CredentialInitializationException( "getCredentials failed: " + e, e); - } catch (AWSSecurityTokenServiceException e) { - LOG.error("Failed to get credentials for role {}", + } catch (SdkClientException e) { + LOG.error("Failed to resolve credentials for role {}", arn, e); throw e; } } - @Override - public void refresh() { - stsProvider.refresh(); - } - /** * Propagate the close() call to the inner stsProvider. */ @Override public void close() { - S3AUtils.closeAutocloseables(LOG, stsProvider, credentialsToSTS); + S3AUtils.closeAutocloseables(LOG, stsProvider, credentialsToSTS, stsClient); } @Override public String toString() { - final StringBuilder sb = new StringBuilder( - "AssumedRoleCredentialProvider{"); - sb.append("role='").append(arn).append('\''); - sb.append(", session'").append(sessionName).append('\''); - sb.append(", duration=").append(duration); - sb.append('}'); - return sb.toString(); + String sb = "AssumedRoleCredentialProvider{" + "role='" + arn + '\'' + + ", session'" + sessionName + '\'' + + ", duration=" + duration + + '}'; + return sb; } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java new file mode 100644 index 0000000000000..b106777dd29cc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.auth; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; +import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.fs.s3a.S3AUtils; +import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; +import org.apache.hadoop.fs.s3native.S3xLoginHelper; +import org.apache.hadoop.fs.store.LogExactlyOnce; + +import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; +import static org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport.isAwsV1SdkAvailable; + +/** + * This class provides methods to create a {@link AWSCredentialProviderList} + * list of AWS credential providers. + */ +public final class CredentialProviderListFactory { + + private static final Logger LOG = LoggerFactory.getLogger(CredentialProviderListFactory.class); + + /** + * A v1 entry has been remapped. warn once about this and then shut up. + */ + private static final LogExactlyOnce LOG_REMAPPED_ENTRY = new LogExactlyOnce(LOG); + + /** + * Error message when the AWS provider list built up contains a forbidden + * entry. + */ + @VisibleForTesting + public static final String E_FORBIDDEN_AWS_PROVIDER + = "AWS provider class cannot be used"; + + /** + * The standard AWS provider list for AWS connections. + */ + public static final List> + STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( + Arrays.asList( + EnvironmentVariableCredentialsProvider.class, + IAMInstanceCredentialsProvider.class, + SimpleAWSCredentialsProvider.class, + TemporaryAWSCredentialsProvider.class)); + + /** V1 credential provider: {@value}. */ + public static final String ANONYMOUS_CREDENTIALS_V1 = + "com.amazonaws.auth.AnonymousAWSCredentials"; + + /** V1 credential provider: {@value}. */ + public static final String EC2_CONTAINER_CREDENTIALS_V1 = + "com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper"; + + /** V1 credential provider: {@value}. */ + public static final String EC2_IAM_CREDENTIALS_V1 = + "com.amazonaws.auth.InstanceProfileCredentialsProvider"; + + /** V2 EC2 instance/container credential provider. */ + public static final String EC2_IAM_CREDENTIALS_V2 = + IAMInstanceCredentialsProvider.class.getName(); + + /** V1 env var credential provider: {@value}. */ + public static final String ENVIRONMENT_CREDENTIALS_V1 = + "com.amazonaws.auth.EnvironmentVariableCredentialsProvider"; + + /** V2 environment variables credential provider. */ + public static final String ENVIRONMENT_CREDENTIALS_V2 = + EnvironmentVariableCredentialsProvider.class.getName(); + + /** V1 profile credential provider: {@value}. */ + public static final String PROFILE_CREDENTIALS_V1 = + "com.amazonaws.auth.profile.ProfileCredentialsProvider"; + + /** V2 environment variables credential provider. */ + public static final String PROFILE_CREDENTIALS_V2 = + ProfileCredentialsProvider.class.getName(); + + /** + * Private map of v1 to v2 credential provider name mapping. + */ + private static final Map V1_V2_CREDENTIAL_PROVIDER_MAP = + initCredentialProvidersMap(); + + private CredentialProviderListFactory() { + } + + /** + * Create the AWS credentials from the providers, the URI and + * the key {@link Constants#AWS_CREDENTIALS_PROVIDER} in the configuration. + * @param binding Binding URI -may be null + * @param conf filesystem configuration + * @return a credentials provider list + * @throws IOException Problems loading the providers (including reading + * secrets from credential files). + */ + public static AWSCredentialProviderList createAWSCredentialProviderList( + @Nullable URI binding, + Configuration conf) throws IOException { + // this will reject any user:secret entries in the URI + S3xLoginHelper.rejectSecretsInURIs(binding); + AWSCredentialProviderList credentials = + buildAWSProviderList(binding, + conf, + AWS_CREDENTIALS_PROVIDER, + STANDARD_AWS_PROVIDERS, + new HashSet<>()); + // make sure the logging message strips out any auth details + LOG.debug("For URI {}, using credentials {}", + binding, credentials); + return credentials; + } + + /** + * Load list of AWS credential provider/credential provider factory classes. + * @param conf configuration + * @param key key + * @param defaultValue list of default values + * @return the list of classes, empty if the default list is empty and + * there was no match for the key in the configuration. + * @throws IOException on a failure to load the list. + */ + private static Collection loadAWSProviderClasses(Configuration conf, + String key, + Class... defaultValue) throws IOException { + final Collection classnames = conf.getTrimmedStringCollection(key); + if (classnames.isEmpty()) { + // empty list; return the defaults + return Arrays.stream(defaultValue).map(c -> c.getName()).collect(Collectors.toList()); + } else { + return classnames; + } + } + + /** + * Maps V1 credential providers to either their equivalent SDK V2 class or hadoop provider. + */ + private static Map initCredentialProvidersMap() { + Map v1v2CredentialProviderMap = new HashMap<>(); + + v1v2CredentialProviderMap.put(ANONYMOUS_CREDENTIALS_V1, + AnonymousAWSCredentialsProvider.NAME); + v1v2CredentialProviderMap.put(EC2_CONTAINER_CREDENTIALS_V1, + EC2_IAM_CREDENTIALS_V2); + v1v2CredentialProviderMap.put(EC2_IAM_CREDENTIALS_V1, + EC2_IAM_CREDENTIALS_V2); + v1v2CredentialProviderMap.put(ENVIRONMENT_CREDENTIALS_V1, + ENVIRONMENT_CREDENTIALS_V2); + v1v2CredentialProviderMap.put(PROFILE_CREDENTIALS_V1, + PROFILE_CREDENTIALS_V2); + + return v1v2CredentialProviderMap; + } + + /** + * Load list of AWS credential provider/credential provider factory classes; + * support a forbidden list to prevent loops, mandate full secrets, etc. + * @param binding Binding URI -may be null + * @param conf configuration + * @param key configuration key to use + * @param forbidden a possibly empty set of forbidden classes. + * @param defaultValues list of default providers. + * @return the list of classes, possibly empty + * @throws IOException on a failure to load the list. + */ + public static AWSCredentialProviderList buildAWSProviderList( + @Nullable final URI binding, + final Configuration conf, + final String key, + final List> defaultValues, + final Set> forbidden) throws IOException { + + // build up the base provider + Collection awsClasses = loadAWSProviderClasses(conf, + key, + defaultValues.toArray(new Class[defaultValues.size()])); + + Map v1v2CredentialProviderMap = V1_V2_CREDENTIAL_PROVIDER_MAP; + final Set forbiddenClassnames = + forbidden.stream().map(c -> c.getName()).collect(Collectors.toSet()); + + + // iterate through, checking for forbidden values and then instantiating + // each provider + AWSCredentialProviderList providers = new AWSCredentialProviderList(); + for (String className : awsClasses) { + if (v1v2CredentialProviderMap.containsKey(className)) { + // mapping + + final String mapped = v1v2CredentialProviderMap.get(className); + LOG_REMAPPED_ENTRY.warn("Credentials option {} contains AWS v1 SDK entry {}; mapping to {}", + key, className, mapped); + className = mapped; + } + // now scan the forbidden list. doing this after any mappings ensures the v1 names + // are also blocked + if (forbiddenClassnames.contains(className)) { + throw new InstantiationIOException(InstantiationIOException.Kind.Forbidden, + binding, className, key, E_FORBIDDEN_AWS_PROVIDER, null); + } + + AwsCredentialsProvider provider; + try { + provider = createAWSV2CredentialProvider(conf, className, binding, key); + } catch (InstantiationIOException e) { + // failed to create a v2; try to see if it is a v1 + if (e.getKind() == InstantiationIOException.Kind.IsNotImplementation) { + if (isAwsV1SdkAvailable()) { + // try to create v1 + LOG.debug("Failed to create {} as v2 credentials, trying to instantiate as v1", + className); + try { + provider = + AwsV1BindingSupport.createAWSV1CredentialProvider(conf, className, binding, key); + LOG_REMAPPED_ENTRY.warn("Credentials option {} contains AWS v1 SDK entry {}", + key, className); + } catch (InstantiationIOException ex) { + // if it is something other than non-implementation, throw. + // that way, non-impl messages are about v2 not v1 in the error + if (ex.getKind() != InstantiationIOException.Kind.IsNotImplementation) { + throw ex; + } else { + throw e; + } + } + } else { + LOG.warn("Failed to instantiate {} as AWS v2 SDK credential provider;" + + " AWS V1 SDK is not on the classpth so unable to attempt to" + + " instantiate as a v1 provider", className, e); + throw e; + } + } else { + // any other problem + throw e; + + } + LOG.debug("From provider class {} created Aws provider {}", className, provider); + } + providers.add(provider); + } + return providers; + } + + /** + * Create an AWS v2 credential provider from its class by using reflection. + * @param conf configuration + * @param className credential class name + * @param uri URI of the FS + * @param key configuration key to use + * @return the instantiated class + * @throws IOException on any instantiation failure. + * @see S3AUtils#getInstanceFromReflection + */ + private static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf, + String className, + @Nullable URI uri, final String key) throws IOException { + LOG.debug("Credential provider class is {}", className); + return S3AUtils.getInstanceFromReflection(className, conf, uri, AwsCredentialsProvider.class, + "create", key); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java index ca9c518d30048..2e39b275b4a4d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java @@ -21,17 +21,18 @@ import java.io.Closeable; import java.io.IOException; -import com.amazonaws.AmazonClientException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkClientException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; /** * This is an IAM credential provider which wraps - * an {@code EC2ContainerCredentialsProviderWrapper} + * an {@code ContainerCredentialsProvider} * to provide credentials when the S3A connector is instantiated on AWS EC2 * or the AWS container services. *

    @@ -41,17 +42,14 @@ *

    * It is implicitly public; marked evolving as we can change its semantics. * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving -@Deprecated public class IAMInstanceCredentialsProvider - implements AWSCredentialsProvider, Closeable { + implements AwsCredentialsProvider, Closeable { - private final AWSCredentialsProvider provider = - new EC2ContainerCredentialsProviderWrapper(); + private final AwsCredentialsProvider containerCredentialsProvider = + ContainerCredentialsProvider.builder().build(); public IAMInstanceCredentialsProvider() { } @@ -63,23 +61,40 @@ public IAMInstanceCredentialsProvider() { * @throws NoAwsCredentialsException on auth failure to indicate non-recoverable. */ @Override - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { try { - return provider.getCredentials(); - } catch (AmazonClientException e) { + return getCredentials(); + } catch (SdkClientException e) { throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider", e.getMessage(), e); } } - @Override - public void refresh() { - provider.refresh(); + /** + * First try {@link ContainerCredentialsProvider}, which will throw an exception if credentials + * cannot be retrieved from the container. Then resolve credentials + * using {@link InstanceProfileCredentialsProvider}. + * + * @return credentials + */ + private AwsCredentials getCredentials() { + try { + return containerCredentialsProvider.resolveCredentials(); + } catch (SdkClientException e) { + return InstanceProfileCredentialsProvider.create().resolveCredentials(); + } } @Override public void close() throws IOException { // no-op. } + + @Override + public String toString() { + return "IAMInstanceCredentialsProvider{" + + "containerCredentialsProvider=" + containerCredentialsProvider + + '}'; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java index 29e815560a8a9..a84318891e9fa 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java @@ -24,19 +24,18 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.SdkClientException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSSessionCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.services.securitytoken.AWSSecurityTokenService; -import com.amazonaws.services.securitytoken.model.Credentials; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.Credentials; import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; @@ -77,10 +76,10 @@ private MarshalledCredentialBinding() { public static MarshalledCredentials fromSTSCredentials( final Credentials credentials) { MarshalledCredentials marshalled = new MarshalledCredentials( - credentials.getAccessKeyId(), - credentials.getSecretAccessKey(), - credentials.getSessionToken()); - Date date = credentials.getExpiration(); + credentials.accessKeyId(), + credentials.secretAccessKey(), + credentials.sessionToken()); + Date date = Date.from(credentials.expiration()); marshalled.setExpiration(date != null ? date.getTime() : 0); return marshalled; } @@ -91,11 +90,11 @@ public static MarshalledCredentials fromSTSCredentials( * @return a set of marshalled credentials. */ public static MarshalledCredentials fromAWSCredentials( - final AWSSessionCredentials credentials) { + final AwsSessionCredentials credentials) { return new MarshalledCredentials( - credentials.getAWSAccessKeyId(), - credentials.getAWSSecretKey(), - credentials.getSessionToken()); + credentials.accessKeyId(), + credentials.secretAccessKey(), + credentials.sessionToken()); } /** @@ -156,7 +155,7 @@ public static MarshalledCredentials fromFileSystem( * @throws NoAuthWithAWSException validation failure * @throws NoAwsCredentialsException the credentials are actually empty. */ - public static AWSCredentials toAWSCredentials( + public static AwsCredentials toAWSCredentials( final MarshalledCredentials marshalled, final MarshalledCredentials.CredentialTypeRequired typeRequired, final String component) @@ -173,46 +172,49 @@ public static AWSCredentials toAWSCredentials( final String secretKey = marshalled.getSecretKey(); if (marshalled.hasSessionToken()) { // a session token was supplied, so return session credentials - return new BasicSessionCredentials(accessKey, secretKey, + return AwsSessionCredentials.create(accessKey, secretKey, marshalled.getSessionToken()); } else { // these are full credentials - return new BasicAWSCredentials(accessKey, secretKey); + return AwsBasicCredentials.create(accessKey, secretKey); } } /** * Request a set of credentials from an STS endpoint. * @param parentCredentials the parent credentials needed to talk to STS - * @param awsConf AWS client configuration + * @param configuration AWS client configuration * @param stsEndpoint an endpoint, use "" for none * @param stsRegion region; use if the endpoint isn't the AWS default. * @param duration duration of the credentials in seconds. Minimum value: 900. * @param invoker invoker to use for retrying the call. + * @param bucket bucket name. * @return the credentials * @throws IOException on a failure of the request */ @Retries.RetryTranslated public static MarshalledCredentials requestSessionCredentials( - final AWSCredentialsProvider parentCredentials, - final ClientConfiguration awsConf, + final AwsCredentialsProvider parentCredentials, + final Configuration configuration, final String stsEndpoint, final String stsRegion, final int duration, - final Invoker invoker) throws IOException { + final Invoker invoker, + final String bucket) throws IOException { try { - final AWSSecurityTokenService tokenService = + final StsClient tokenService = STSClientFactory.builder(parentCredentials, - awsConf, + configuration, stsEndpoint.isEmpty() ? null : stsEndpoint, - stsRegion) + stsRegion, + bucket) .build(); try (STSClientFactory.STSClient stsClient = STSClientFactory.createClientConnection( tokenService, invoker)) { return fromSTSCredentials(stsClient.requestSessionCredentials(duration, TimeUnit.SECONDS)); } - } catch (SdkClientException e) { + } catch (SdkException e) { if (stsRegion.isEmpty()) { LOG.error("Region must be provided when requesting session credentials.", e); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java index 8bd04744cd8c0..4bb5f65e14cb3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.net.URI; -import com.amazonaws.auth.AWSCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -40,7 +40,6 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -@SuppressWarnings("deprecation") public class MarshalledCredentialProvider extends AbstractSessionCredentialsProvider { @@ -85,7 +84,7 @@ public MarshalledCredentialProvider( * @throws IOException on a failure */ @Override - protected AWSCredentials createCredentials(final Configuration config) + protected AwsCredentials createCredentials(final Configuration config) throws IOException { return toAWSCredentials(credentials, typeRequired, component); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java index 7ec13b092c9bc..8f92153b2e1d6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/NoAuthWithAWSException.java @@ -21,7 +21,7 @@ import org.apache.hadoop.fs.s3a.CredentialInitializationException; /** - * A specific subclass of {@code AmazonClientException} which is + * A specific subclass of {@code SdkException} which is * used in the S3A retry policy to fail fast when there is any * authentication problem. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java index 82d4fa588164d..dcfc0a7ee8416 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java @@ -20,31 +20,38 @@ import java.io.Closeable; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.concurrent.TimeUnit; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.securitytoken.AWSSecurityTokenService; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; -import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; -import com.amazonaws.services.securitytoken.model.Credentials; -import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.http.apache.ProxyConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.Credentials; +import software.amazon.awssdk.services.sts.model.GetSessionTokenRequest; +import software.amazon.awssdk.thirdparty.org.apache.http.client.utils.URIBuilder; +import org.apache.hadoop.fs.s3a.impl.AWSClientConfig; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; -import org.apache.hadoop.fs.s3a.S3AUtils; import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotEmpty; +import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_STS; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; /** @@ -71,17 +78,15 @@ public class STSClientFactory { * @return the builder to call {@code build()} * @throws IOException problem reading proxy secrets */ - public static AWSSecurityTokenServiceClientBuilder builder( + public static StsClientBuilder builder( final Configuration conf, final String bucket, - final AWSCredentialsProvider credentials) throws IOException { - final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket, - Constants.AWS_SERVICE_IDENTIFIER_STS); + final AwsCredentialsProvider credentials) throws IOException { String endpoint = conf.getTrimmed(DELEGATION_TOKEN_ENDPOINT, DEFAULT_DELEGATION_TOKEN_ENDPOINT); String region = conf.getTrimmed(DELEGATION_TOKEN_REGION, DEFAULT_DELEGATION_TOKEN_REGION); - return builder(credentials, awsConf, endpoint, region); + return builder(credentials, conf, endpoint, region, bucket); } /** @@ -96,64 +101,89 @@ public static AWSSecurityTokenServiceClientBuilder builder( * @return the builder to call {@code build()} * @throws IOException problem reading proxy secrets */ - public static AWSSecurityTokenServiceClientBuilder builder( + public static StsClientBuilder builder( final Configuration conf, final String bucket, - final AWSCredentialsProvider credentials, + final AwsCredentialsProvider credentials, final String stsEndpoint, final String stsRegion) throws IOException { - final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket, - Constants.AWS_SERVICE_IDENTIFIER_STS); - return builder(credentials, awsConf, stsEndpoint, stsRegion); + return builder(credentials, conf, stsEndpoint, stsRegion, bucket); } /** * Create the builder ready for any final configuration options. * Picks up connection settings from the Hadoop configuration, including * proxy secrets. - * @param awsConf AWS configuration. + * @param conf AWS configuration. * @param credentials AWS credential chain to use * @param stsEndpoint optional endpoint "https://sns.us-west-1.amazonaws.com" * @param stsRegion the region, e.g "us-west-1". Must be set if endpoint is. + * @param bucket bucket name * @return the builder to call {@code build()} + * @throws IOException problem reading proxy secrets */ - public static AWSSecurityTokenServiceClientBuilder builder( - final AWSCredentialsProvider credentials, - final ClientConfiguration awsConf, - final String stsEndpoint, - final String stsRegion) { - final AWSSecurityTokenServiceClientBuilder builder - = AWSSecurityTokenServiceClientBuilder.standard(); + public static StsClientBuilder builder(final AwsCredentialsProvider credentials, + final Configuration conf, final String stsEndpoint, final String stsRegion, + final String bucket) throws IOException { + final StsClientBuilder stsClientBuilder = StsClient.builder(); + Preconditions.checkArgument(credentials != null, "No credentials"); - builder.withClientConfiguration(awsConf); - builder.withCredentials(credentials); + + final ClientOverrideConfiguration.Builder clientOverrideConfigBuilder = + AWSClientConfig.createClientConfigBuilder(conf, AWS_SERVICE_IDENTIFIER_STS); + + final ApacheHttpClient.Builder httpClientBuilder = + AWSClientConfig.createHttpClientBuilder(conf); + + final RetryPolicy.Builder retryPolicyBuilder = AWSClientConfig.createRetryPolicyBuilder(conf); + + final ProxyConfiguration proxyConfig = AWSClientConfig.createProxyConfiguration(conf, bucket); + + clientOverrideConfigBuilder.retryPolicy(retryPolicyBuilder.build()); + httpClientBuilder.proxyConfiguration(proxyConfig); + + stsClientBuilder.httpClientBuilder(httpClientBuilder) + .overrideConfiguration(clientOverrideConfigBuilder.build()) + .credentialsProvider(credentials); + boolean destIsStandardEndpoint = STS_STANDARD.equals(stsEndpoint); if (isNotEmpty(stsEndpoint) && !destIsStandardEndpoint) { - Preconditions.checkArgument( - isNotEmpty(stsRegion), - "STS endpoint is set to %s but no signing region was provided", - stsEndpoint); + Preconditions.checkArgument(isNotEmpty(stsRegion), + "STS endpoint is set to %s but no signing region was provided", stsEndpoint); LOG.debug("STS Endpoint={}; region='{}'", stsEndpoint, stsRegion); - builder.withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(stsEndpoint, stsRegion)); + stsClientBuilder.endpointOverride(getSTSEndpoint(stsEndpoint)).region(Region.of(stsRegion)); } else { Preconditions.checkArgument(isEmpty(stsRegion), - "STS signing region set set to %s but no STS endpoint specified", - stsRegion); + "STS signing region set set to %s but no STS endpoint specified", stsRegion); + } + return stsClientBuilder; + } + + /** + * Given a endpoint string, create the endpoint URI. + * + * @param endpoint possibly null endpoint. + * @return an endpoint uri + */ + private static URI getSTSEndpoint(String endpoint) { + try { + return new URIBuilder().setScheme("https").setHost(endpoint).build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); } - return builder; } + /** * Create an STS Client instance. - * @param tokenService STS instance + * @param stsClient STS instance * @param invoker invoker to use * @return an STS client bonded to that interface. */ public static STSClient createClientConnection( - final AWSSecurityTokenService tokenService, + final StsClient stsClient, final Invoker invoker) { - return new STSClient(tokenService, invoker); + return new STSClient(stsClient, invoker); } /** @@ -161,21 +191,19 @@ public static STSClient createClientConnection( */ public static final class STSClient implements Closeable { - private final AWSSecurityTokenService tokenService; + private final StsClient stsClient; private final Invoker invoker; - private STSClient(final AWSSecurityTokenService tokenService, + private STSClient(final StsClient stsClient, final Invoker invoker) { - this.tokenService = tokenService; + this.stsClient = stsClient; this.invoker = invoker; } @Override public void close() throws IOException { - // Since we are not using AbstractAWSSecurityTokenService, we - // don't need to worry about catching UnsupportedOperationException. - tokenService.shutdown(); + stsClient.close(); } /** @@ -192,13 +220,13 @@ public Credentials requestSessionCredentials( final TimeUnit timeUnit) throws IOException { int durationSeconds = (int) timeUnit.toSeconds(duration); LOG.debug("Requesting session token of duration {}", duration); - final GetSessionTokenRequest request = new GetSessionTokenRequest(); - request.setDurationSeconds(durationSeconds); + final GetSessionTokenRequest request = + GetSessionTokenRequest.builder().durationSeconds(durationSeconds).build(); return invoker.retry("request session credentials", "", true, () ->{ LOG.info("Requesting Amazon STS Session credentials"); - return tokenService.getSessionToken(request).getCredentials(); + return stsClient.getSessionToken(request).credentials(); }); } @@ -222,15 +250,14 @@ public Credentials requestRole( final TimeUnit timeUnit) throws IOException { LOG.debug("Requesting role {} with duration {}; policy = {}", roleARN, duration, policy); - AssumeRoleRequest request = new AssumeRoleRequest(); - request.setDurationSeconds((int) timeUnit.toSeconds(duration)); - request.setRoleArn(roleARN); - request.setRoleSessionName(sessionName); + AssumeRoleRequest.Builder requestBuilder = + AssumeRoleRequest.builder().durationSeconds((int) timeUnit.toSeconds(duration)) + .roleArn(roleARN).roleSessionName(sessionName); if (isNotEmpty(policy)) { - request.setPolicy(policy); + requestBuilder.policy(policy); } return invoker.retry("request role credentials", "", true, - () -> tokenService.assumeRole(request).getCredentials()); + () -> stsClient.assumeRole(requestBuilder.build()).credentials()); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java new file mode 100644 index 0000000000000..c786086947fac --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.auth; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.signer.Aws4Signer; +import software.amazon.awssdk.auth.signer.Aws4UnsignedPayloadSigner; +import software.amazon.awssdk.auth.signer.AwsS3V4Signer; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.core.signer.Signer; + +import org.apache.hadoop.fs.s3a.S3AUtils; + + +/** + * Signer factory used to register and create signers. + */ +public final class SignerFactory { + + private static final Logger LOG = LoggerFactory.getLogger(SignerFactory.class); + public static final String VERSION_FOUR_SIGNER = "AWS4SignerType"; + public static final String VERSION_FOUR_UNSIGNED_PAYLOAD_SIGNER = "AWS4UnsignedPayloadSignerType"; + public static final String NO_OP_SIGNER = "NoOpSignerType"; + private static final String S3_V4_SIGNER = "AWSS3V4SignerType"; + + private static final Map> SIGNERS + = new ConcurrentHashMap<>(); + + static { + // Register the standard signer types. + SIGNERS.put(VERSION_FOUR_SIGNER, Aws4Signer.class); + SIGNERS.put(VERSION_FOUR_UNSIGNED_PAYLOAD_SIGNER, Aws4UnsignedPayloadSigner.class); + SIGNERS.put(NO_OP_SIGNER, NoOpSigner.class); + SIGNERS.put(S3_V4_SIGNER, AwsS3V4Signer.class); + } + + + private SignerFactory() { + } + + /** + * Register an implementation class for the given signer type. + * + * @param signerType The name of the signer type to register. + * @param signerClass The class implementing the given signature protocol. + */ + public static void registerSigner( + final String signerType, + final Class signerClass) { + + if (signerType == null) { + throw new IllegalArgumentException("signerType cannot be null"); + } + if (signerClass == null) { + throw new IllegalArgumentException("signerClass cannot be null"); + } + + SIGNERS.put(signerType, signerClass); + } + + /** + * Check if the signer has already been registered. + * @param signerType signer to get + */ + public static void verifySignerRegistered(String signerType) { + Class signerClass = SIGNERS.get(signerType); + if (signerClass == null) { + throw new IllegalArgumentException("unknown signer type: " + signerType); + } + } + + + /** + * Create an instance of the given signer. + * + * @param signerType The signer type. + * @param configKey Config key used to configure the signer. + * @return The new signer instance. + * @throws IOException on any problem. + */ + public static Signer createSigner(String signerType, String configKey) throws IOException { + Class signerClass = SIGNERS.get(signerType); + String className = signerClass.getName(); + + LOG.debug("Signer class is {}", className); + + Signer signer = + S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create", + configKey); + + return signer; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index e162428787cc4..198bc66133a95 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -22,14 +22,12 @@ import java.util.LinkedList; import java.util.List; -import com.amazonaws.auth.Signer; -import com.amazonaws.auth.SignerFactory; +import software.amazon.awssdk.core.signer.Signer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; -import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -71,8 +69,6 @@ public void initCustomSigners() { return; } - V2Migration.v1CustomSignerUsed(); - for (String customSigner : customSigners) { String[] parts = customSigner.split(":"); if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) { @@ -119,7 +115,7 @@ public void initCustomSigners() { private static void maybeRegisterSigner(String signerName, String signerClassName, Configuration conf) { try { - SignerFactory.getSignerByTypeAndService(signerName, null); + SignerFactory.verifySignerRegistered(signerName); } catch (IllegalArgumentException e) { // Signer is not registered with the AWS SDK. // Load the class and register the signer. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java index 6526f9a947815..bcd358e2d1672 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java @@ -20,9 +20,6 @@ import java.util.Optional; -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams; -import com.amazonaws.services.s3.model.SSECustomerKey; - import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; /** @@ -35,37 +32,30 @@ public final class EncryptionSecretOperations { private EncryptionSecretOperations() { } - /** - * Create SSE-C client side key encryption options on demand. - * @return an optional key to attach to a request. + /*** + * Gets the SSE-C client side key if present. + * * @param secrets source of the encryption secrets. + * @return an optional key to attach to a request. */ - public static Optional createSSECustomerKey( - final EncryptionSecrets secrets) { - if (secrets.hasEncryptionKey() && - secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_C) { - return Optional.of(new SSECustomerKey(secrets.getEncryptionKey())); + public static Optional getSSECustomerKey(final EncryptionSecrets secrets) { + if (secrets.hasEncryptionKey() && secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_C) { + return Optional.of(secrets.getEncryptionKey()); } else { return Optional.empty(); } } /** - * Create SSE-KMS options for a request, iff the encryption is SSE-KMS. - * @return an optional SSE-KMS param to attach to a request. + * Gets the SSE-KMS key if present, else let S3 use AWS managed key. + * * @param secrets source of the encryption secrets. + * @return an optional key to attach to a request. */ - public static Optional createSSEAwsKeyManagementParams( - final EncryptionSecrets secrets) { - - //Use specified key, otherwise default to default master aws/s3 key by AWS - if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS) { - if (secrets.hasEncryptionKey()) { - return Optional.of(new SSEAwsKeyManagementParams( - secrets.getEncryptionKey())); - } else { - return Optional.of(new SSEAwsKeyManagementParams()); - } + public static Optional getSSEAwsKMSKey(final EncryptionSecrets secrets) { + if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS + && secrets.hasEncryptionKey()) { + return Optional.of(secrets.getEncryptionKey()); } else { return Optional.empty(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java index 9b06031d5866a..e83462b92a086 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java @@ -23,7 +23,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; -import com.amazonaws.services.securitytoken.model.Credentials; +import software.amazon.awssdk.services.sts.model.Credentials; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java index 0d9b2d64b3ec3..f5c9c6267ce10 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java @@ -120,11 +120,6 @@ public class S3ADelegationTokens extends AbstractDTService { */ private AbstractDelegationTokenBinding tokenBinding; - /** - * List of cred providers; unset until {@link #bindToDelegationToken(Token)}. - */ - //private Optional credentialProviders = Optional.empty(); - /** * delegation binding information; unset until {@link #bindToDelegationToken(Token)}. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index 2f0a71767edfb..09a1ab1c46e77 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -26,21 +26,18 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSSessionCredentials; -import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.services.sts.StsClient; import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; -import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.S3ARetryPolicy; -import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; import org.apache.hadoop.fs.s3a.auth.RoleModel; @@ -50,8 +47,8 @@ import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; import static org.apache.hadoop.fs.s3a.Invoker.once; -import static org.apache.hadoop.fs.s3a.S3AUtils.STANDARD_AWS_PROVIDERS; -import static org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.STANDARD_AWS_PROVIDERS; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList; import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromAWSCredentials; import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromSTSCredentials; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; @@ -105,7 +102,8 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding { private boolean hasSessionCreds; /** - * The auth chain for the parent options. + * The parent authentication chain: that used to request + * session/role credentials when deployed unbonded. */ private AWSCredentialProviderList parentAuthChain; @@ -164,12 +162,14 @@ protected void serviceStart() throws Exception { DEFAULT_DELEGATION_TOKEN_REGION); // create the provider set for session credentials. - parentAuthChain = buildAWSProviderList( + final AWSCredentialProviderList chain = buildAWSProviderList( getCanonicalUri(), conf, AWS_CREDENTIALS_PROVIDER, STANDARD_AWS_PROVIDERS, new HashSet<>()); + LOG.debug("Setting parent authentication chain to {}", chain); + setParentAuthChain(chain); } @Override @@ -192,7 +192,7 @@ protected void serviceStop() throws Exception { public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); - return parentAuthChain; + return getParentAuthChain(); } /** @@ -292,23 +292,22 @@ private synchronized Optional maybeInitSTS() // chain. // As no codepath (session propagation, STS creation) will work, // throw this. - final AWSCredentials parentCredentials = once("get credentials", + final AwsCredentials parentCredentials = once("get credentials", "", - () -> parentAuthChain.getCredentials()); - hasSessionCreds = parentCredentials instanceof AWSSessionCredentials; + () -> getParentAuthChain().resolveCredentials()); + hasSessionCreds = parentCredentials instanceof AwsSessionCredentials; if (!hasSessionCreds) { LOG.debug("Creating STS client for {}", getDescription()); invoker = new Invoker(new S3ARetryPolicy(conf), LOG_EVENT); - ClientConfiguration awsConf = - S3AUtils.createAwsConf(conf, uri.getHost(), - Constants.AWS_SERVICE_IDENTIFIER_STS); - AWSSecurityTokenService tokenService = - STSClientFactory.builder(parentAuthChain, - awsConf, + + StsClient tokenService = + STSClientFactory.builder(getParentAuthChain(), + conf, endpoint, - region) + region, + uri.getHost()) .build(); stsClient = Optional.of( STSClientFactory.createClientConnection(tokenService, invoker)); @@ -374,11 +373,11 @@ public SessionTokenIdentifier createTokenIdentifier( + " -duration unknown", getCanonicalUri()); } origin += " " + CREDENTIALS_CONVERTED_TO_DELEGATION_TOKEN; - final AWSCredentials awsCredentials - = parentAuthChain.getCredentials(); - if (awsCredentials instanceof AWSSessionCredentials) { + final AwsCredentials awsCredentials + = getParentAuthChain().resolveCredentials(); + if (awsCredentials instanceof AwsSessionCredentials) { marshalledCredentials = fromAWSCredentials( - (AWSSessionCredentials) awsCredentials); + (AwsSessionCredentials) awsCredentials); } else { throw new DelegationTokenIOException( "AWS Authentication chain is no longer supplying session secrets"); @@ -425,4 +424,16 @@ protected void setTokenIdentifier(Optional tokenIdentifier) { this.tokenIdentifier = tokenIdentifier; } + + /** + * The auth chain for the parent options. + * @return the parent authentication chain. + */ + protected AWSCredentialProviderList getParentAuthChain() { + return parentAuthChain; + } + + protected void setParentAuthChain(AWSCredentialProviderList parentAuthChain) { + this.parentAuthChain = parentAuthChain; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java index e53c690431ee0..09664a6dbdf63 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java @@ -27,10 +27,11 @@ import java.util.List; import java.util.UUID; -import com.amazonaws.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.MultipartUpload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; @@ -975,7 +976,7 @@ protected void abortPendingUploadsInCleanup( .executeWith(commitContext.getOuterSubmitter()) .suppressExceptions(suppressExceptions) .run(u -> commitContext.abortMultipartCommit( - u.getKey(), u.getUploadId())); + u.key(), u.uploadId())); } else { LOG.info("No pending uploads were found"); } @@ -1300,8 +1301,8 @@ protected void warnOnActiveUploads(final Path path) { DateFormat df = DateFormat.getDateTimeInstance(); pending.forEach(u -> LOG.info("[{}] {}", - df.format(u.getInitiated()), - u.getKey())); + df.format(Date.from(u.initiated())), + u.key())); if (shouldAbortUploadsInCleanup()) { LOG.warn("This committer will abort these uploads in job cleanup"); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java index 10440f77e7277..6c3cf3942d527 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.List; -import com.amazonaws.services.s3.model.PartETag; +import software.amazon.awssdk.services.s3.model.CompletedPart; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.statistics.IOStatistics; @@ -76,7 +76,7 @@ public boolean outputImmediatelyVisible() { * @throws IOException I/O problem or validation failure. */ public boolean aboutToComplete(String uploadId, - List parts, + List parts, long bytesWritten, final IOStatistics iostatistics) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java index 77c3fed11fb24..e4541ba4da370 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java @@ -31,9 +31,10 @@ import java.util.List; import java.util.Map; -import com.amazonaws.services.s3.model.PartETag; +import software.amazon.awssdk.services.s3.model.CompletedPart; import com.fasterxml.jackson.annotation.JsonProperty; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -215,13 +216,13 @@ public void touch(long millis) { * @param parts ordered list of etags. * @throws ValidationFailure if the data is invalid */ - public void bindCommitData(List parts) throws ValidationFailure { + public void bindCommitData(List parts) throws ValidationFailure { etags = new ArrayList<>(parts.size()); int counter = 1; - for (PartETag part : parts) { - verify(part.getPartNumber() == counter, - "Expected part number %s but got %s", counter, part.getPartNumber()); - etags.add(part.getETag()); + for (CompletedPart part : parts) { + verify(part.partNumber() == counter, + "Expected part number %s but got %s", counter, part.partNumber()); + etags.add(part.eTag()); counter++; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java index ef56d82978158..d1943fa47773f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -32,13 +34,15 @@ import javax.annotation.Nullable; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -155,9 +159,9 @@ public CommitOperations(S3AFileSystem fs, * @param tagIds list of tags * @return same list, now in numbered tuples */ - public static List toPartEtags(List tagIds) { + public static List toPartEtags(List tagIds) { return IntStream.range(0, tagIds.size()) - .mapToObj(i -> new PartETag(i + 1, tagIds.get(i))) + .mapToObj(i -> CompletedPart.builder().partNumber(i + 1).eTag(tagIds.get(i)).build()) .collect(Collectors.toList()); } @@ -566,26 +570,30 @@ public SinglePendingCommit uploadFileToPendingCommit(File localFile, numParts, length)); } - List parts = new ArrayList<>((int) numParts); + List parts = new ArrayList<>((int) numParts); LOG.debug("File size is {}, number of parts to upload = {}", length, numParts); - for (int partNumber = 1; partNumber <= numParts; partNumber += 1) { - progress.progress(); - long size = Math.min(length - offset, uploadPartSize); - UploadPartRequest part; - part = writeOperations.newUploadPartRequest( - destKey, - uploadId, - partNumber, - (int) size, - null, - localFile, - offset); - part.setLastPart(partNumber == numParts); - UploadPartResult partResult = writeOperations.uploadPart(part, statistics); - offset += uploadPartSize; - parts.add(partResult.getPartETag()); + + // Open the file to upload. + try (InputStream fileStream = Files.newInputStream(localFile.toPath())) { + for (int partNumber = 1; partNumber <= numParts; partNumber += 1) { + progress.progress(); + long size = Math.min(length - offset, uploadPartSize); + UploadPartRequest part = writeOperations.newUploadPartRequestBuilder( + destKey, + uploadId, + partNumber, + size).build(); + // Read from the file input stream at current position. + RequestBody body = RequestBody.fromInputStream(fileStream, size); + UploadPartResponse response = writeOperations.uploadPart(part, body, statistics); + offset += uploadPartSize; + parts.add(CompletedPart.builder() + .partNumber(partNumber) + .eTag(response.eTag()) + .build()); + } } commitData.bindCommitData(parts); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java index 1a5451df801dd..b2e703e1b088d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java @@ -20,19 +20,22 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.Retries; +import org.apache.hadoop.fs.s3a.S3ADataBlocks; import org.apache.hadoop.fs.s3a.WriteOperationHelper; import org.apache.hadoop.fs.s3a.commit.PutTracker; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; @@ -125,7 +128,7 @@ public boolean outputImmediatelyVisible() { */ @Override public boolean aboutToComplete(String uploadId, - List parts, + List parts, long bytesWritten, final IOStatistics iostatistics) throws IOException { @@ -144,10 +147,9 @@ public boolean aboutToComplete(String uploadId, headers.put(X_HEADER_MAGIC_MARKER, Long.toString(bytesWritten)); PutObjectRequest originalDestPut = writer.createPutObjectRequest( originalDestKey, - new ByteArrayInputStream(EMPTY), 0, - new PutObjectOptions(true, null, headers)); - upload(originalDestPut); + new PutObjectOptions(true, null, headers), false); + upload(originalDestPut, new ByteArrayInputStream(EMPTY)); // build the commit summary SinglePendingCommit commitData = new SinglePendingCommit(); @@ -170,22 +172,22 @@ public boolean aboutToComplete(String uploadId, path, pendingPartKey, commitData); PutObjectRequest put = writer.createPutObjectRequest( pendingPartKey, - new ByteArrayInputStream(bytes), - bytes.length, null); - upload(put); + bytes.length, null, false); + upload(put, new ByteArrayInputStream(bytes)); return false; } /** * PUT an object. * @param request the request + * @param inputStream input stream of data to be uploaded * @throws IOException on problems */ @Retries.RetryTranslated - private void upload(PutObjectRequest request) throws IOException { - trackDurationOfInvocation(trackerStatistics, - COMMITTER_MAGIC_MARKER_PUT.getSymbol(), () -> - writer.putObject(request, PutObjectOptions.keepingDirs(), null)); + private void upload(PutObjectRequest request, InputStream inputStream) throws IOException { + trackDurationOfInvocation(trackerStatistics, COMMITTER_MAGIC_MARKER_PUT.getSymbol(), + () -> writer.putObject(request, PutObjectOptions.keepingDirs(), + new S3ADataBlocks.BlockUploadData(inputStream), false, null)); } @Override diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java new file mode 100644 index 0000000000000..12a89d50f6b0d --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSCannedACL.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +/** + * Enum to map AWS SDK V1 Acl values to SDK V2. + */ +public enum AWSCannedACL { + Private("private"), + PublicRead("public-read"), + PublicReadWrite("public-read-write"), + AuthenticatedRead("authenticated-read"), + AwsExecRead("aws-exec-read"), + BucketOwnerRead("bucket-owner-read"), + BucketOwnerFullControl("bucket-owner-full-control"), + LogDeliveryWrite("log-delivery-write"); + + private final String value; + + AWSCannedACL(String value){ + this.value = value; + } + + public String toString() { + return this.value; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java new file mode 100644 index 0000000000000..4ff2ec0b0b9f4 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.http.apache.ProxyConfiguration; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.S3AUtils; +import org.apache.hadoop.fs.s3a.auth.SignerFactory; +import org.apache.hadoop.util.VersionInfo; +import org.apache.http.client.utils.URIBuilder; + +import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3; +import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_STS; +import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_TTL; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_ERROR_RETRIES; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.MAX_ERROR_RETRIES; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_DOMAIN; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_PASSWORD; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_USERNAME; +import static org.apache.hadoop.fs.s3a.Constants.PROXY_WORKSTATION; +import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_S3; +import static org.apache.hadoop.fs.s3a.Constants.SIGNING_ALGORITHM_STS; +import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.USER_AGENT_PREFIX; +import static org.apache.hadoop.fs.s3a.S3AUtils.longOption; + +/** + * Methods for configuring the S3 client. + * These methods are used when creating and configuring + * {@link software.amazon.awssdk.services.s3.S3Client} which communicates with the S3 service. + */ +public final class AWSClientConfig { + private static final Logger LOG = LoggerFactory.getLogger(AWSClientConfig.class); + + private AWSClientConfig() { + } + + public static ClientOverrideConfiguration.Builder createClientConfigBuilder(Configuration conf, + String awsServiceIdentifier) throws IOException { + ClientOverrideConfiguration.Builder overrideConfigBuilder = + ClientOverrideConfiguration.builder(); + + initRequestTimeout(conf, overrideConfigBuilder); + + initUserAgent(conf, overrideConfigBuilder); + + String signer = conf.getTrimmed(SIGNING_ALGORITHM, ""); + if (!signer.isEmpty()) { + LOG.debug("Signer override = {}", signer); + overrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.SIGNER, + SignerFactory.createSigner(signer, SIGNING_ALGORITHM)); + } + + initSigner(conf, overrideConfigBuilder, awsServiceIdentifier); + + return overrideConfigBuilder; + } + + /** + * Configures the http client. + * + * @param conf The Hadoop configuration + * @return Http client builder + * @throws IOException on any problem + */ + public static ApacheHttpClient.Builder createHttpClientBuilder(Configuration conf) + throws IOException { + ApacheHttpClient.Builder httpClientBuilder = + ApacheHttpClient.builder(); + + httpClientBuilder.maxConnections(S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS, + DEFAULT_MAXIMUM_CONNECTIONS, 1)); + + int connectionEstablishTimeout = + S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0); + int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0); + + httpClientBuilder.connectionTimeout(Duration.ofSeconds(connectionEstablishTimeout)); + httpClientBuilder.socketTimeout(Duration.ofSeconds(socketTimeout)); + + // set the connection TTL irrespective of whether the connection is in use or not. + // this can balance requests over different S3 servers, and avoid failed + // connections. See HADOOP-18845. + long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1); + httpClientBuilder.connectionTimeToLive(Duration.ofSeconds(ttl)); + + NetworkBinding.bindSSLChannelMode(conf, httpClientBuilder); + + return httpClientBuilder; + } + + /** + * Configures the async http client. + * + * @param conf The Hadoop configuration + * @return Http client builder + */ + public static NettyNioAsyncHttpClient.Builder createAsyncHttpClientBuilder(Configuration conf) { + NettyNioAsyncHttpClient.Builder httpClientBuilder = + NettyNioAsyncHttpClient.builder(); + + httpClientBuilder.maxConcurrency(S3AUtils.intOption(conf, MAXIMUM_CONNECTIONS, + DEFAULT_MAXIMUM_CONNECTIONS, 1)); + + int connectionEstablishTimeout = + S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0); + int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0); + + httpClientBuilder.connectionTimeout(Duration.ofSeconds(connectionEstablishTimeout)); + httpClientBuilder.readTimeout(Duration.ofSeconds(socketTimeout)); + httpClientBuilder.writeTimeout(Duration.ofSeconds(socketTimeout)); + + // set the connection TTL irrespective of whether the connection is in use or not. + // this can balance requests over different S3 servers, and avoid failed + // connections. See HADOOP-18845. + long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1); + httpClientBuilder.connectionTimeToLive(Duration.ofSeconds(ttl)); + + // TODO: Don't think you can set a socket factory for the netty client. + // NetworkBinding.bindSSLChannelMode(conf, awsConf); + + return httpClientBuilder; + } + + /** + * Configures the retry policy. + * + * @param conf The Hadoop configuration + * @return Retry policy builder + */ + public static RetryPolicy.Builder createRetryPolicyBuilder(Configuration conf) { + + RetryPolicy.Builder retryPolicyBuilder = RetryPolicy.builder(); + + retryPolicyBuilder.numRetries(S3AUtils.intOption(conf, MAX_ERROR_RETRIES, + DEFAULT_MAX_ERROR_RETRIES, 0)); + + return retryPolicyBuilder; + } + + /** + * Configures the proxy. + * + * @param conf The Hadoop configuration + * @param bucket Optional bucket to use to look up per-bucket proxy secrets + * @return Proxy configuration + * @throws IOException on any IO problem + */ + public static ProxyConfiguration createProxyConfiguration(Configuration conf, + String bucket) throws IOException { + + ProxyConfiguration.Builder proxyConfigBuilder = ProxyConfiguration.builder(); + + String proxyHost = conf.getTrimmed(PROXY_HOST, ""); + int proxyPort = conf.getInt(PROXY_PORT, -1); + + if (!proxyHost.isEmpty()) { + if (proxyPort >= 0) { + String scheme = conf.getBoolean(PROXY_SECURED, false) ? "https" : "http"; + proxyConfigBuilder.endpoint(buildURI(scheme, proxyHost, proxyPort)); + } else { + if (conf.getBoolean(PROXY_SECURED, false)) { + LOG.warn("Proxy host set without port. Using HTTPS default 443"); + proxyConfigBuilder.endpoint(buildURI("https", proxyHost, 443)); + } else { + LOG.warn("Proxy host set without port. Using HTTP default 80"); + proxyConfigBuilder.endpoint(buildURI("http", proxyHost, 80)); + } + } + final String proxyUsername = S3AUtils.lookupPassword(bucket, conf, PROXY_USERNAME, + null, null); + final String proxyPassword = S3AUtils.lookupPassword(bucket, conf, PROXY_PASSWORD, + null, null); + if ((proxyUsername == null) != (proxyPassword == null)) { + String msg = "Proxy error: " + PROXY_USERNAME + " or " + + PROXY_PASSWORD + " set without the other."; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + proxyConfigBuilder.username(proxyUsername); + proxyConfigBuilder.password(proxyPassword); + proxyConfigBuilder.ntlmDomain(conf.getTrimmed(PROXY_DOMAIN)); + proxyConfigBuilder.ntlmWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); + if (LOG.isDebugEnabled()) { + LOG.debug("Using proxy server {}:{} as user {} with password {} on " + + "domain {} as workstation {}", proxyHost, proxyPort, proxyUsername, proxyPassword, + PROXY_DOMAIN, PROXY_WORKSTATION); + } + } else if (proxyPort >= 0) { + String msg = + "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + + return proxyConfigBuilder.build(); + } + + /** + * Configures the proxy for the async http client. + * + * @param conf The Hadoop configuration + * @param bucket Optional bucket to use to look up per-bucket proxy secrets + * @return Proxy configuration + * @throws IOException on any IO problem + */ + public static software.amazon.awssdk.http.nio.netty.ProxyConfiguration + createAsyncProxyConfiguration(Configuration conf, + String bucket) throws IOException { + + software.amazon.awssdk.http.nio.netty.ProxyConfiguration.Builder proxyConfigBuilder = + software.amazon.awssdk.http.nio.netty.ProxyConfiguration.builder(); + + String proxyHost = conf.getTrimmed(PROXY_HOST, ""); + int proxyPort = conf.getInt(PROXY_PORT, -1); + + if (!proxyHost.isEmpty()) { + if (proxyPort >= 0) { + String scheme = conf.getBoolean(PROXY_SECURED, false) ? "https" : "http"; + proxyConfigBuilder.host(proxyHost); + proxyConfigBuilder.port(proxyPort); + proxyConfigBuilder.scheme(scheme); + } else { + if (conf.getBoolean(PROXY_SECURED, false)) { + LOG.warn("Proxy host set without port. Using HTTPS default 443"); + proxyConfigBuilder.host(proxyHost); + proxyConfigBuilder.port(443); + proxyConfigBuilder.scheme("https"); + } else { + LOG.warn("Proxy host set without port. Using HTTP default 80"); + proxyConfigBuilder.host(proxyHost); + proxyConfigBuilder.port(80); + proxyConfigBuilder.scheme("http"); + } + } + final String proxyUsername = S3AUtils.lookupPassword(bucket, conf, PROXY_USERNAME, + null, null); + final String proxyPassword = S3AUtils.lookupPassword(bucket, conf, PROXY_PASSWORD, + null, null); + if ((proxyUsername == null) != (proxyPassword == null)) { + String msg = "Proxy error: " + PROXY_USERNAME + " or " + + PROXY_PASSWORD + " set without the other."; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + proxyConfigBuilder.username(proxyUsername); + proxyConfigBuilder.password(proxyPassword); + // TODO: check NTLM support + // proxyConfigBuilder.ntlmDomain(conf.getTrimmed(PROXY_DOMAIN)); + // proxyConfigBuilder.ntlmWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); + if (LOG.isDebugEnabled()) { + LOG.debug("Using proxy server {}:{} as user {} with password {} on " + + "domain {} as workstation {}", proxyHost, proxyPort, proxyUsername, proxyPassword, + PROXY_DOMAIN, PROXY_WORKSTATION); + } + } else if (proxyPort >= 0) { + String msg = + "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } else { + return null; + } + + return proxyConfigBuilder.build(); + } + + /*** + * Builds a URI, throws an IllegalArgumentException in case of errors. + * + * @param host proxy host + * @param port proxy port + * @return uri with host and port + */ + private static URI buildURI(String scheme, String host, int port) { + try { + return new URIBuilder().setScheme(scheme).setHost(host).setPort(port).build(); + } catch (URISyntaxException e) { + String msg = + "Proxy error: incorrect " + PROXY_HOST + " or " + PROXY_PORT; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + } + + /** + * Initializes the User-Agent header to send in HTTP requests to AWS + * services. We always include the Hadoop version number. The user also + * may set an optional custom prefix to put in front of the Hadoop version + * number. The AWS SDK internally appends its own information, which seems + * to include the AWS SDK version, OS and JVM version. + * + * @param conf Hadoop configuration + * @param clientConfig AWS SDK configuration to update + */ + private static void initUserAgent(Configuration conf, + ClientOverrideConfiguration.Builder clientConfig) { + String userAgent = "Hadoop " + VersionInfo.getVersion(); + String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, ""); + if (!userAgentPrefix.isEmpty()) { + userAgent = userAgentPrefix + ", " + userAgent; + } + LOG.debug("Using User-Agent: {}", userAgent); + clientConfig.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX, userAgent); + } + + private static void initSigner(Configuration conf, + ClientOverrideConfiguration.Builder clientConfig, String awsServiceIdentifier) + throws IOException { + String configKey = null; + switch (awsServiceIdentifier) { + case AWS_SERVICE_IDENTIFIER_S3: + configKey = SIGNING_ALGORITHM_S3; + break; + case AWS_SERVICE_IDENTIFIER_STS: + configKey = SIGNING_ALGORITHM_STS; + break; + default: + // Nothing to do. The original signer override is already setup + } + if (configKey != null) { + String signerOverride = conf.getTrimmed(configKey, ""); + if (!signerOverride.isEmpty()) { + LOG.debug("Signer override for {} = {}", awsServiceIdentifier, signerOverride); + clientConfig.putAdvancedOption(SdkAdvancedClientOption.SIGNER, + SignerFactory.createSigner(signerOverride, configKey)); + } + } + } + + /** + * Configures request timeout. + * + * @param conf Hadoop configuration + * @param clientConfig AWS SDK configuration to update + */ + private static void initRequestTimeout(Configuration conf, + ClientOverrideConfiguration.Builder clientConfig) { + long requestTimeoutMillis = conf.getTimeDuration(REQUEST_TIMEOUT, + DEFAULT_REQUEST_TIMEOUT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS); + + if (requestTimeoutMillis > Integer.MAX_VALUE) { + LOG.debug("Request timeout is too high({} ms). Setting to {} ms instead", + requestTimeoutMillis, Integer.MAX_VALUE); + requestTimeoutMillis = Integer.MAX_VALUE; + } + + if(requestTimeoutMillis > 0) { + clientConfig.apiCallAttemptTimeout(Duration.ofMillis(requestTimeoutMillis)); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java new file mode 100644 index 0000000000000..e0d6fa5aecc0b --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +/** + * Common S3 HTTP header values used throughout the Amazon Web Services S3 Java client. + */ +public interface AWSHeaders { + + /* + * Standard HTTP Headers. + */ + + String CACHE_CONTROL = "Cache-Control"; + String CONTENT_DISPOSITION = "Content-Disposition"; + String CONTENT_ENCODING = "Content-Encoding"; + String CONTENT_LENGTH = "Content-Length"; + String CONTENT_RANGE = "Content-Range"; + String CONTENT_MD5 = "Content-MD5"; + String CONTENT_TYPE = "Content-Type"; + String CONTENT_LANGUAGE = "Content-Language"; + String DATE = "Date"; + String ETAG = "ETag"; + String LAST_MODIFIED = "Last-Modified"; + + /* + * Amazon HTTP Headers used by S3A. + */ + + /** S3's version ID header. */ + String S3_VERSION_ID = "x-amz-version-id"; + + /** Header describing what class of storage a user wants. */ + String STORAGE_CLASS = "x-amz-storage-class"; + + /** Header describing what archive tier the object is in, if any. */ + String ARCHIVE_STATUS = "x-amz-archive-status"; + + /** Header for optional server-side encryption algorithm. */ + String SERVER_SIDE_ENCRYPTION = "x-amz-server-side-encryption"; + + /** Range header for the get object request. */ + String RANGE = "Range"; + + /** + * Encrypted symmetric key header that is used in the Encryption Only (EO) envelope + * encryption mechanism. + */ + @Deprecated + String CRYPTO_KEY = "x-amz-key"; + + /** JSON-encoded description of encryption materials used during encryption. */ + String MATERIALS_DESCRIPTION = "x-amz-matdesc"; + + /** Header for the optional restore information of an object. */ + String RESTORE = "x-amz-restore"; + + /** + * Key wrapping algorithm such as "AESWrap" and "RSA/ECB/OAEPWithSHA-256AndMGF1Padding". + */ + String CRYPTO_KEYWRAP_ALGORITHM = "x-amz-wrap-alg"; + /** + * Content encryption algorithm, such as "AES/GCM/NoPadding". + */ + String CRYPTO_CEK_ALGORITHM = "x-amz-cek-alg"; + + /** + * Headers in request indicating that the requester must be charged for data + * transfer. + */ + String REQUESTER_PAYS_HEADER = "x-amz-request-payer"; + + /** Header for the replication status of an Amazon S3 Object.*/ + String OBJECT_REPLICATION_STATUS = "x-amz-replication-status"; + + String OBJECT_LOCK_MODE = "x-amz-object-lock-mode"; + + String OBJECT_LOCK_RETAIN_UNTIL_DATE = "x-amz-object-lock-retain-until-date"; + + String OBJECT_LOCK_LEGAL_HOLD_STATUS = "x-amz-object-lock-legal-hold"; + +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java index 4169a9899cb15..5808607762ea6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteRetryHandler.java @@ -20,8 +20,9 @@ import java.util.List; -import com.amazonaws.SdkClientException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,7 @@ import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; + import static org.apache.hadoop.fs.s3a.S3AUtils.isThrottleException; import static org.apache.hadoop.fs.s3a.Statistic.IGNORED_ERRORS; import static org.apache.hadoop.fs.s3a.Statistic.STORE_IO_THROTTLED; @@ -113,15 +115,15 @@ public void bulkDeleteRetried( * @param deleteRequest request which failed. */ private void onDeleteThrottled(final DeleteObjectsRequest deleteRequest) { - final List keys = deleteRequest.getKeys(); + final List keys = deleteRequest.delete().objects(); final int size = keys.size(); incrementStatistic(STORE_IO_THROTTLED, size); instrumentation.addValueToQuantiles(STORE_IO_THROTTLE_RATE, size); THROTTLE_LOG.info( "Bulk delete {} keys throttled -first key = {}; last = {}", size, - keys.get(0).getKey(), - keys.get(size - 1).getKey()); + keys.get(0).key(), + keys.get(size - 1).key()); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java index dcb538dc668de..25f7c4e9c1a74 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeDetectionPolicy.java @@ -20,11 +20,12 @@ import java.util.Locale; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.transfer.model.CopyResult; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -211,11 +212,24 @@ public String toString() { * null if the attribute is unavailable (such as when the policy says to use * versionId but object versioning is not enabled for the bucket). */ - public abstract String getRevisionId(ObjectMetadata objectMetadata, + public abstract String getRevisionId(HeadObjectResponse objectMetadata, String uri); /** - * Like {{@link #getRevisionId(ObjectMetadata, String)}}, but retrieves the + * Like {{@link #getRevisionId(HeadObjectResponse, String)}}, but retrieves the + * * revision identifier from {@link GetObjectResponse}. + * + * @param getObjectResponse the response instance + * @param uri the URI of the object + * @return the revisionId string as interpreted by this policy, or potentially + * null if the attribute is unavailable (such as when the policy says to use + * versionId but object versioning is not enabled for the bucket). + */ + public abstract String getRevisionId(GetObjectResponse getObjectResponse, + String uri); + + /** + * Like {{@link #getRevisionId(HeadObjectResponse, String)}}, but retrieves the * revision identifier from {@link S3ObjectAttributes}. * * @param s3Attributes the object attributes @@ -226,44 +240,44 @@ public abstract String getRevisionId(ObjectMetadata objectMetadata, public abstract String getRevisionId(S3ObjectAttributes s3Attributes); /** - * Like {{@link #getRevisionId(ObjectMetadata, String)}}, but retrieves the - * revision identifier from {@link CopyResult}. + * Like {{@link #getRevisionId(HeadObjectResponse, String)}}, but retrieves the + * revision identifier from {@link CopyObjectResponse}. * - * @param copyResult the copy result + * @param copyObjectResponse the copy response * @return the revisionId string as interpreted by this policy, or potentially * null if the attribute is unavailable (such as when the policy says to use * versionId but object versioning is not enabled for the bucket). */ - public abstract String getRevisionId(CopyResult copyResult); + public abstract String getRevisionId(CopyObjectResponse copyObjectResponse); /** - * Applies the given {@link #getRevisionId(ObjectMetadata, String) revisionId} - * as a server-side qualification on the {@code GetObjectRequest}. + * Applies the given {@link #getRevisionId(HeadObjectResponse, String) revisionId} + * as a server-side qualification on the {@code GetObjectRequest.Builder}. * * @param request the request * @param revisionId the revision id */ - public abstract void applyRevisionConstraint(GetObjectRequest request, + public abstract void applyRevisionConstraint(GetObjectRequest.Builder request, String revisionId); /** - * Applies the given {@link #getRevisionId(ObjectMetadata, String) revisionId} + * Applies the given {@link #getRevisionId(HeadObjectResponse, String) revisionId} * as a server-side qualification on the {@code CopyObjectRequest}. * - * @param request the request + * @param requestBuilder the copy object request builder * @param revisionId the revision id */ - public abstract void applyRevisionConstraint(CopyObjectRequest request, + public abstract void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder, String revisionId); /** - * Applies the given {@link #getRevisionId(ObjectMetadata, String) revisionId} + * Applies the given {@link #getRevisionId(HeadObjectResponse, String) revisionId} * as a server-side qualification on the {@code GetObjectMetadataRequest}. * - * @param request the request + * @param requestBuilder the head object request builder * @param revisionId the revision id */ - public abstract void applyRevisionConstraint(GetObjectMetadataRequest request, + public abstract void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder, String revisionId); /** @@ -323,7 +337,7 @@ public ImmutablePair onChangeDetected( } /** - * Change detection policy based on {@link ObjectMetadata#getETag() eTag}. + * Change detection policy based on {@link HeadObjectResponse#eTag()} () eTag}. */ static class ETagChangeDetectionPolicy extends ChangeDetectionPolicy { @@ -332,8 +346,13 @@ static class ETagChangeDetectionPolicy extends ChangeDetectionPolicy { } @Override - public String getRevisionId(ObjectMetadata objectMetadata, String uri) { - return objectMetadata.getETag(); + public String getRevisionId(GetObjectResponse objectMetadata, String uri) { + return objectMetadata.eTag(); + } + + @Override + public String getRevisionId(HeadObjectResponse objectMetadata, String uri) { + return objectMetadata.eTag(); } @Override @@ -342,34 +361,34 @@ public String getRevisionId(S3ObjectAttributes s3Attributes) { } @Override - public String getRevisionId(CopyResult copyResult) { - return copyResult.getETag(); + public String getRevisionId(CopyObjectResponse copyObjectResponse) { + return copyObjectResponse.copyObjectResult().eTag(); } @Override - public void applyRevisionConstraint(GetObjectRequest request, + public void applyRevisionConstraint(GetObjectRequest.Builder builder, String revisionId) { if (revisionId != null) { LOG.debug("Restricting get request to etag {}", revisionId); - request.withMatchingETagConstraint(revisionId); + builder.ifMatch(revisionId); } else { LOG.debug("No etag revision ID to use as a constraint"); } } @Override - public void applyRevisionConstraint(CopyObjectRequest request, + public void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder, String revisionId) { if (revisionId != null) { LOG.debug("Restricting copy request to etag {}", revisionId); - request.withMatchingETagConstraint(revisionId); + requestBuilder.copySourceIfMatch(revisionId); } else { LOG.debug("No etag revision ID to use as a constraint"); } } @Override - public void applyRevisionConstraint(GetObjectMetadataRequest request, + public void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder, String revisionId) { LOG.debug("Unable to restrict HEAD request to etag; will check later"); } @@ -388,7 +407,7 @@ public String toString() { /** * Change detection policy based on - * {@link ObjectMetadata#getVersionId() versionId}. + * {@link HeadObjectResponse#versionId()} () versionId}. */ static class VersionIdChangeDetectionPolicy extends ChangeDetectionPolicy { @@ -398,8 +417,16 @@ static class VersionIdChangeDetectionPolicy extends } @Override - public String getRevisionId(ObjectMetadata objectMetadata, String uri) { - String versionId = objectMetadata.getVersionId(); + public String getRevisionId(HeadObjectResponse objectMetadata, String uri) { + return logIfNull(objectMetadata.versionId(), uri); + } + + @Override + public String getRevisionId(GetObjectResponse getObjectResponse, String uri) { + return logIfNull(getObjectResponse.versionId(), uri); + } + + private String logIfNull(String versionId, String uri) { if (versionId == null) { // this policy doesn't work if the bucket doesn't have object versioning // enabled (which isn't by default) @@ -419,38 +446,38 @@ public String getRevisionId(S3ObjectAttributes s3Attributes) { } @Override - public String getRevisionId(CopyResult copyResult) { - return copyResult.getVersionId(); + public String getRevisionId(CopyObjectResponse copyObjectResponse) { + return copyObjectResponse.versionId(); } @Override - public void applyRevisionConstraint(GetObjectRequest request, + public void applyRevisionConstraint(GetObjectRequest.Builder builder, String revisionId) { if (revisionId != null) { LOG.debug("Restricting get request to version {}", revisionId); - request.withVersionId(revisionId); + builder.versionId(revisionId); } else { LOG.debug("No version ID to use as a constraint"); } } @Override - public void applyRevisionConstraint(CopyObjectRequest request, + public void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder, String revisionId) { if (revisionId != null) { LOG.debug("Restricting copy request to version {}", revisionId); - request.withSourceVersionId(revisionId); + requestBuilder.sourceVersionId(revisionId); } else { LOG.debug("No version ID to use as a constraint"); } } @Override - public void applyRevisionConstraint(GetObjectMetadataRequest request, + public void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder, String revisionId) { if (revisionId != null) { LOG.debug("Restricting metadata request to version {}", revisionId); - request.withVersionId(revisionId); + requestBuilder.versionId(revisionId); } else { LOG.debug("No version ID to use as a constraint"); } @@ -482,7 +509,13 @@ public Source getSource() { } @Override - public String getRevisionId(final ObjectMetadata objectMetadata, + public String getRevisionId(final GetObjectResponse objectMetadata, + final String uri) { + return null; + } + + @Override + public String getRevisionId(final HeadObjectResponse objectMetadata, final String uri) { return null; } @@ -493,24 +526,24 @@ public String getRevisionId(final S3ObjectAttributes s3ObjectAttributes) { } @Override - public String getRevisionId(CopyResult copyResult) { + public String getRevisionId(CopyObjectResponse copyObjectResponse) { return null; } @Override - public void applyRevisionConstraint(final GetObjectRequest request, + public void applyRevisionConstraint(final GetObjectRequest.Builder builder, final String revisionId) { } @Override - public void applyRevisionConstraint(CopyObjectRequest request, + public void applyRevisionConstraint(CopyObjectRequest.Builder requestBuilder, String revisionId) { } @Override - public void applyRevisionConstraint(GetObjectMetadataRequest request, + public void applyRevisionConstraint(HeadObjectRequest.Builder requestBuilder, String revisionId) { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java index e7dd75c581131..2c9d6857b46a2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java @@ -18,14 +18,14 @@ package org.apache.hadoop.fs.s3a.impl; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.transfer.model.CopyResult; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.classification.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.s3a.S3ObjectAttributes; import org.apache.hadoop.fs.s3a.statistics.ChangeTrackerStatistics; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_412_PRECONDITION_FAILED; import static org.apache.hadoop.util.Preconditions.checkNotNull; /** @@ -55,8 +56,6 @@ public class ChangeTracker { private static final Logger LOG = LoggerFactory.getLogger(ChangeTracker.class); - /** {@code 412 Precondition Failed} (HTTP/1.1 - RFC 2616) */ - public static final int SC_PRECONDITION_FAILED = 412; public static final String CHANGE_REPORTED_BY_S3 = "Change reported by S3"; /** Policy to use. */ @@ -117,15 +116,15 @@ public long getVersionMismatches() { /** * Apply any revision control set by the policy if it is to be * enforced on the server. - * @param request request to modify + * @param builder request builder to modify * @return true iff a constraint was added. */ public boolean maybeApplyConstraint( - final GetObjectRequest request) { + final GetObjectRequest.Builder builder) { if (policy.getMode() == ChangeDetectionPolicy.Mode.Server && revisionId != null) { - policy.applyRevisionConstraint(request, revisionId); + policy.applyRevisionConstraint(builder, revisionId); return true; } return false; @@ -134,26 +133,26 @@ public boolean maybeApplyConstraint( /** * Apply any revision control set by the policy if it is to be * enforced on the server. - * @param request request to modify + * @param requestBuilder copy object request builder to modify * @return true iff a constraint was added. */ public boolean maybeApplyConstraint( - final CopyObjectRequest request) { + final CopyObjectRequest.Builder requestBuilder) { if (policy.getMode() == ChangeDetectionPolicy.Mode.Server && revisionId != null) { - policy.applyRevisionConstraint(request, revisionId); + policy.applyRevisionConstraint(requestBuilder, revisionId); return true; } return false; } public boolean maybeApplyConstraint( - final GetObjectMetadataRequest request) { + final HeadObjectRequest.Builder requestBuilder) { if (policy.getMode() == ChangeDetectionPolicy.Mode.Server && revisionId != null) { - policy.applyRevisionConstraint(request, revisionId); + policy.applyRevisionConstraint(requestBuilder, revisionId); return true; } return false; @@ -168,7 +167,7 @@ public boolean maybeApplyConstraint( * @throws PathIOException raised on failure * @throws RemoteFileChangedException if the remote file has changed. */ - public void processResponse(final S3Object object, + public void processResponse(final GetObjectResponse object, final String operation, final long pos) throws PathIOException { if (object == null) { @@ -191,24 +190,24 @@ public void processResponse(final S3Object object, } } - processMetadata(object.getObjectMetadata(), operation); + processMetadata(object, operation); } /** * Process the response from the server for validation against the * change policy. - * @param copyResult result of a copy operation + * @param copyObjectResponse response of a copy operation * @throws PathIOException raised on failure * @throws RemoteFileChangedException if the remote file has changed. */ - public void processResponse(final CopyResult copyResult) + public void processResponse(final CopyObjectResponse copyObjectResponse) throws PathIOException { // ETag (sometimes, depending on encryption and/or multipart) is not the // same on the copied object as the original. Version Id seems to never // be the same on the copy. As such, there isn't really anything that // can be verified on the response, except that a revision ID is present // if required. - String newRevisionId = policy.getRevisionId(copyResult); + String newRevisionId = policy.getRevisionId(copyObjectResponse); LOG.debug("Copy result {}: {}", policy.getSource(), newRevisionId); if (newRevisionId == null && policy.isRequireVersion()) { throw new NoVersionAttributeException(uri, String.format( @@ -227,13 +226,11 @@ public void processResponse(final CopyResult copyResult) * generated (e.g. "copy", "read", "select"). * @throws RemoteFileChangedException if the remote file has changed. */ - public void processException(SdkBaseException e, String operation) throws + public void processException(SdkException e, String operation) throws RemoteFileChangedException { - if (e instanceof AmazonServiceException) { - AmazonServiceException serviceException = (AmazonServiceException) e; - // This isn't really going to be hit due to - // https://github.com/aws/aws-sdk-java/issues/1644 - if (serviceException.getStatusCode() == SC_PRECONDITION_FAILED) { + if (e instanceof AwsServiceException) { + AwsServiceException serviceException = (AwsServiceException)e; + if (serviceException.statusCode() == SC_412_PRECONDITION_FAILED) { versionMismatches.versionMismatchError(); throw new RemoteFileChangedException(uri, operation, String.format( RemoteFileChangedException.PRECONDITIONS_FAILED @@ -254,12 +251,26 @@ public void processException(SdkBaseException e, String operation) throws * @throws PathIOException raised on failure * @throws RemoteFileChangedException if the remote file has changed. */ - public void processMetadata(final ObjectMetadata metadata, + public void processMetadata(final HeadObjectResponse metadata, final String operation) throws PathIOException { final String newRevisionId = policy.getRevisionId(metadata, uri); processNewRevision(newRevisionId, operation, -1); } + /** + * Process the response from server for validation against the change + * policy. + * @param getObjectResponse response returned from server + * @param operation operation in progress + * @throws PathIOException raised on failure + * @throws RemoteFileChangedException if the remote file has changed. + */ + public void processMetadata(final GetObjectResponse getObjectResponse, + final String operation) throws PathIOException { + final String newRevisionId = policy.getRevisionId(getObjectResponse, uri); + processNewRevision(newRevisionId, operation, -1); + } + /** * Validate a revision from the server against our expectations. * @param newRevisionId new revision. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java index 8205668e8f354..ba21f6028a5f8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java @@ -21,8 +21,8 @@ import javax.net.ssl.HostnameVerifier; import java.io.IOException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.thirdparty.org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; @@ -35,13 +35,12 @@ public class ConfigureShadedAWSSocketFactory implements NetworkBinding.ConfigureAWSSocketFactory { @Override - public void configureSocketFactory(final ClientConfiguration awsConf, + public void configureSocketFactory(final ApacheHttpClient.Builder httpClientBuilder, final DelegatingSSLSocketFactory.SSLChannelMode channelMode) throws IOException { DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode); - awsConf.getApacheHttpClientConfig().setSslSocketFactory( - new SSLConnectionSocketFactory( - DelegatingSSLSocketFactory.getDefaultFactory(), - (HostnameVerifier) null)); + httpClientBuilder.socketFactory(new SSLConnectionSocketFactory( + DelegatingSSLSocketFactory.getDefaultFactory(), + (HostnameVerifier) null)); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java deleted file mode 100644 index 16459ac45b850..0000000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyOutcome.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a.impl; - -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.s3.transfer.Copy; -import com.amazonaws.services.s3.transfer.model.CopyResult; - -/** - * Extracts the outcome of a TransferManager-executed copy operation. - */ -public final class CopyOutcome { - - /** - * Result of a successful copy. - */ - private final CopyResult copyResult; - - /** the copy was interrupted. */ - private final InterruptedException interruptedException; - - /** - * The copy raised an AWS Exception of some form. - */ - private final SdkBaseException awsException; - - public CopyOutcome(CopyResult copyResult, - InterruptedException interruptedException, - SdkBaseException awsException) { - this.copyResult = copyResult; - this.interruptedException = interruptedException; - this.awsException = awsException; - } - - public CopyResult getCopyResult() { - return copyResult; - } - - public InterruptedException getInterruptedException() { - return interruptedException; - } - - public SdkBaseException getAwsException() { - return awsException; - } - - /** - * Calls {@code Copy.waitForCopyResult()} to await the result, converts - * it to a copy outcome. - * Exceptions caught and - * @param copy the copy operation. - * @return the outcome. - */ - public static CopyOutcome waitForCopy(Copy copy) { - try { - CopyResult result = copy.waitForCopyResult(); - return new CopyOutcome(result, null, null); - } catch (SdkBaseException e) { - return new CopyOutcome(null, null, e); - } catch (InterruptedException e) { - return new CopyOutcome(null, e, null); - } - } -} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java index a45bfe46f169f..314d7cb82d1dd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java @@ -24,7 +24,7 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.slf4j.Logger; @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.util.DurationInfo; + import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.maybeAwaitCompletion; @@ -386,9 +387,9 @@ private void asyncDeleteAction( "Delete page of %d keys", keyList.size())) { if (!keyList.isEmpty()) { // first delete the files. - List files = keyList.stream() + List files = keyList.stream() .filter(e -> !e.isDirMarker) - .map(e -> e.keyVersion) + .map(e -> e.objectIdentifier) .collect(Collectors.toList()); LOG.debug("Deleting of {} file objects", files.size()); Invoker.once("Remove S3 Files", @@ -398,9 +399,9 @@ private void asyncDeleteAction( false )); // now the dirs - List dirs = keyList.stream() + List dirs = keyList.stream() .filter(e -> e.isDirMarker) - .map(e -> e.keyVersion) + .map(e -> e.objectIdentifier) .collect(Collectors.toList()); LOG.debug("Deleting of {} directory markers", dirs.size()); // This is invoked with deleteFakeDir. @@ -422,17 +423,17 @@ private void asyncDeleteAction( * to choose which statistics to update. */ private static final class DeleteEntry { - private final DeleteObjectsRequest.KeyVersion keyVersion; + private final ObjectIdentifier objectIdentifier; private final boolean isDirMarker; private DeleteEntry(final String key, final boolean isDirMarker) { - this.keyVersion = new DeleteObjectsRequest.KeyVersion(key); + this.objectIdentifier = ObjectIdentifier.builder().key(key).build(); this.isDirMarker = isDirMarker; } public String getKey() { - return keyVersion.getKey(); + return objectIdentifier.key(); } @Override diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java index f7e06413a3761..54a91323bc2e2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java @@ -18,9 +18,9 @@ package org.apache.hadoop.fs.s3a.impl; -import com.amazonaws.AmazonServiceException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND; /** * Translate from AWS SDK-wrapped exceptions into IOExceptions with @@ -49,9 +49,9 @@ private ErrorTranslation() { * @return true if the status code and error code mean that the * remote bucket is unknown. */ - public static boolean isUnknownBucket(AmazonServiceException e) { - return e.getStatusCode() == SC_404 - && AwsErrorCodes.E_NO_SUCH_BUCKET.equals(e.getErrorCode()); + public static boolean isUnknownBucket(AwsServiceException e) { + return e.statusCode() == SC_404_NOT_FOUND + && AwsErrorCodes.E_NO_SUCH_BUCKET.equals(e.awsErrorDetails().errorCode()); } /** @@ -62,8 +62,8 @@ public static boolean isUnknownBucket(AmazonServiceException e) { * @return true if the status code and error code mean that the * HEAD request returned 404 but the bucket was there. */ - public static boolean isObjectNotFound(AmazonServiceException e) { - return e.getStatusCode() == SC_404 && !isUnknownBucket(e); + public static boolean isObjectNotFound(AwsServiceException e) { + return e.statusCode() == SC_404_NOT_FOUND && !isUnknownBucket(e); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java index f75066e049d3e..d42dda59caa5f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java @@ -23,13 +23,15 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; -import com.amazonaws.services.s3.Headers; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,50 +77,50 @@ public class HeaderProcessing extends AbstractStoreOperation { * Standard HTTP header found on some S3 objects: {@value}. */ public static final String XA_CACHE_CONTROL = - XA_HEADER_PREFIX + Headers.CACHE_CONTROL; + XA_HEADER_PREFIX + AWSHeaders.CACHE_CONTROL; /** * Standard HTTP header found on some S3 objects: {@value}. */ public static final String XA_CONTENT_DISPOSITION = - XA_HEADER_PREFIX + Headers.CONTENT_DISPOSITION; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_DISPOSITION; /** * Content encoding; can be configured: {@value}. */ public static final String XA_CONTENT_ENCODING = - XA_HEADER_PREFIX + Headers.CONTENT_ENCODING; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_ENCODING; /** * Standard HTTP header found on some S3 objects: {@value}. */ public static final String XA_CONTENT_LANGUAGE = - XA_HEADER_PREFIX + Headers.CONTENT_LANGUAGE; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_LANGUAGE; /** * Length XAttr: {@value}. */ public static final String XA_CONTENT_LENGTH = - XA_HEADER_PREFIX + Headers.CONTENT_LENGTH; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_LENGTH; /** * Standard HTTP header found on some S3 objects: {@value}. */ public static final String XA_CONTENT_MD5 = - XA_HEADER_PREFIX + Headers.CONTENT_MD5; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_MD5; /** * Content range: {@value}. * This is returned on GET requests with ranges. */ public static final String XA_CONTENT_RANGE = - XA_HEADER_PREFIX + Headers.CONTENT_RANGE; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_RANGE; /** * Content type: may be set when uploading. * {@value}. */ public static final String XA_CONTENT_TYPE = - XA_HEADER_PREFIX + Headers.CONTENT_TYPE; + XA_HEADER_PREFIX + AWSHeaders.CONTENT_TYPE; /** * Etag Header {@value}. @@ -126,14 +128,14 @@ public class HeaderProcessing extends AbstractStoreOperation { * it can be retrieved via {@code getFileChecksum(path)} if * the S3A connector is enabled. */ - public static final String XA_ETAG = XA_HEADER_PREFIX + Headers.ETAG; + public static final String XA_ETAG = XA_HEADER_PREFIX + AWSHeaders.ETAG; /** * last modified XAttr: {@value}. */ public static final String XA_LAST_MODIFIED = - XA_HEADER_PREFIX + Headers.LAST_MODIFIED; + XA_HEADER_PREFIX + AWSHeaders.LAST_MODIFIED; /* AWS Specific Headers. May not be found on other S3 endpoints. */ @@ -144,50 +146,50 @@ public class HeaderProcessing extends AbstractStoreOperation { * Value {@value}. */ public static final String XA_ARCHIVE_STATUS = - XA_HEADER_PREFIX + Headers.ARCHIVE_STATUS; + XA_HEADER_PREFIX + AWSHeaders.ARCHIVE_STATUS; /** * Object legal hold status. {@value}. */ public static final String XA_OBJECT_LOCK_LEGAL_HOLD_STATUS = - XA_HEADER_PREFIX + Headers.OBJECT_LOCK_LEGAL_HOLD_STATUS; + XA_HEADER_PREFIX + AWSHeaders.OBJECT_LOCK_LEGAL_HOLD_STATUS; /** * Object lock mode. {@value}. */ public static final String XA_OBJECT_LOCK_MODE = - XA_HEADER_PREFIX + Headers.OBJECT_LOCK_MODE; + XA_HEADER_PREFIX + AWSHeaders.OBJECT_LOCK_MODE; /** * ISO8601 expiry date of object lock hold. {@value}. */ public static final String XA_OBJECT_LOCK_RETAIN_UNTIL_DATE = - XA_HEADER_PREFIX + Headers.OBJECT_LOCK_RETAIN_UNTIL_DATE; + XA_HEADER_PREFIX + AWSHeaders.OBJECT_LOCK_RETAIN_UNTIL_DATE; /** * Replication status for cross-region replicated objects. {@value}. */ public static final String XA_OBJECT_REPLICATION_STATUS = - XA_HEADER_PREFIX + Headers.OBJECT_REPLICATION_STATUS; + XA_HEADER_PREFIX + AWSHeaders.OBJECT_REPLICATION_STATUS; /** * Version ID; empty for non-versioned buckets/data. {@value}. */ public static final String XA_S3_VERSION_ID = - XA_HEADER_PREFIX + Headers.S3_VERSION_ID; + XA_HEADER_PREFIX + AWSHeaders.S3_VERSION_ID; /** * The server-side encryption algorithm to use * with AWS-managed keys: {@value}. */ public static final String XA_SERVER_SIDE_ENCRYPTION = - XA_HEADER_PREFIX + Headers.SERVER_SIDE_ENCRYPTION; + XA_HEADER_PREFIX + AWSHeaders.SERVER_SIDE_ENCRYPTION; /** * Storage Class XAttr: {@value}. */ public static final String XA_STORAGE_CLASS = - XA_HEADER_PREFIX + Headers.STORAGE_CLASS; + XA_HEADER_PREFIX + AWSHeaders.STORAGE_CLASS; /** * HTTP Referrer for logs: {@value}. @@ -275,9 +277,28 @@ private Map retrieveHeaders( final Statistic statistic) throws IOException { StoreContext context = getStoreContext(); String objectKey = context.pathToKey(path); - ObjectMetadata md; String symbol = statistic.getSymbol(); S3AStatisticsContext instrumentation = context.getInstrumentation(); + Map headers = new TreeMap<>(); + HeadObjectResponse md; + + // Attempting to get metadata for the root, so use head bucket. + if (objectKey.isEmpty()) { + HeadBucketResponse headBucketResponse = + trackDuration(instrumentation, symbol, () -> callbacks.getBucketMetadata()); + + if (headBucketResponse.sdkHttpResponse() != null + && headBucketResponse.sdkHttpResponse().headers() != null + && headBucketResponse.sdkHttpResponse().headers().get(AWSHeaders.CONTENT_TYPE) != null) { + maybeSetHeader(headers, XA_CONTENT_TYPE, + headBucketResponse.sdkHttpResponse().headers().get(AWSHeaders.CONTENT_TYPE).get(0)); + } + + maybeSetHeader(headers, XA_CONTENT_LENGTH, 0); + + return headers; + } + try { md = trackDuration(instrumentation, symbol, () -> callbacks.getObjectMetadata(objectKey)); @@ -287,59 +308,64 @@ private Map retrieveHeaders( callbacks.getObjectMetadata(objectKey + "/")); } // all user metadata - Map rawHeaders = md.getUserMetadata(); - Map headers = new TreeMap<>(); + Map rawHeaders = md.metadata(); rawHeaders.forEach((key, value) -> headers.put(XA_HEADER_PREFIX + key, encodeBytes(value))); // and add the usual content length &c, if set maybeSetHeader(headers, XA_CACHE_CONTROL, - md.getCacheControl()); + md.cacheControl()); maybeSetHeader(headers, XA_CONTENT_DISPOSITION, - md.getContentDisposition()); + md.contentDisposition()); maybeSetHeader(headers, XA_CONTENT_ENCODING, - md.getContentEncoding()); + md.contentEncoding()); maybeSetHeader(headers, XA_CONTENT_LANGUAGE, - md.getContentLanguage()); + md.contentLanguage()); // If CSE is enabled, use the unencrypted content length. - if (md.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null - && md.getUserMetaDataOf(Headers.UNENCRYPTED_CONTENT_LENGTH) != null) { - maybeSetHeader(headers, XA_CONTENT_LENGTH, - md.getUserMetaDataOf(Headers.UNENCRYPTED_CONTENT_LENGTH)); - } else { - maybeSetHeader(headers, XA_CONTENT_LENGTH, - md.getContentLength()); + // TODO: CSE is not supported yet, add these headers in during CSE work. +// if (md.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null +// && md.getUserMetaDataOf(Headers.UNENCRYPTED_CONTENT_LENGTH) != null) { +// maybeSetHeader(headers, XA_CONTENT_LENGTH, +// md.getUserMetaDataOf(Headers.UNENCRYPTED_CONTENT_LENGTH)); +// } else { +// maybeSetHeader(headers, XA_CONTENT_LENGTH, +// md.contentLength()); +// } +// maybeSetHeader(headers, XA_CONTENT_MD5, +// md.getContentMD5()); + // TODO: Add back in else block during CSE work. + maybeSetHeader(headers, XA_CONTENT_LENGTH, + md.contentLength()); + if (md.sdkHttpResponse() != null && md.sdkHttpResponse().headers() != null + && md.sdkHttpResponse().headers().get("Content-Range") != null) { + maybeSetHeader(headers, XA_CONTENT_RANGE, + md.sdkHttpResponse().headers().get("Content-Range").get(0)); } - maybeSetHeader(headers, XA_CONTENT_MD5, - md.getContentMD5()); - maybeSetHeader(headers, XA_CONTENT_RANGE, - md.getContentRange()); maybeSetHeader(headers, XA_CONTENT_TYPE, - md.getContentType()); + md.contentType()); maybeSetHeader(headers, XA_ETAG, - md.getETag()); + md.eTag()); maybeSetHeader(headers, XA_LAST_MODIFIED, - md.getLastModified()); + Date.from(md.lastModified())); // AWS custom headers maybeSetHeader(headers, XA_ARCHIVE_STATUS, - md.getArchiveStatus()); + md.archiveStatus()); maybeSetHeader(headers, XA_OBJECT_LOCK_LEGAL_HOLD_STATUS, - md.getObjectLockLegalHoldStatus()); + md.objectLockLegalHoldStatus()); maybeSetHeader(headers, XA_OBJECT_LOCK_MODE, - md.getObjectLockMode()); + md.objectLockMode()); maybeSetHeader(headers, XA_OBJECT_LOCK_RETAIN_UNTIL_DATE, - md.getObjectLockRetainUntilDate()); + md.objectLockRetainUntilDate()); maybeSetHeader(headers, XA_OBJECT_REPLICATION_STATUS, - md.getReplicationStatus()); + md.replicationStatus()); maybeSetHeader(headers, XA_S3_VERSION_ID, - md.getVersionId()); + md.versionId()); maybeSetHeader(headers, XA_SERVER_SIDE_ENCRYPTION, - md.getSSEAlgorithm()); + md.serverSideEncryptionAsString()); maybeSetHeader(headers, XA_STORAGE_CLASS, - md.getStorageClass()); - maybeSetHeader(headers, XA_STORAGE_CLASS, - md.getReplicationStatus()); + md.storageClassAsString()); + return headers; } @@ -458,70 +484,51 @@ public static Optional extractXAttrLongValue(byte[] data) { } /** - * Creates a copy of the passed {@link ObjectMetadata}. - * Does so without using the {@link ObjectMetadata#clone()} method, - * to avoid copying unnecessary headers. + * Creates a copy of the passed metadata. * This operation does not copy the {@code X_HEADER_MAGIC_MARKER} * header to avoid confusion. If a marker file is renamed, * it loses information about any remapped file. * If new fields are added to ObjectMetadata which are not * present in the user metadata headers, they will not be picked * up or cloned unless this operation is updated. - * @param source the {@link ObjectMetadata} to copy + * @param source the source metadata to copy * @param dest the metadata to update; this is the return value. + * @param copyObjectRequestBuilder CopyObjectRequest builder */ - public static void cloneObjectMetadata(ObjectMetadata source, - ObjectMetadata dest) { + public static void cloneObjectMetadata(HeadObjectResponse source, + Map dest, CopyObjectRequest.Builder copyObjectRequestBuilder) { // Possibly null attributes // Allowing nulls to pass breaks it during later use - if (source.getCacheControl() != null) { - dest.setCacheControl(source.getCacheControl()); - } - if (source.getContentDisposition() != null) { - dest.setContentDisposition(source.getContentDisposition()); - } - if (source.getContentEncoding() != null) { - dest.setContentEncoding(source.getContentEncoding()); - } - if (source.getContentMD5() != null) { - dest.setContentMD5(source.getContentMD5()); - } - if (source.getContentType() != null) { - dest.setContentType(source.getContentType()); + if (source.cacheControl() != null) { + copyObjectRequestBuilder.cacheControl(source.cacheControl()); } - if (source.getExpirationTime() != null) { - dest.setExpirationTime(source.getExpirationTime()); + if (source.contentDisposition() != null) { + copyObjectRequestBuilder.contentDisposition(source.contentDisposition()); } - if (source.getExpirationTimeRuleId() != null) { - dest.setExpirationTimeRuleId(source.getExpirationTimeRuleId()); + if (source.contentEncoding() != null) { + copyObjectRequestBuilder.contentEncoding(source.contentEncoding()); } - if (source.getHttpExpiresDate() != null) { - dest.setHttpExpiresDate(source.getHttpExpiresDate()); - } - if (source.getLastModified() != null) { - dest.setLastModified(source.getLastModified()); - } - if (source.getOngoingRestore() != null) { - dest.setOngoingRestore(source.getOngoingRestore()); - } - if (source.getRestoreExpirationTime() != null) { - dest.setRestoreExpirationTime(source.getRestoreExpirationTime()); + + if (source.contentType() != null) { + copyObjectRequestBuilder.contentType(source.contentType()); } - if (source.getSSEAlgorithm() != null) { - dest.setSSEAlgorithm(source.getSSEAlgorithm()); + + if (source.serverSideEncryption() != null) { + copyObjectRequestBuilder.serverSideEncryption(source.serverSideEncryption()); } - if (source.getSSECustomerAlgorithm() != null) { - dest.setSSECustomerAlgorithm(source.getSSECustomerAlgorithm()); + + if (source.sseCustomerAlgorithm() != null) { + copyObjectRequestBuilder.copySourceSSECustomerAlgorithm(source.sseCustomerAlgorithm()); } - if (source.getSSECustomerKeyMd5() != null) { - dest.setSSECustomerKeyMd5(source.getSSECustomerKeyMd5()); + if (source.sseCustomerKeyMD5() != null) { + copyObjectRequestBuilder.copySourceSSECustomerKeyMD5(source.sseCustomerKeyMD5()); } // copy user metadata except the magic marker header. - source.getUserMetadata().entrySet().stream() + source.metadata().entrySet().stream() .filter(e -> !e.getKey().equals(X_HEADER_MAGIC_MARKER)) - .forEach(e -> dest.addUserMetadata(e.getKey(), e.getValue())); + .forEach(e -> dest.put(e.getKey(), e.getValue())); } public interface HeaderProcessingCallbacks { @@ -534,6 +541,15 @@ public interface HeaderProcessingCallbacks { * @throws IOException IO and object access problems. */ @Retries.RetryTranslated - ObjectMetadata getObjectMetadata(String key) throws IOException; + HeadObjectResponse getObjectMetadata(String key) throws IOException; + + /** + * Retrieve the bucket metadata. + * + * @return metadata + * @throws IOException IO and object access problems. + */ + @Retries.RetryTranslated + HeadBucketResponse getBucketMetadata() throws IOException; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java new file mode 100644 index 0000000000000..435db879fabf8 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.net.URI; + +import javax.annotation.Nullable; + +import org.apache.hadoop.fs.PathIOException; + +/** + * An instantiation exception raised during reflection-based creation + * of classes. + * Uses an enum of kind so tests/code can examine it, without + * creating a full hierarchy of exception classes. + */ +public class InstantiationIOException extends PathIOException { + + public static final String ABSTRACT_PROVIDER = + "is abstract and therefore cannot be created"; + + public static final String CONSTRUCTOR_EXCEPTION = "constructor exception"; + + public static final String INSTANTIATION_EXCEPTION + = "instantiation exception"; + + public static final String DOES_NOT_IMPLEMENT + = "does not implement"; + + /** + * Exception kind. + */ + private final Kind kind; + + /** + * Class being instantiated. + */ + private final String classname; + + /** + * key used. + */ + private final String key; + + /** + * An (extensible) enum of kinds of instantiation failure. + */ + public enum Kind { + Forbidden, + InstantiationFailure, + IsAbstract, + IsNotImplementation, + Other, + Unavailable, + UnsupportedConstructor, + } + + public InstantiationIOException( + Kind kind, + @Nullable URI uri, String classname, + @Nullable String key, + String message, + Throwable cause) { + super(uri!= null ? uri.toString() : "", + "Class " + classname + " " + message + + (key != null ? (" (configuration key " + key + ")") : ""), + cause); + this.kind = kind; + this.classname = classname; + this.key = key; + } + + public String getClassname() { + return classname; + } + + public Kind getKind() { + return kind; + } + + public String getKey() { + return key; + } + + /** + * Class is abstract. + * @param uri URI of filesystem + * @param classname classname. + * @param key configuration key + * @return an exception. + */ + public static InstantiationIOException isAbstract(URI uri, String classname, String key) { + return new InstantiationIOException(Kind.IsAbstract, + uri, classname, key, ABSTRACT_PROVIDER, null); + } + + /** + * Class does not implement the desired interface. + * @param uri URI of filesystem + * @param classname classname. + * @param interfaceName required interface + * @param key configuration key + * @return an exception. + */ + public static InstantiationIOException isNotInstanceOf( + @Nullable URI uri, + String classname, + String interfaceName, + String key) { + return new InstantiationIOException(Kind.IsNotImplementation, uri, classname, + key, DOES_NOT_IMPLEMENT + " " + interfaceName, null); + } + + /** + * Class is unavailable for some reason, probably a missing dependency. + * @param uri URI of filesystem + * @param classname classname. + * @param key configuration key + * @param text text to include + * @return an exception. + */ + public static InstantiationIOException unavailable( + @Nullable URI uri, + String classname, + String key, + String text) { + return new InstantiationIOException(Kind.Unavailable, + uri, classname, key, text, null); + } + + /** + * Failure to find a valid constructor (signature, visibility) or + * factory method. + * @param uri URI of filesystem + * @param classname classname. + * @param key configuration key + * @return an exception. + */ + public static InstantiationIOException unsupportedConstructor( + @Nullable URI uri, + String classname, + String key) { + return new InstantiationIOException(Kind.UnsupportedConstructor, + uri, classname, key, CONSTRUCTOR_EXCEPTION, null); + } + + /** + * General instantiation failure. + * @param uri URI of filesystem + * @param classname classname. + * @param key configuration key + * @param t thrown + * @return an exception. + */ + public static InstantiationIOException instantiationException( + @Nullable URI uri, + String classname, + String key, + Throwable t) { + return new InstantiationIOException(Kind.InstantiationFailure, + uri, classname, key, INSTANTIATION_EXCEPTION + " " + t, t); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java index 2c34e7b9b6ec2..7af82f70aebf6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java @@ -110,11 +110,58 @@ private InternalConstants() { S3A_OPENFILE_KEYS = Collections.unmodifiableSet(keys); } + /** 200 status code: OK. */ + public static final int SC_200_OK = 200; + + /** 301 status code: Moved Permanently. */ + public static final int SC_301_MOVED_PERMANENTLY = 301; + + /** 307 status code: Temporary Redirect. */ + public static final int SC_307_TEMPORARY_REDIRECT = 307; + + /** 400 status code: Bad Request. */ + public static final int SC_400_BAD_REQUEST = 400; + + /** 401 status code: Unauthorized. */ + public static final int SC_401_UNAUTHORIZED = 401; + + /** 403 status code: Forbidden. */ + public static final int SC_403_FORBIDDEN = 403; + /** 403 error code. */ - public static final int SC_403 = 403; + @Deprecated + public static final int SC_403 = SC_403_FORBIDDEN; + + /** 404 status code: Not Found. */ + public static final int SC_404_NOT_FOUND = 404; /** 404 error code. */ - public static final int SC_404 = 404; + @Deprecated + public static final int SC_404 = SC_404_NOT_FOUND; + + /** 405 status code: Method Not Allowed. */ + public static final int SC_405_METHOD_NOT_ALLOWED = 405; + + /** 410 status code: Gone. */ + public static final int SC_410_GONE = 410; + + /** 412 status code: Precondition Failed. */ + public static final int SC_412_PRECONDITION_FAILED = 412; + + /** 416 status code: Range Not Satisfiable. */ + public static final int SC_416_RANGE_NOT_SATISFIABLE = 416; + + /** 443 status code: No Response (unofficial). */ + public static final int SC_443_NO_RESPONSE = 443; + + /** 444 status code: No Response (unofficial). */ + public static final int SC_444_NO_RESPONSE = 444; + + /** 500 status code: Internal Server Error. */ + public static final int SC_500_INTERNAL_SERVER_ERROR = 500; + + /** 503 status code: Service Unavailable. */ + public static final int SC_503_SERVICE_UNAVAILABLE = 503; /** Name of the log for throttling events. Value: {@value}. */ public static final String THROTTLE_LOG_NAME = diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java similarity index 62% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java index 96e32f362dfd9..6082c2f08daff 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java @@ -22,26 +22,32 @@ import java.nio.file.AccessDeniedException; import java.util.List; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Exception; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.s3a.AWSS3IOException; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_200_OK; /** - * Support for Multi Object Deletion. - * This is used to be a complex piece of code as it was required to - * update s3guard. - * Now all that is left is the exception extraction for better - * reporting, + * Exception raised in {@link S3AFileSystem#deleteObjects} when + * one or more of the keys could not be deleted. + * + * Used to reproduce the behaviour of SDK v1 for partial failures + * on DeleteObjects. In SDK v2, the errors are returned as part of + * the response objects. */ -public final class MultiObjectDeleteSupport { +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class MultiObjectDeleteException extends S3Exception { private static final Logger LOG = LoggerFactory.getLogger( - MultiObjectDeleteSupport.class); - - private MultiObjectDeleteSupport() { - } + MultiObjectDeleteException.class); /** * This is the exception exit code if access was denied on a delete. @@ -49,6 +55,17 @@ private MultiObjectDeleteSupport() { */ public static final String ACCESS_DENIED = "AccessDenied"; + private final List errors; + + public MultiObjectDeleteException(List errors) { + super(builder().message(errors.toString()).statusCode(SC_200_OK)); + this.errors = errors; + } + + public List errors() { + return errors; + } + /** * A {@code MultiObjectDeleteException} is raised if one or more * paths listed in a bulk DELETE operation failed. @@ -58,29 +75,23 @@ private MultiObjectDeleteSupport() { * the causes, otherwise grabs the status code and uses it in the * returned exception. * @param message text for the exception - * @param deleteException the delete exception. to translate * @return an IOE with more detail. */ - public static IOException translateDeleteException( - final String message, - final MultiObjectDeleteException deleteException) { - List errors - = deleteException.getErrors(); + public IOException translateException(final String message) { LOG.info("Bulk delete operation failed to delete all objects;" + " failure count = {}", - errors.size()); + errors().size()); final StringBuilder result = new StringBuilder( - errors.size() * 256); + errors().size() * 256); result.append(message).append(": "); String exitCode = ""; - for (MultiObjectDeleteException.DeleteError error : - deleteException.getErrors()) { - String code = error.getCode(); - String item = String.format("%s: %s%s: %s%n", code, error.getKey(), - (error.getVersionId() != null - ? (" (" + error.getVersionId() + ")") + for (S3Error error : errors()) { + String code = error.code(); + String item = String.format("%s: %s%s: %s%n", code, error.key(), + (error.versionId() != null + ? (" (" + error.versionId() + ")") : ""), - error.getMessage()); + error.message()); LOG.info(item); result.append(item); if (exitCode == null || exitCode.isEmpty() || ACCESS_DENIED.equals(code)) { @@ -89,9 +100,9 @@ public static IOException translateDeleteException( } if (ACCESS_DENIED.equals(exitCode)) { return (IOException) new AccessDeniedException(result.toString()) - .initCause(deleteException); + .initCause(this); } else { - return new AWSS3IOException(result.toString(), deleteException); + return new AWSS3IOException(result.toString(), this); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java index 575a3d1b2de81..34b4049b06123 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java @@ -23,9 +23,9 @@ import java.net.URI; import java.net.URISyntaxException; -import com.amazonaws.ClientConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; @@ -56,13 +56,12 @@ private NetworkBinding() { * so as to avoid * @param conf the {@link Configuration} used to get the client specified * value of {@code SSL_CHANNEL_MODE} - * @param awsConf the {@code ClientConfiguration} to set the - * SSLConnectionSocketFactory for. + * @param httpClientBuilder the http client builder. * @throws IOException if there is an error while initializing the * {@code SSLSocketFactory} other than classloader problems. */ public static void bindSSLChannelMode(Configuration conf, - ClientConfiguration awsConf) throws IOException { + ApacheHttpClient.Builder httpClientBuilder) throws IOException { // Validate that SSL_CHANNEL_MODE is set to a valid value. String channelModeString = conf.getTrimmed( @@ -89,7 +88,7 @@ public static void bindSSLChannelMode(Configuration conf, (Class) Class.forName(BINDING_CLASSNAME); clazz.getConstructor() .newInstance() - .configureSocketFactory(awsConf, channelMode); + .configureSocketFactory(httpClientBuilder, channelMode); } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException | LinkageError e) { @@ -103,7 +102,7 @@ public static void bindSSLChannelMode(Configuration conf, * works with the shaded AWS libraries to exist in their own class. */ interface ConfigureAWSSocketFactory { - void configureSocketFactory(ClientConfiguration awsConf, + void configureSocketFactory(ApacheHttpClient.Builder httpClientBuilder, DelegatingSSLSocketFactory.SSLChannelMode channelMode) throws IOException; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java index 5d17ae91b81e7..e0d9c7c6aada7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java @@ -22,10 +22,9 @@ import java.io.InterruptedIOException; import java.util.List; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; -import com.amazonaws.services.s3.transfer.model.CopyResult; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.InvalidRequestException; @@ -127,7 +126,7 @@ RemoteIterator listFilesAndDirectoryMarkers( * @throws IOException Other IO problems */ @Retries.RetryTranslated - CopyResult copyFile(String srcKey, + CopyObjectResponse copyFile(String srcKey, String destKey, S3ObjectAttributes srcAttributes, S3AReadOpContext readContext) @@ -142,14 +141,14 @@ CopyResult copyFile(String srcKey, * a mistaken attempt to delete the root directory. * @throws MultiObjectDeleteException one or more of the keys could not * be deleted in a multiple object delete operation. - * @throws AmazonClientException amazon-layer failure. + * @throws AwsServiceException amazon-layer failure. * @throws IOException other IO Exception. */ @Retries.RetryRaw void removeKeys( - List keysToDelete, + List keysToDelete, boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, + throws MultiObjectDeleteException, AwsServiceException, IOException; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListener.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListener.java new file mode 100644 index 0000000000000..5e4c3cf37e5ce --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListener.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +/** + * Interface for progress listeners to implement. + */ +public interface ProgressListener { + default void progressChanged(ProgressListenerEvent eventType, long bytesTransferred) {}; +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListenerEvent.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListenerEvent.java new file mode 100644 index 0000000000000..f3f9fb61e434d --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListenerEvent.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +/** + * Enum for progress listener events. + */ +public enum ProgressListenerEvent { + REQUEST_BYTE_TRANSFER_EVENT, + TRANSFER_PART_STARTED_EVENT, + TRANSFER_PART_COMPLETED_EVENT, + TRANSFER_PART_FAILED_EVENT; +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java index ae4d2fe7a3431..4bb15f74965a9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java @@ -25,9 +25,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; -import com.amazonaws.AmazonClientException; -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -122,7 +121,7 @@ public class RenameOperation extends ExecutingStoreOperation { /** * list of keys to delete on the next (bulk) delete call. */ - private final List keysToDelete = + private final List keysToDelete = new ArrayList<>(); /** @@ -199,7 +198,7 @@ private void completeActiveCopies(String reason) throws IOException { */ private void queueToDelete(Path path, String key) { LOG.debug("Queueing to delete {}", path); - keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); + keysToDelete.add(ObjectIdentifier.builder().key(key).build()); } /** @@ -268,7 +267,7 @@ public Long execute() throws IOException { } else { recursiveDirectoryRename(); } - } catch (AmazonClientException | IOException ex) { + } catch (SdkException | IOException ex) { // rename failed. // block for all ongoing copies to complete, successfully or not try { @@ -572,7 +571,7 @@ private Path copySource( */ @Retries.RetryTranslated private void removeSourceObjects( - final List keys) + final List keys) throws IOException { // remove the keys @@ -580,9 +579,9 @@ private void removeSourceObjects( // who is trying to debug why objects are no longer there. if (LOG.isDebugEnabled()) { LOG.debug("Initiating delete operation for {} objects", keys.size()); - for (DeleteObjectsRequest.KeyVersion key : keys) { - LOG.debug(" {} {}", key.getKey(), - key.getVersion() != null ? key.getVersion() : ""); + for (ObjectIdentifier objectIdentifier : keys) { + LOG.debug(" {} {}", objectIdentifier.key(), + objectIdentifier.versionId() != null ? objectIdentifier.versionId() : ""); } } @@ -619,10 +618,10 @@ private String maybeAddTrailingSlash(String key) { protected IOException convertToIOException(final Exception ex) { if (ex instanceof IOException) { return (IOException) ex; - } else if (ex instanceof SdkBaseException) { + } else if (ex instanceof SdkException) { return translateException("rename " + sourcePath + " to " + destPath, sourcePath.toString(), - (SdkBaseException) ex); + (SdkException) ex); } else { // should never happen, but for completeness return new IOException(ex); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index 7227941e34438..cacbee381bebc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -18,38 +18,36 @@ package org.apache.hadoop.fs.s3a.impl; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import javax.annotation.Nullable; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.CannedAccessControlList; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams; -import com.amazonaws.services.s3.model.SSECustomerKey; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.StorageClass; -import com.amazonaws.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.MetadataDirective; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.ServerSideEncryption; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.utils.Md5Utils; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,8 +77,8 @@ * This is where audit span information is added to the requests, * until it is done in the AWS SDK itself. * - * All created requests will be passed through - * {@link PrepareRequest#prepareRequest(AmazonWebServiceRequest)} before + * All created request builders will be passed to + * {@link PrepareRequest#prepareRequest(SdkRequest.Builder)} before * being returned to the caller. */ public class RequestFactoryImpl implements RequestFactory { @@ -101,7 +99,7 @@ public class RequestFactoryImpl implements RequestFactory { /** * ACL For new objects. */ - private final CannedAccessControlList cannedACL; + private final String cannedACL; /** * Max number of multipart entries allowed in a large @@ -147,14 +145,15 @@ protected RequestFactoryImpl( /** * Preflight preparation of AWS request. - * @param web service request - * @return prepared entry. + * @param web service request builder + * @return prepared builder. */ @Retries.OnceRaw - private T prepareRequest(T t) { - return requestPreparer != null - ? requestPreparer.prepareRequest(t) - : t; + private T prepareRequest(T t) { + if (requestPreparer != null) { + requestPreparer.prepareRequest(t); + } + return t; } /** @@ -162,7 +161,7 @@ private T prepareRequest(T t) { * @return an ACL, if any */ @Override - public CannedAccessControlList getCannedACL() { + public String getCannedACL() { return cannedACL; } @@ -174,29 +173,6 @@ protected String getBucket() { return bucket; } - /** - * Create the AWS SDK structure used to configure SSE, - * if the encryption secrets contain the information/settings for this. - * @return an optional set of KMS Key settings - */ - @Override - public Optional generateSSEAwsKeyParams() { - return EncryptionSecretOperations.createSSEAwsKeyManagementParams( - encryptionSecrets); - } - - /** - * Create the SSE-C structure for the AWS SDK, if the encryption secrets - * contain the information/settings for this. - * This will contain a secret extracted from the bucket/configuration. - * @return an optional customer key. - */ - @Override - public Optional generateSSECustomerKey() { - return EncryptionSecretOperations.createSSECustomerKey( - encryptionSecrets); - } - /** * Get the encryption algorithm of this endpoint. * @return the encryption algorithm. @@ -227,309 +203,323 @@ public StorageClass getStorageClass() { /** * Sets server side encryption parameters to the part upload * request when encryption is enabled. - * @param request upload part request - */ - protected void setOptionalUploadPartRequestParameters( - UploadPartRequest request) { - generateSSECustomerKey().ifPresent(request::setSSECustomerKey); - } - - /** - * Sets server side encryption parameters to the GET reuquest. - * request when encryption is enabled. - * @param request upload part request + * @param builder upload part request builder */ - protected void setOptionalGetObjectMetadataParameters( - GetObjectMetadataRequest request) { - generateSSECustomerKey().ifPresent(request::setSSECustomerKey); + protected void uploadPartEncryptionParameters( + UploadPartRequest.Builder builder) { + // need to set key to get objects encrypted with SSE_C + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets).ifPresent(base64customerKey -> { + builder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5(Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); } - /** - * Set the optional parameters when initiating the request (encryption, - * headers, storage, etc). - * @param request request to patch. - */ - protected void setOptionalMultipartUploadRequestParameters( - InitiateMultipartUploadRequest request) { - generateSSEAwsKeyParams().ifPresent(request::setSSEAwsKeyManagementParams); - generateSSECustomerKey().ifPresent(request::setSSECustomerKey); - } + private CopyObjectRequest.Builder buildCopyObjectRequest() { - /** - * Set the optional parameters for a PUT request. - * @param request request to patch. - */ - protected void setOptionalPutRequestParameters(PutObjectRequest request) { - generateSSEAwsKeyParams().ifPresent(request::setSSEAwsKeyManagementParams); - generateSSECustomerKey().ifPresent(request::setSSECustomerKey); - } + CopyObjectRequest.Builder copyObjectRequestBuilder = CopyObjectRequest.builder(); - /** - * Set the optional metadata for an object being created or copied. - * @param metadata to update. - * @param isDirectoryMarker is this for a directory marker? - */ - protected void setOptionalObjectMetadata(ObjectMetadata metadata, - boolean isDirectoryMarker) { - final S3AEncryptionMethods algorithm - = getServerSideEncryptionAlgorithm(); - if (S3AEncryptionMethods.SSE_S3 == algorithm) { - metadata.setSSEAlgorithm(algorithm.getMethod()); - } - if (contentEncoding != null && !isDirectoryMarker) { - metadata.setContentEncoding(contentEncoding); + if (contentEncoding != null) { + copyObjectRequestBuilder.contentEncoding(contentEncoding); } + + return copyObjectRequestBuilder; } - /** - * Create a new object metadata instance. - * Any standard metadata headers are added here, for example: - * encryption. - * - * @param length length of data to set in header; Ignored if negative - * @return a new metadata instance - */ @Override - public ObjectMetadata newObjectMetadata(long length) { - return createObjectMetadata(length, false); - } + public CopyObjectRequest.Builder newCopyObjectRequestBuilder(String srcKey, + String dstKey, + HeadObjectResponse srcom) { - /** - * Create a new object metadata instance. - * Any standard metadata headers are added here, for example: - * encryption. - * - * @param length length of data to set in header; Ignored if negative - * @param isDirectoryMarker is this for a directory marker? - * @return a new metadata instance - */ - private ObjectMetadata createObjectMetadata(long length, boolean isDirectoryMarker) { - final ObjectMetadata om = new ObjectMetadata(); - setOptionalObjectMetadata(om, isDirectoryMarker); - if (length >= 0) { - om.setContentLength(length); + CopyObjectRequest.Builder copyObjectRequestBuilder = buildCopyObjectRequest(); + + Map dstom = new HashMap<>(); + HeaderProcessing.cloneObjectMetadata(srcom, dstom, copyObjectRequestBuilder); + copyEncryptionParameters(srcom, copyObjectRequestBuilder); + + copyObjectRequestBuilder + .metadata(dstom) + .metadataDirective(MetadataDirective.REPLACE) + .acl(cannedACL); + + if (srcom.storageClass() != null) { + copyObjectRequestBuilder.storageClass(srcom.storageClass()); } - return om; - } - @Override - public CopyObjectRequest newCopyObjectRequest(String srcKey, - String dstKey, - ObjectMetadata srcom) { - CopyObjectRequest copyObjectRequest = - new CopyObjectRequest(getBucket(), srcKey, getBucket(), dstKey); - ObjectMetadata dstom = newObjectMetadata(srcom.getContentLength()); - HeaderProcessing.cloneObjectMetadata(srcom, dstom); - setOptionalObjectMetadata(dstom, false); - copyEncryptionParameters(srcom, copyObjectRequest); - copyObjectRequest.setCannedAccessControlList(cannedACL); - copyObjectRequest.setNewObjectMetadata(dstom); - Optional.ofNullable(srcom.getStorageClass()) - .ifPresent(copyObjectRequest::setStorageClass); - return prepareRequest(copyObjectRequest); + copyObjectRequestBuilder.destinationBucket(getBucket()) + .destinationKey(dstKey).sourceBucket(getBucket()).sourceKey(srcKey); + + return prepareRequest(copyObjectRequestBuilder); } /** * Propagate encryption parameters from source file if set else use the * current filesystem encryption settings. + * @param copyObjectRequestBuilder copy object request builder. * @param srcom source object metadata. - * @param copyObjectRequest copy object request body. */ - protected void copyEncryptionParameters( - ObjectMetadata srcom, - CopyObjectRequest copyObjectRequest) { - String sourceKMSId = srcom.getSSEAwsKmsKeyId(); + protected void copyEncryptionParameters(HeadObjectResponse srcom, + CopyObjectRequest.Builder copyObjectRequestBuilder) { + + final S3AEncryptionMethods algorithm = getServerSideEncryptionAlgorithm(); + + String sourceKMSId = srcom.ssekmsKeyId(); if (isNotEmpty(sourceKMSId)) { // source KMS ID is propagated LOG.debug("Propagating SSE-KMS settings from source {}", sourceKMSId); - copyObjectRequest.setSSEAwsKeyManagementParams( - new SSEAwsKeyManagementParams(sourceKMSId)); + copyObjectRequestBuilder.ssekmsKeyId(sourceKMSId); + return; } - switch (getServerSideEncryptionAlgorithm()) { - case SSE_S3: - /* no-op; this is set in destination object metadata */ - break; - - case SSE_C: - generateSSECustomerKey().ifPresent(customerKey -> { - copyObjectRequest.setSourceSSECustomerKey(customerKey); - copyObjectRequest.setDestinationSSECustomerKey(customerKey); - }); - break; - - case SSE_KMS: - generateSSEAwsKeyParams().ifPresent( - copyObjectRequest::setSSEAwsKeyManagementParams); - break; - default: + + if (S3AEncryptionMethods.SSE_S3 == algorithm) { + copyObjectRequestBuilder.serverSideEncryption(algorithm.getMethod()); + } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); + // Set the KMS key if present, else S3 uses AWS managed key. + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(kmsKey -> copyObjectRequestBuilder.ssekmsKeyId(kmsKey)); + } else if (S3AEncryptionMethods.SSE_C == algorithm) { + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) + .ifPresent(base64customerKey -> { + copyObjectRequestBuilder.copySourceSSECustomerAlgorithm( + ServerSideEncryption.AES256.name()).copySourceSSECustomerKey(base64customerKey) + .copySourceSSECustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))) + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); } } /** * Create a putObject request. * Adds the ACL, storage class and metadata * @param key key of object - * @param metadata metadata header * @param options options for the request, including headers - * @param srcfile source file - * @return the request + * @param length length of object to be uploaded + * @param isDirectoryMarker true if object to be uploaded is a directory marker + * @return the request builder */ @Override - public PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, + public PutObjectRequest.Builder newPutObjectRequestBuilder(String key, final PutObjectOptions options, - File srcfile) { - Preconditions.checkNotNull(srcfile); - PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key, - srcfile); - maybeSetMetadata(options, metadata); - setOptionalPutRequestParameters(putObjectRequest); - putObjectRequest.setCannedAcl(cannedACL); + long length, + boolean isDirectoryMarker) { + + Preconditions.checkArgument(isNotEmpty(key), "Null/empty key"); + + PutObjectRequest.Builder putObjectRequestBuilder = + buildPutObjectRequest(length, isDirectoryMarker); + putObjectRequestBuilder.bucket(getBucket()).key(key); + + if (options != null) { + putObjectRequestBuilder.metadata(options.getHeaders()); + } + + putEncryptionParameters(putObjectRequestBuilder); + if (storageClass != null) { - putObjectRequest.setStorageClass(storageClass); + putObjectRequestBuilder.storageClass(storageClass); } - putObjectRequest.setMetadata(metadata); - return prepareRequest(putObjectRequest); + + return prepareRequest(putObjectRequestBuilder); } - /** - * Create a {@link PutObjectRequest} request. - * The metadata is assumed to have been configured with the size of the - * operation. - * @param key key of object - * @param metadata metadata header - * @param options options for the request - * @param inputStream source data. - * @return the request - */ - @Override - public PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, - @Nullable final PutObjectOptions options, - InputStream inputStream) { - Preconditions.checkNotNull(inputStream); - Preconditions.checkArgument(isNotEmpty(key), "Null/empty key"); - maybeSetMetadata(options, metadata); - PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key, - inputStream, metadata); - setOptionalPutRequestParameters(putObjectRequest); - putObjectRequest.setCannedAcl(cannedACL); - if (storageClass != null) { - putObjectRequest.setStorageClass(storageClass); + private PutObjectRequest.Builder buildPutObjectRequest(long length, boolean isDirectoryMarker) { + + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder(); + + putObjectRequestBuilder.acl(cannedACL); + + if (length >= 0) { + putObjectRequestBuilder.contentLength(length); + } + + if (contentEncoding != null && !isDirectoryMarker) { + putObjectRequestBuilder.contentEncoding(contentEncoding); + } + + return putObjectRequestBuilder; + } + + private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBuilder) { + final S3AEncryptionMethods algorithm + = getServerSideEncryptionAlgorithm(); + + if (S3AEncryptionMethods.SSE_S3 == algorithm) { + putObjectRequestBuilder.serverSideEncryption(algorithm.getMethod()); + } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); + // Set the KMS key if present, else S3 uses AWS managed key. + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(kmsKey -> putObjectRequestBuilder.ssekmsKeyId(kmsKey)); + } else if (S3AEncryptionMethods.SSE_C == algorithm) { + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) + .ifPresent(base64customerKey -> { + putObjectRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); } - return prepareRequest(putObjectRequest); } @Override - public PutObjectRequest newDirectoryMarkerRequest(String directory) { + public PutObjectRequest.Builder newDirectoryMarkerRequest(String directory) { String key = directory.endsWith("/") ? directory : (directory + "/"); - // an input stream which is always empty - final InputStream inputStream = new InputStream() { - @Override - public int read() throws IOException { - return -1; - } - }; + // preparation happens in here - final ObjectMetadata metadata = createObjectMetadata(0L, true); - metadata.setContentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY); + PutObjectRequest.Builder putObjectRequestBuilder = buildPutObjectRequest(0L, true); + + putObjectRequestBuilder.bucket(getBucket()).key(key) + .contentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY); - PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key, - inputStream, metadata); - setOptionalPutRequestParameters(putObjectRequest); - putObjectRequest.setCannedAcl(cannedACL); - return prepareRequest(putObjectRequest); + putEncryptionParameters(putObjectRequestBuilder); + + return prepareRequest(putObjectRequestBuilder); } @Override - public ListMultipartUploadsRequest - newListMultipartUploadsRequest(String prefix) { - ListMultipartUploadsRequest request = new ListMultipartUploadsRequest( - getBucket()); + public ListMultipartUploadsRequest.Builder + newListMultipartUploadsRequestBuilder(String prefix) { + + ListMultipartUploadsRequest.Builder requestBuilder = ListMultipartUploadsRequest.builder(); + + requestBuilder.bucket(getBucket()); if (prefix != null) { - request.setPrefix(prefix); + requestBuilder.prefix(prefix); } - return prepareRequest(request); + return prepareRequest(requestBuilder); } @Override - public AbortMultipartUploadRequest newAbortMultipartUploadRequest( + public AbortMultipartUploadRequest.Builder newAbortMultipartUploadRequestBuilder( String destKey, String uploadId) { - return prepareRequest(new AbortMultipartUploadRequest(getBucket(), - destKey, - uploadId)); + AbortMultipartUploadRequest.Builder requestBuilder = + AbortMultipartUploadRequest.builder().bucket(getBucket()).key(destKey).uploadId(uploadId); + + return prepareRequest(requestBuilder); + } + + private void multipartUploadEncryptionParameters( + CreateMultipartUploadRequest.Builder mpuRequestBuilder) { + final S3AEncryptionMethods algorithm = getServerSideEncryptionAlgorithm(); + + if (S3AEncryptionMethods.SSE_S3 == algorithm) { + mpuRequestBuilder.serverSideEncryption(algorithm.getMethod()); + } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); + // Set the KMS key if present, else S3 uses AWS managed key. + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(kmsKey -> mpuRequestBuilder.ssekmsKeyId(kmsKey)); + } else if (S3AEncryptionMethods.SSE_C == algorithm) { + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) + .ifPresent(base64customerKey -> { + mpuRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); + } } @Override - public InitiateMultipartUploadRequest newMultipartUploadRequest( + public CreateMultipartUploadRequest.Builder newMultipartUploadRequestBuilder( final String destKey, @Nullable final PutObjectOptions options) throws PathIOException { if (!isMultipartUploadEnabled) { throw new PathIOException(destKey, "Multipart uploads are disabled."); } - final ObjectMetadata objectMetadata = newObjectMetadata(-1); - maybeSetMetadata(options, objectMetadata); - final InitiateMultipartUploadRequest initiateMPURequest = - new InitiateMultipartUploadRequest(getBucket(), - destKey, - objectMetadata); - initiateMPURequest.setCannedACL(getCannedACL()); - if (getStorageClass() != null) { - initiateMPURequest.withStorageClass(getStorageClass()); + + CreateMultipartUploadRequest.Builder requestBuilder = CreateMultipartUploadRequest.builder(); + + if (contentEncoding != null) { + requestBuilder.contentEncoding(contentEncoding); + } + + if (options != null) { + requestBuilder.metadata(options.getHeaders()); } - setOptionalMultipartUploadRequestParameters(initiateMPURequest); - return prepareRequest(initiateMPURequest); + + requestBuilder.bucket(getBucket()).key(destKey).acl(cannedACL); + + multipartUploadEncryptionParameters(requestBuilder); + + if (storageClass != null) { + requestBuilder.storageClass(storageClass); + } + + return prepareRequest(requestBuilder); } @Override - public CompleteMultipartUploadRequest newCompleteMultipartUploadRequest( + public CompleteMultipartUploadRequest.Builder newCompleteMultipartUploadRequestBuilder( String destKey, String uploadId, - List partETags) { + List partETags) { // a copy of the list is required, so that the AWS SDK doesn't // attempt to sort an unmodifiable list. - return prepareRequest(new CompleteMultipartUploadRequest(bucket, - destKey, uploadId, new ArrayList<>(partETags))); + CompleteMultipartUploadRequest.Builder requestBuilder = + CompleteMultipartUploadRequest.builder().bucket(bucket).key(destKey).uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(partETags).build()); + + return prepareRequest(requestBuilder); + } + @Override + public HeadObjectRequest.Builder newHeadObjectRequestBuilder(String key) { + + HeadObjectRequest.Builder headObjectRequestBuilder = + HeadObjectRequest.builder().bucket(getBucket()).key(key); + + // need to set key to get metadata for objects encrypted with SSE_C + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets).ifPresent(base64customerKey -> { + headObjectRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5(Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); + + return prepareRequest(headObjectRequestBuilder); } @Override - public GetObjectMetadataRequest newGetObjectMetadataRequest(String key) { - GetObjectMetadataRequest request = - new GetObjectMetadataRequest(getBucket(), key); - //SSE-C requires to be filled in if enabled for object metadata - setOptionalGetObjectMetadataParameters(request); - return prepareRequest(request); + public HeadBucketRequest.Builder newHeadBucketRequestBuilder(String bucketName) { + + HeadBucketRequest.Builder headBucketRequestBuilder = + HeadBucketRequest.builder().bucket(bucketName); + + return prepareRequest(headBucketRequestBuilder); } @Override - public GetObjectRequest newGetObjectRequest(String key) { - GetObjectRequest request = new GetObjectRequest(bucket, key); - generateSSECustomerKey().ifPresent(request::setSSECustomerKey); + public GetObjectRequest.Builder newGetObjectRequestBuilder(String key) { + GetObjectRequest.Builder builder = GetObjectRequest.builder() + .bucket(bucket) + .key(key); - return prepareRequest(request); + // need to set key to get objects encrypted with SSE_C + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets).ifPresent(base64customerKey -> { + builder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5(Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); + + return prepareRequest(builder); } @Override - public UploadPartRequest newUploadPartRequest( + public UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, - long size, - InputStream uploadStream, - File sourceFile, - long offset) throws PathIOException { + long size) throws PathIOException { checkNotNull(uploadId); - // exactly one source must be set; xor verifies this - checkArgument((uploadStream != null) ^ (sourceFile != null), - "Data source"); checkArgument(size >= 0, "Invalid partition size %s", size); checkArgument(partNumber > 0, "partNumber must be between 1 and %s inclusive, but is %s", - DEFAULT_UPLOAD_PART_COUNT_LIMIT, partNumber); + multipartPartCountLimit, partNumber); LOG.debug("Creating part upload request for {} #{} size {}", uploadId, partNumber, size); @@ -539,88 +529,76 @@ public UploadPartRequest newUploadPartRequest( throw new PathIOException(destKey, String.format(pathErrorMsg, partNumber, multipartPartCountLimit)); } - UploadPartRequest request = new UploadPartRequest() - .withBucketName(getBucket()) - .withKey(destKey) - .withUploadId(uploadId) - .withPartNumber(partNumber) - .withPartSize(size); - if (uploadStream != null) { - // there's an upload stream. Bind to it. - request.setInputStream(uploadStream); - } else { - checkArgument(sourceFile.exists(), - "Source file does not exist: %s", sourceFile); - checkArgument(sourceFile.isFile(), - "Source is not a file: %s", sourceFile); - checkArgument(offset >= 0, "Invalid offset %s", offset); - long length = sourceFile.length(); - checkArgument(offset == 0 || offset < length, - "Offset %s beyond length of file %s", offset, length); - request.setFile(sourceFile); - request.setFileOffset(offset); - } - setOptionalUploadPartRequestParameters(request); - return prepareRequest(request); + UploadPartRequest.Builder builder = UploadPartRequest.builder() + .bucket(getBucket()) + .key(destKey) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength(size); + uploadPartEncryptionParameters(builder); + return prepareRequest(builder); } @Override - public SelectObjectContentRequest newSelectRequest(String key) { - SelectObjectContentRequest request = new SelectObjectContentRequest(); - request.setBucketName(bucket); - request.setKey(key); - generateSSECustomerKey().ifPresent(request::setSSECustomerKey); - return prepareRequest(request); + public SelectObjectContentRequest.Builder newSelectRequestBuilder(String key) { + SelectObjectContentRequest.Builder requestBuilder = + SelectObjectContentRequest.builder().bucket(bucket).key(key); + + EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets).ifPresent(base64customerKey -> { + requestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5(Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); + }); + + return prepareRequest(requestBuilder); } @Override - public ListObjectsRequest newListObjectsV1Request( + public ListObjectsRequest.Builder newListObjectsV1RequestBuilder( final String key, final String delimiter, final int maxKeys) { - ListObjectsRequest request = new ListObjectsRequest() - .withBucketName(bucket) - .withMaxKeys(maxKeys) - .withPrefix(key); + + ListObjectsRequest.Builder requestBuilder = + ListObjectsRequest.builder().bucket(bucket).maxKeys(maxKeys).prefix(key); + if (delimiter != null) { - request.setDelimiter(delimiter); + requestBuilder.delimiter(delimiter); } - return prepareRequest(request); - } - @Override - public ListNextBatchOfObjectsRequest newListNextBatchOfObjectsRequest( - ObjectListing prev) { - return prepareRequest(new ListNextBatchOfObjectsRequest(prev)); + return prepareRequest(requestBuilder); } @Override - public ListObjectsV2Request newListObjectsV2Request( + public ListObjectsV2Request.Builder newListObjectsV2RequestBuilder( final String key, final String delimiter, final int maxKeys) { - final ListObjectsV2Request request = new ListObjectsV2Request() - .withBucketName(bucket) - .withMaxKeys(maxKeys) - .withPrefix(key); + + final ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() + .bucket(bucket) + .maxKeys(maxKeys) + .prefix(key); + if (delimiter != null) { - request.setDelimiter(delimiter); + requestBuilder.delimiter(delimiter); } - return prepareRequest(request); + + return prepareRequest(requestBuilder); } @Override - public DeleteObjectRequest newDeleteObjectRequest(String key) { - return prepareRequest(new DeleteObjectRequest(bucket, key)); + public DeleteObjectRequest.Builder newDeleteObjectRequestBuilder(String key) { + return prepareRequest(DeleteObjectRequest.builder().bucket(bucket).key(key)); } @Override - public DeleteObjectsRequest newBulkDeleteRequest( - List keysToDelete) { - return prepareRequest( - new DeleteObjectsRequest(bucket) - .withKeys(keysToDelete) - .withQuiet(true)); + public DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder( + List keysToDelete) { + return prepareRequest(DeleteObjectsRequest + .builder() + .bucket(bucket) + .delete(d -> d.objects(keysToDelete).quiet(true))); } @Override @@ -628,23 +606,6 @@ public void setEncryptionSecrets(final EncryptionSecrets secrets) { encryptionSecrets = secrets; } - /** - * Set the metadata from the options if the options are not - * null and the metadata contains headers. - * @param options options for the request - * @param objectMetadata metadata to patch - */ - private void maybeSetMetadata( - @Nullable PutObjectOptions options, - final ObjectMetadata objectMetadata) { - if (options != null) { - Map headers = options.getHeaders(); - if (headers != null) { - objectMetadata.setUserMetadata(headers); - } - } - } - /** * Create a builder. * @return new builder. @@ -671,7 +632,7 @@ public static final class RequestFactoryBuilder { /** * ACL For new objects. */ - private CannedAccessControlList cannedACL = null; + private String cannedACL = null; /** Content Encoding. */ private String contentEncoding; @@ -754,7 +715,7 @@ public RequestFactoryBuilder withEncryptionSecrets( * @return the builder */ public RequestFactoryBuilder withCannedACL( - final CannedAccessControlList value) { + final String value) { cannedACL = value; return this; } @@ -806,11 +767,9 @@ public interface PrepareRequest { /** * Post-creation preparation of AWS request. - * @param t request - * @param request type. - * @return prepared entry. + * @param t request builder */ @Retries.OnceRaw - T prepareRequest(T t); + void prepareRequest(SdkRequest.Builder t); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java index 4ab5bc6a99245..b7eae8ead7096 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java @@ -34,10 +34,12 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.commons.lang3.StringUtils; @@ -152,18 +154,18 @@ public CompletableFuture putPart( Charsets.UTF_8); return context.submit(new CompletableFuture<>(), () -> { - UploadPartRequest request = writeOperations.newUploadPartRequest(key, - uploadIdString, partNumber, (int) lengthInBytes, inputStream, - null, 0L); - UploadPartResult result = writeOperations.uploadPart(request, statistics); + UploadPartRequest request = writeOperations.newUploadPartRequestBuilder(key, + uploadIdString, partNumber, lengthInBytes).build(); + RequestBody body = RequestBody.fromInputStream(inputStream, lengthInBytes); + UploadPartResponse response = writeOperations.uploadPart(request, body, statistics); statistics.partPut(lengthInBytes); - String eTag = result.getETag(); + String eTag = response.eTag(); return BBPartHandle.from( ByteBuffer.wrap( buildPartHandlePayload( filePath.toUri().toString(), uploadIdString, - result.getPartNumber(), + partNumber, eTag, lengthInBytes))); }); @@ -188,7 +190,7 @@ public CompletableFuture complete( String uploadIdStr = new String(uploadIdBytes, 0, uploadIdBytes.length, Charsets.UTF_8); - ArrayList eTags = new ArrayList<>(); + ArrayList eTags = new ArrayList<>(); eTags.ensureCapacity(handles.size()); long totalLength = 0; // built up to identify duplicates -if the size of this set is @@ -201,7 +203,8 @@ public CompletableFuture complete( payload.validate(uploadIdStr, filePath); ids.add(payload.getPartNumber()); totalLength += payload.getLen(); - eTags.add(new PartETag(handle.getKey(), payload.getEtag())); + eTags.add( + CompletedPart.builder().partNumber(handle.getKey()).eTag(payload.getEtag()).build()); } Preconditions.checkArgument(ids.size() == count, "Duplicate PartHandles"); @@ -210,7 +213,7 @@ public CompletableFuture complete( long finalLen = totalLength; return context.submit(new CompletableFuture<>(), trackDurationOfCallable(statistics, MULTIPART_UPLOAD_COMPLETED.getSymbol(), () -> { - CompleteMultipartUploadResult result = + CompleteMultipartUploadResponse result = writeOperations.commitUpload( key, uploadIdStr, @@ -218,7 +221,7 @@ public CompletableFuture complete( finalLen ); - byte[] eTag = result.getETag().getBytes(Charsets.UTF_8); + byte[] eTag = result.eTag().getBytes(Charsets.UTF_8); statistics.uploadCompleted(); return (PathHandle) () -> ByteBuffer.wrap(eTag); })); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java index b566f9ad42765..49c2fb8947dce 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java @@ -18,12 +18,10 @@ package org.apache.hadoop.fs.s3a.impl; -import java.io.Closeable; +import java.io.InputStream; import java.util.concurrent.atomic.AtomicBoolean; -import javax.annotation.Nullable; - -import com.amazonaws.internal.SdkFilterInputStream; +import software.amazon.awssdk.http.Abortable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,23 +29,18 @@ import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.util.functional.CallableRaisingIOE; + import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DRAIN_BUFFER_SIZE; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration; -import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; /** * Drains/aborts s3 or other AWS SDK streams. * It is callable so can be passed directly to a submitter * for async invocation. - * A request object may be passed in; it will be implicitly - * cached until this object is GCd. - * This is because in some versions of the AWS SDK, the S3Object - * has a finalize() method which releases the http connection, - * even when the stream is still open. - * See HADOOP-17338 for details. */ -public class SDKStreamDrainer implements CallableRaisingIOE { +public class SDKStreamDrainer + implements CallableRaisingIOE { private static final Logger LOG = LoggerFactory.getLogger( SDKStreamDrainer.class); @@ -58,17 +51,9 @@ public class SDKStreamDrainer implements CallableRaisingIOE { private final String uri; /** - * Request object; usually S3Object - * Never used, but needed to keep the http connection - * open long enough for draining to take place. + * Stream from the getObject response for draining and closing. */ - @Nullable - private final Closeable requestObject; - - /** - * Stream from the {@link #requestObject} for draining and closing. - */ - private final SdkFilterInputStream sdkStream; + private final TStream sdkStream; /** * Should the request be aborted? @@ -118,7 +103,6 @@ public class SDKStreamDrainer implements CallableRaisingIOE { /** * Prepare to drain the stream. * @param uri URI for messages - * @param requestObject http request object; needed to avoid GC issues. * @param sdkStream stream to close. * @param shouldAbort force an abort; used if explicitly requested. * @param streamStatistics stats to update @@ -126,14 +110,12 @@ public class SDKStreamDrainer implements CallableRaisingIOE { * @param remaining remaining bytes */ public SDKStreamDrainer(final String uri, - @Nullable final Closeable requestObject, - final SdkFilterInputStream sdkStream, + final TStream sdkStream, final boolean shouldAbort, final int remaining, final S3AInputStreamStatistics streamStatistics, final String reason) { this.uri = uri; - this.requestObject = requestObject; this.sdkStream = requireNonNull(sdkStream); this.shouldAbort = shouldAbort; this.remaining = remaining; @@ -233,7 +215,6 @@ private boolean drainOrAbortHttpStream() { LOG.debug("Closing stream"); sdkStream.close(); - cleanupWithLogger(LOG, requestObject); // this MUST come after the close, so that if the IO operations fail // and an abort is triggered, the initial attempt's statistics // aren't collected. @@ -255,8 +236,6 @@ private boolean drainOrAbortHttpStream() { LOG.warn("When aborting {} stream after failing to close it for {}", uri, reason, e); thrown = e; - } finally { - cleanupWithLogger(LOG, requestObject); } streamStatistics.streamClose(true, remaining); @@ -269,11 +248,7 @@ public String getUri() { return uri; } - public Object getRequestObject() { - return requestObject; - } - - public SdkFilterInputStream getSdkStream() { + public TStream getSdkStream() { return sdkStream; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java index 3aa8ad270eedd..bc9b0e49a37b0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -23,11 +23,22 @@ import org.apache.hadoop.fs.store.LogExactlyOnce; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; /** * This class provides utility methods required for migrating S3A to AWS Java SDK V2. * For more information on the upgrade, see HADOOP-18073. + * + *

    in HADOOP-18382. Upgrade AWS SDK to V2 - Prerequisites, + * this class contained a series of `LogExactlyOnce` loggers to warn on + * the first use of a feature which would change incompatibly; this shipped in Hadoop 3.3.5. + *

    + * With the move to v2 completed, attempts to use the v1 classes, will fail + * -except for the special case of support for v1 credential providers. + *

    + * The warning methods are still present, where appropriate, but downgraded to debug + * and only retained for debugging migration issues. */ public final class V2Migration { @@ -35,64 +46,17 @@ private V2Migration() { } public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); - private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - - private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - - private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - - private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = + private static final LogExactlyOnce WARN_OF_REQUEST_HANDLERS = new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - - /** - * Warns on an AWS V1 credential provider being referenced directly. - * @param name name of the credential provider - */ - public static void v1ProviderReferenced(String name) { - WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn( - "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " - + "providers will be removed once S3A is upgraded to SDK V2", name); - } - - /** - * Warns on the v1 s3 client being requested. - */ - public static void v1S3ClientRequested() { - WARN_ON_GET_S3_CLIENT.warn( - "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2"); - } - - /** - * Warns when v1 credential providers are used with delegation tokens. - */ - public static void v1DelegationTokenCredentialProvidersUsed() { - WARN_ON_DELEGATION_TOKENS.warn( - "The credential provider interface has changed in AWS SDK V2, custom credential " - + "providers used in delegation tokens binding classes will need to be updated once " - + "S3A is upgraded to SDK V2"); - } - - /** - * Warns on use of custom signers. - */ - public static void v1CustomSignerUsed() { - WARN_OF_CUSTOM_SIGNER.warn( - "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " - + "once S3A is upgraded to SDK V2"); - } - /** - * Warns on use of getObjectMetadata. + * Notes use of request handlers. + * @param handlers handlers declared */ - public static void v1GetObjectMetadataCalled() { - WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response " - + "will be changed as part of upgrading S3A to AWS SDK V2"); + public static void v1RequestHandlersUsed(final String handlers) { + WARN_OF_REQUEST_HANDLERS.warn( + "Ignoring V1 SDK request handlers set in {}: {}", + AUDIT_REQUEST_HANDLERS, handlers); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java index 3ab0022bb082e..ec6e3700226e0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java @@ -19,15 +19,11 @@ package org.apache.hadoop.fs.s3a.prefetch; - import java.io.IOException; -import java.io.InputStream; -import java.util.IdentityHashMap; -import java.util.Map; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,12 +31,14 @@ import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3AInputStream; import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.S3ObjectAttributes; import org.apache.hadoop.fs.s3a.impl.ChangeTracker; import org.apache.hadoop.fs.s3a.impl.SDKStreamDrainer; import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.fs.statistics.DurationTracker; + /** * Encapsulates low level interactions with S3 object on AWS. */ @@ -74,12 +72,6 @@ public class S3ARemoteObject { */ private final ChangeTracker changeTracker; - /** - * Maps a stream returned by openForRead() to the associated S3 object. - * That allows us to close the object when closing the stream. - */ - private final Map s3Objects; - /** * uri of the object being read. */ @@ -123,7 +115,6 @@ public S3ARemoteObject( this.client = client; this.streamStatistics = streamStatistics; this.changeTracker = changeTracker; - this.s3Objects = new IdentityHashMap<>(); this.uri = this.getPath(); } @@ -187,21 +178,23 @@ public long size() { * @throws IllegalArgumentException if offset is greater than or equal to file size. * @throws IllegalArgumentException if size is greater than the remaining bytes. */ - public InputStream openForRead(long offset, int size) throws IOException { + public ResponseInputStream openForRead(long offset, int size) + throws IOException { Validate.checkNotNegative(offset, "offset"); Validate.checkLessOrEqual(offset, "offset", size(), "size()"); Validate.checkLessOrEqual(size, "size", size() - offset, "size() - offset"); streamStatistics.streamOpened(); - final GetObjectRequest request = - client.newGetRequest(s3Attributes.getKey()) - .withRange(offset, offset + size - 1); - changeTracker.maybeApplyConstraint(request); + final GetObjectRequest request = client + .newGetRequestBuilder(s3Attributes.getKey()) + .range(S3AUtils.formatRange(offset, offset + size - 1)) + .applyMutation(changeTracker::maybeApplyConstraint) + .build(); String operation = String.format( "%s %s at %d", S3AInputStream.OPERATION_OPEN, uri, offset); DurationTracker tracker = streamStatistics.initiateGetRequest(); - S3Object object = null; + ResponseInputStream object = null; try { object = Invoker.once(operation, uri, () -> client.getObject(request)); @@ -212,27 +205,14 @@ public InputStream openForRead(long offset, int size) throws IOException { tracker.close(); } - changeTracker.processResponse(object, operation, offset); - InputStream stream = object.getObjectContent(); - synchronized (s3Objects) { - s3Objects.put(stream, object); - } - - return stream; + changeTracker.processResponse(object.response(), operation, offset); + return object; } - void close(InputStream inputStream, int numRemainingBytes) { - S3Object obj; - synchronized (s3Objects) { - obj = s3Objects.remove(inputStream); - if (obj == null) { - throw new IllegalArgumentException("inputStream not found"); - } - } + void close(ResponseInputStream inputStream, int numRemainingBytes) { SDKStreamDrainer drainer = new SDKStreamDrainer( uri, - obj, - (S3ObjectInputStream)inputStream, + inputStream, false, numRemainingBytes, streamStatistics, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java index 89ea77d6d0ebb..b49b2699f916b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java @@ -22,7 +22,6 @@ import java.io.Closeable; import java.io.EOFException; import java.io.IOException; -import java.io.InputStream; import java.net.SocketTimeoutException; import java.nio.ByteBuffer; @@ -33,6 +32,9 @@ import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_REMOTE_BLOCK_READ; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation; @@ -144,7 +146,8 @@ private void readOneBlock(ByteBuffer buffer, long offset, int size) return; } - InputStream inputStream = remoteObject.openForRead(offset, readSize); + ResponseInputStream inputStream = + remoteObject.openForRead(offset, readSize); int numRemainingBytes = readSize; byte[] bytes = new byte[READ_BUFFER_SIZE]; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 608f9168c24cc..ec68168bd0ffd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import com.amazonaws.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.MultipartUpload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -694,11 +694,11 @@ private void processUploads(PrintStream out) throws IOException { count++; if (mode == Mode.ABORT || mode == Mode.LIST || verbose) { println(out, "%s%s %s", mode == Mode.ABORT ? "Deleting: " : "", - upload.getKey(), upload.getUploadId()); + upload.key(), upload.uploadId()); } if (mode == Mode.ABORT) { writeOperationHelper - .abortMultipartUpload(upload.getKey(), upload.getUploadId(), + .abortMultipartUpload(upload.key(), upload.uploadId(), true, LOG_EVENT); } } @@ -726,7 +726,7 @@ private boolean olderThan(MultipartUpload u, long msec) { return true; } Date ageDate = new Date(System.currentTimeMillis() - msec); - return ageDate.compareTo(u.getInitiated()) >= 0; + return ageDate.compareTo(Date.from(u.initiated())) >= 0; } private void processArgs(List args, PrintStream out) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/BlockingEnumeration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/BlockingEnumeration.java new file mode 100644 index 0000000000000..42000f1017259 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/BlockingEnumeration.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.select; + +import java.util.Enumeration; +import java.util.NoSuchElementException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingQueue; + +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.exception.SdkException; + +/** + * Implements the {@link Enumeration} interface by subscribing to a + * {@link SdkPublisher} instance. The enumeration will buffer a fixed + * number of elements and only request new ones from the publisher + * when they are consumed. Calls to {@link #hasMoreElements()} and + * {@link #nextElement()} may block while waiting for new elements. + * @param the type of element. + */ +public final class BlockingEnumeration implements Enumeration { + private static final class Signal { + private final T element; + private final Throwable error; + + Signal(T element) { + this.element = element; + this.error = null; + } + + Signal(Throwable error) { + this.element = null; + this.error = error; + } + } + + private final Signal endSignal = new Signal<>((Throwable)null); + private final CompletableFuture subscription = new CompletableFuture<>(); + private final BlockingQueue> signalQueue; + private final int bufferSize; + private Signal current = null; + + /** + * Create an enumeration with a fixed buffer size and an + * optional injected first element. + * @param publisher the publisher feeding the enumeration. + * @param bufferSize the buffer size. + * @param firstElement (optional) first element the enumeration will return. + */ + public BlockingEnumeration(SdkPublisher publisher, + final int bufferSize, + final T firstElement) { + this.signalQueue = new LinkedBlockingQueue<>(); + this.bufferSize = bufferSize; + if (firstElement != null) { + this.current = new Signal<>(firstElement); + } + publisher.subscribe(new EnumerationSubscriber()); + } + + /** + * Create an enumeration with a fixed buffer size. + * @param publisher the publisher feeding the enumeration. + * @param bufferSize the buffer size. + */ + public BlockingEnumeration(SdkPublisher publisher, + final int bufferSize) { + this(publisher, bufferSize, null); + } + + @Override + public boolean hasMoreElements() { + if (current == null) { + try { + current = signalQueue.take(); + } catch (InterruptedException e) { + current = new Signal<>(e); + subscription.thenAccept(Subscription::cancel); + Thread.currentThread().interrupt(); + } + } + if (current.error != null) { + Throwable error = current.error; + current = endSignal; + if (error instanceof Error) { + throw (Error)error; + } else if (error instanceof SdkException) { + throw (SdkException)error; + } else { + throw SdkException.create("Unexpected error", error); + } + } + return current != endSignal; + } + + @Override + public T nextElement() { + if (!hasMoreElements()) { + throw new NoSuchElementException(); + } + T element = current.element; + current = null; + subscription.thenAccept(s -> s.request(1)); + return element; + } + + private final class EnumerationSubscriber implements Subscriber { + + @Override + public void onSubscribe(Subscription s) { + long request = bufferSize; + if (current != null) { + request--; + } + if (request > 0) { + s.request(request); + } + subscription.complete(s); + } + + @Override + public void onNext(T t) { + signalQueue.add(new Signal<>(t)); + } + + @Override + public void onError(Throwable t) { + signalQueue.add(new Signal<>(t)); + } + + @Override + public void onComplete() { + signalQueue.add(endSignal); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java index 9c79cc1004ce1..c3b8abbc2ea88 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java @@ -21,13 +21,13 @@ import java.io.IOException; import java.util.Locale; -import com.amazonaws.services.s3.model.CSVInput; -import com.amazonaws.services.s3.model.CSVOutput; -import com.amazonaws.services.s3.model.ExpressionType; -import com.amazonaws.services.s3.model.InputSerialization; -import com.amazonaws.services.s3.model.OutputSerialization; -import com.amazonaws.services.s3.model.QuoteFields; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.CSVInput; +import software.amazon.awssdk.services.s3.model.CSVOutput; +import software.amazon.awssdk.services.s3.model.ExpressionType; +import software.amazon.awssdk.services.s3.model.InputSerialization; +import software.amazon.awssdk.services.s3.model.OutputSerialization; +import software.amazon.awssdk.services.s3.model.QuoteFields; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -145,9 +145,9 @@ public SelectObjectContentRequest buildSelectRequest( Preconditions.checkState(isEnabled(), "S3 Select is not enabled for %s", path); - SelectObjectContentRequest request = operations.newSelectRequest(path); + SelectObjectContentRequest.Builder request = operations.newSelectRequestBuilder(path); buildRequest(request, expression, builderOptions); - return request; + return request.build(); } /** @@ -175,14 +175,14 @@ private SelectInputStream executeSelect( } boolean sqlInErrors = builderOptions.getBoolean(SELECT_ERRORS_INCLUDE_SQL, errorsIncludeSql); - String expression = request.getExpression(); + String expression = request.expression(); final String errorText = sqlInErrors ? expression : "Select"; if (sqlInErrors) { LOG.info("Issuing SQL request {}", expression); } + SelectEventStreamPublisher selectPublisher = operations.select(path, request, errorText); return new SelectInputStream(readContext, - objectAttributes, - operations.select(path, request, errorText)); + objectAttributes, selectPublisher); } /** @@ -197,14 +197,14 @@ private SelectInputStream executeSelect( *

  • The default values in {@link SelectConstants}
    • * * - * @param request request to build up + * @param requestBuilder request to build up * @param expression SQL expression * @param builderOptions the options which came in from the openFile builder. * @throws IllegalArgumentException if an option is somehow invalid. * @throws IOException if an option is somehow invalid. */ void buildRequest( - final SelectObjectContentRequest request, + final SelectObjectContentRequest.Builder requestBuilder, final String expression, final Configuration builderOptions) throws IllegalArgumentException, IOException { @@ -213,7 +213,6 @@ void buildRequest( final Configuration ownerConf = operations.getConf(); - String inputFormat = builderOptions.get(SELECT_INPUT_FORMAT, SELECT_FORMAT_CSV).toLowerCase(Locale.ENGLISH); Preconditions.checkArgument(SELECT_FORMAT_CSV.equals(inputFormat), @@ -224,34 +223,24 @@ void buildRequest( Preconditions.checkArgument(SELECT_FORMAT_CSV.equals(outputFormat), "Unsupported output format %s", outputFormat); - request.setExpressionType(ExpressionType.SQL); - request.setExpression(expandBackslashChars(expression)); - - InputSerialization inputSerialization = buildCsvInputRequest(ownerConf, - builderOptions); - String compression = opt(builderOptions, - ownerConf, - SELECT_INPUT_COMPRESSION, - COMPRESSION_OPT_NONE, - true).toUpperCase(Locale.ENGLISH); - if (isNotEmpty(compression)) { - inputSerialization.setCompressionType(compression); - } - request.setInputSerialization(inputSerialization); - - request.setOutputSerialization(buildCSVOutput(ownerConf, builderOptions)); + requestBuilder.expressionType(ExpressionType.SQL); + requestBuilder.expression(expandBackslashChars(expression)); + requestBuilder.inputSerialization( + buildCsvInput(ownerConf, builderOptions)); + requestBuilder.outputSerialization( + buildCSVOutput(ownerConf, builderOptions)); } /** - * Build the CSV input request. + * Build the CSV input format for a request. * @param ownerConf FS owner configuration * @param builderOptions options on the specific request - * @return the constructed request + * @return the input format * @throws IllegalArgumentException argument failure * @throws IOException validation failure */ - public InputSerialization buildCsvInputRequest( + public InputSerialization buildCsvInput( final Configuration ownerConf, final Configuration builderOptions) throws IllegalArgumentException, IOException { @@ -283,28 +272,35 @@ public InputSerialization buildCsvInputRequest( CSV_INPUT_QUOTE_ESCAPE_CHARACTER_DEFAULT); // CSV input - CSVInput csv = new CSVInput(); - csv.setFieldDelimiter(fieldDelimiter); - csv.setRecordDelimiter(recordDelimiter); - csv.setComments(commentMarker); - csv.setQuoteCharacter(quoteCharacter); + CSVInput.Builder csvBuilder = CSVInput.builder() + .fieldDelimiter(fieldDelimiter) + .recordDelimiter(recordDelimiter) + .comments(commentMarker) + .quoteCharacter(quoteCharacter); if (StringUtils.isNotEmpty(quoteEscapeCharacter)) { - csv.setQuoteEscapeCharacter(quoteEscapeCharacter); + csvBuilder.quoteEscapeCharacter(quoteEscapeCharacter); } - csv.setFileHeaderInfo(headerInfo); - - InputSerialization inputSerialization = new InputSerialization(); - inputSerialization.setCsv(csv); - - return inputSerialization; + csvBuilder.fileHeaderInfo(headerInfo); + InputSerialization.Builder inputSerialization = + InputSerialization.builder() + .csv(csvBuilder.build()); + String compression = opt(builderOptions, + ownerConf, + SELECT_INPUT_COMPRESSION, + COMPRESSION_OPT_NONE, + true).toUpperCase(Locale.ENGLISH); + if (isNotEmpty(compression)) { + inputSerialization.compressionType(compression); + } + return inputSerialization.build(); } /** - * Build CSV output for a request. + * Build CSV output format for a request. * @param ownerConf FS owner configuration * @param builderOptions options on the specific request - * @return the constructed request + * @return the output format * @throws IllegalArgumentException argument failure * @throws IOException validation failure */ @@ -333,21 +329,19 @@ public OutputSerialization buildCSVOutput( CSV_OUTPUT_QUOTE_FIELDS, CSV_OUTPUT_QUOTE_FIELDS_ALWAYS).toUpperCase(Locale.ENGLISH); - // output is CSV, always - OutputSerialization outputSerialization - = new OutputSerialization(); - CSVOutput csvOut = new CSVOutput(); - csvOut.setQuoteCharacter(quoteCharacter); - csvOut.setQuoteFields( - QuoteFields.fromValue(quoteFields)); - csvOut.setFieldDelimiter(fieldDelimiter); - csvOut.setRecordDelimiter(recordDelimiter); + CSVOutput.Builder csvOutputBuilder = CSVOutput.builder() + .quoteCharacter(quoteCharacter) + .quoteFields(QuoteFields.fromValue(quoteFields)) + .fieldDelimiter(fieldDelimiter) + .recordDelimiter(recordDelimiter); if (!quoteEscapeCharacter.isEmpty()) { - csvOut.setQuoteEscapeCharacter(quoteEscapeCharacter); + csvOutputBuilder.quoteEscapeCharacter(quoteEscapeCharacter); } - outputSerialization.setCsv(csvOut); - return outputSerialization; + // output is CSV, always + return OutputSerialization.builder() + .csv(csvOutputBuilder.build()) + .build(); } /** @@ -359,18 +353,18 @@ public OutputSerialization buildCSVOutput( public static String toString(final SelectObjectContentRequest request) { StringBuilder sb = new StringBuilder(); sb.append("SelectObjectContentRequest{") - .append("bucket name=").append(request.getBucketName()) - .append("; key=").append(request.getKey()) - .append("; expressionType=").append(request.getExpressionType()) - .append("; expression=").append(request.getExpression()); - InputSerialization input = request.getInputSerialization(); + .append("bucket name=").append(request.bucket()) + .append("; key=").append(request.key()) + .append("; expressionType=").append(request.expressionType()) + .append("; expression=").append(request.expression()); + InputSerialization input = request.inputSerialization(); if (input != null) { sb.append("; Input") .append(input.toString()); } else { sb.append("; Input Serialization: none"); } - OutputSerialization out = request.getOutputSerialization(); + OutputSerialization out = request.outputSerialization(); if (out != null) { sb.append("; Output") .append(out.toString()); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectEventStreamPublisher.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectEventStreamPublisher.java new file mode 100644 index 0000000000000..c71ea5f1623a1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectEventStreamPublisher.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.select; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +import org.reactivestreams.Subscriber; + +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.services.s3.model.EndEvent; +import software.amazon.awssdk.services.s3.model.RecordsEvent; +import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse; +import software.amazon.awssdk.utils.ToString; + +/** + * Async publisher of {@link SelectObjectContentEventStream}s returned + * from a SelectObjectContent call. + */ +public final class SelectEventStreamPublisher implements + SdkPublisher { + + private final CompletableFuture selectOperationFuture; + private final SelectObjectContentResponse response; + private final SdkPublisher publisher; + + /** + * Create the publisher. + * @param selectOperationFuture SelectObjectContent future + * @param response SelectObjectContent response + * @param publisher SelectObjectContentEventStream publisher to wrap + */ + public SelectEventStreamPublisher( + CompletableFuture selectOperationFuture, + SelectObjectContentResponse response, + SdkPublisher publisher) { + this.selectOperationFuture = selectOperationFuture; + this.response = response; + this.publisher = publisher; + } + + /** + * Retrieve an input stream to the subset of the S3 object that matched the select query. + * This is equivalent to loading the content of all RecordsEvents into an InputStream. + * This will lazily-load the content from S3, minimizing the amount of memory used. + * @param onEndEvent callback on the end event + * @return the input stream + */ + public AbortableInputStream toRecordsInputStream(Consumer onEndEvent) { + SdkPublisher recordInputStreams = this.publisher + .filter(e -> { + if (e instanceof RecordsEvent) { + return true; + } else if (e instanceof EndEvent) { + onEndEvent.accept((EndEvent) e); + } + return false; + }) + .map(e -> ((RecordsEvent) e).payload().asInputStream()); + + // Subscribe to the async publisher using an enumeration that will + // buffer a single chunk (RecordsEvent's payload) at a time and + // block until it is consumed. + // Also inject an empty stream as the first element that + // SequenceInputStream will request on construction. + BlockingEnumeration enumeration = + new BlockingEnumeration(recordInputStreams, 1, EMPTY_STREAM); + return AbortableInputStream.create( + new SequenceInputStream(enumeration), + this::cancel); + } + + /** + * The response from the SelectObjectContent call. + * @return the response object + */ + public SelectObjectContentResponse response() { + return response; + } + + @Override + public void subscribe(Subscriber subscriber) { + publisher.subscribe(subscriber); + } + + /** + * Cancel the operation. + */ + public void cancel() { + selectOperationFuture.cancel(true); + } + + @Override + public String toString() { + return ToString.builder("SelectObjectContentEventStream") + .add("response", response) + .add("publisher", publisher) + .build(); + } + + private static final InputStream EMPTY_STREAM = + new ByteArrayInputStream(new byte[0]); +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java index f6ae52eba5346..3586d83a0a434 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java @@ -23,11 +23,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.amazonaws.AbortedException; -import com.amazonaws.services.s3.model.SelectObjectContentEvent; -import com.amazonaws.services.s3.model.SelectObjectContentEventVisitor; -import com.amazonaws.services.s3.model.SelectObjectContentResult; -import com.amazonaws.services.s3.model.SelectRecordsInputStream; +import software.amazon.awssdk.core.exception.AbortedException; +import software.amazon.awssdk.http.AbortableInputStream; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +41,7 @@ import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.io.IOUtils; + import static org.apache.hadoop.util.Preconditions.checkNotNull; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.hadoop.fs.s3a.Invoker.once; @@ -93,7 +91,7 @@ public class SelectInputStream extends FSInputStream implements * Abortable response stream. * This is guaranteed to never be null. */ - private final SelectRecordsInputStream wrappedStream; + private final AbortableInputStream wrappedStream; private final String bucket; @@ -112,14 +110,14 @@ public class SelectInputStream extends FSInputStream implements * The read attempt is initiated immediately. * @param readContext read context * @param objectAttributes object attributes from a HEAD request - * @param selectResponse response from the already executed call + * @param selectPublisher event stream publisher from the already executed call * @throws IOException failure */ @Retries.OnceTranslated public SelectInputStream( final S3AReadOpContext readContext, final S3ObjectAttributes objectAttributes, - final SelectObjectContentResult selectResponse) throws IOException { + final SelectEventStreamPublisher selectPublisher) throws IOException { Preconditions.checkArgument(isNotEmpty(objectAttributes.getBucket()), "No Bucket"); Preconditions.checkArgument(isNotEmpty(objectAttributes.getKey()), @@ -132,17 +130,17 @@ public SelectInputStream( this.readahead = readContext.getReadahead(); this.streamStatistics = readContext.getS3AStatisticsContext() .newInputStreamStatistics(); - SelectRecordsInputStream stream = once( + + AbortableInputStream stream = once( "S3 Select", uri, - () -> selectResponse.getPayload() - .getRecordsInputStream(new SelectObjectContentEventVisitor() { - @Override - public void visit(final SelectObjectContentEvent.EndEvent event) { - LOG.debug("Completed successful S3 select read from {}", uri); - completedSuccessfully.set(true); - } - })); + () -> { + return selectPublisher.toRecordsInputStream(e -> { + LOG.debug("Completed successful S3 select read from {}", uri); + completedSuccessfully.set(true); + }); + }); + this.wrappedStream = checkNotNull(stream); // this stream is already opened, so mark as such in the statistics. streamStatistics.streamOpened(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectObjectContentHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectObjectContentHelper.java new file mode 100644 index 0000000000000..8233e67eea0a5 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectObjectContentHelper.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.select; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AUtils; + +import static org.apache.hadoop.fs.s3a.WriteOperationHelper.WriteOperationHelperCallbacks; + +/** + * Helper for SelectObjectContent queries against an S3 Bucket. + */ +public final class SelectObjectContentHelper { + + private SelectObjectContentHelper() { + } + + /** + * Execute an S3 Select operation. + * @param writeOperationHelperCallbacks helper callbacks + * @param source source for selection + * @param request Select request to issue. + * @param action the action for use in exception creation + * @return the select response event stream publisher + * @throws IOException on failure + */ + public static SelectEventStreamPublisher select( + WriteOperationHelperCallbacks writeOperationHelperCallbacks, + Path source, + SelectObjectContentRequest request, + String action) + throws IOException { + try { + Handler handler = new Handler(); + CompletableFuture selectOperationFuture = + writeOperationHelperCallbacks.selectObjectContent(request, handler); + return handler.eventPublisher(selectOperationFuture).join(); + } catch (Throwable e) { + if (e instanceof CompletionException) { + e = e.getCause(); + } + IOException translated; + if (e instanceof SdkException) { + translated = S3AUtils.translateException(action, source, + (SdkException)e); + } else { + translated = new IOException(e); + } + throw translated; + } + } + + private static class Handler implements SelectObjectContentResponseHandler { + private volatile CompletableFuture>> responseAndPublisherFuture = + new CompletableFuture<>(); + + private volatile SelectObjectContentResponse response; + + public CompletableFuture eventPublisher( + CompletableFuture selectOperationFuture) { + return responseAndPublisherFuture.thenApply(p -> + new SelectEventStreamPublisher(selectOperationFuture, + p.getLeft(), p.getRight())); + } + + @Override + public void responseReceived(SelectObjectContentResponse selectObjectContentResponse) { + this.response = selectObjectContentResponse; + } + + @Override + public void onEventStream(SdkPublisher publisher) { + responseAndPublisherFuture.complete(Pair.of(response, publisher)); + } + + @Override + public void exceptionOccurred(Throwable error) { + responseAndPublisherFuture.completeExceptionally(error); + } + + @Override + public void complete() { + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/AwsStatisticsCollector.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/AwsStatisticsCollector.java index c002a4a6dee1d..711b582300200 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/AwsStatisticsCollector.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/AwsStatisticsCollector.java @@ -21,23 +21,18 @@ import java.time.Duration; import java.util.function.Consumer; import java.util.function.LongConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; -import com.amazonaws.Request; -import com.amazonaws.Response; -import com.amazonaws.metrics.RequestMetricCollector; -import com.amazonaws.util.TimingInfo; +import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.HttpMetric; +import software.amazon.awssdk.http.HttpStatusCode; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.metrics.SdkMetric; import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk; -import static com.amazonaws.util.AWSRequestMetrics.Field.ClientExecuteTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientRetryCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.HttpRequestTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.RequestCount; -import static com.amazonaws.util.AWSRequestMetrics.Field.RequestMarshallTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.RequestSigningTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.ResponseProcessingTime; -import static com.amazonaws.util.AWSRequestMetrics.Field.ThrottleException; - /** * Collect statistics from the AWS SDK and forward to an instance of * {@link StatisticsFromAwsSdk} and thence into the S3A statistics. @@ -45,9 +40,9 @@ * See {@code com.facebook.presto.hive.s3.PrestoS3FileSystemMetricCollector} * for the inspiration for this. *

    - * See {@code com.amazonaws.util.AWSRequestMetrics} for metric names. + * See {@code software.amazon.awssdk.core.metrics.CoreMetric} for metric names. */ -public class AwsStatisticsCollector extends RequestMetricCollector { +public class AwsStatisticsCollector implements MetricPublisher { /** * final destination of updates. @@ -65,65 +60,122 @@ public AwsStatisticsCollector(final StatisticsFromAwsSdk collector) { /** * This is the callback from the AWS SDK where metrics * can be collected. - * @param request AWS request - * @param response AWS response + * @param metricCollection metrics collection */ @Override - public void collectMetrics( - final Request request, - final Response response) { - - TimingInfo timingInfo = request.getAWSRequestMetrics().getTimingInfo(); - - counter(timingInfo, HttpClientRetryCount.name(), - collector::updateAwsRetryCount); - counter(timingInfo, RequestCount.name(), - collector::updateAwsRequestCount); - counter(timingInfo, ThrottleException.name(), - collector::updateAwsThrottleExceptionsCount); - - timing(timingInfo, ClientExecuteTime.name(), - collector::noteAwsClientExecuteTime); - timing(timingInfo, HttpRequestTime.name(), - collector::noteAwsRequestTime); - timing(timingInfo, RequestMarshallTime.name(), - collector::noteRequestMarshallTime); - timing(timingInfo, RequestSigningTime.name(), - collector::noteRequestSigningTime); - timing(timingInfo, ResponseProcessingTime.name(), - collector::noteResponseProcessingTime); + public void publish(MetricCollection metricCollection) { + // MetricCollections are nested, so we need to traverse through their + // "children" to collect the desired metrics. E.g.: + // + // ApiCall + // ┌─────────────────────────────────────────┐ + // │ MarshallingDuration=PT0.002808333S │ + // │ RetryCount=0 │ + // │ ApiCallSuccessful=true │ + // │ OperationName=DeleteObject │ + // │ ApiCallDuration=PT0.079801458S │ + // │ CredentialsFetchDuration=PT0.000007083S │ + // │ ServiceId=S3 │ + // └─────────────────────────────────────────┘ + // ApiCallAttempt + // ┌─────────────────────────────────────────────────────────────────┐ + // │ SigningDuration=PT0.000319375S │ + // │ ServiceCallDuration=PT0.078908584S │ + // │ AwsExtendedRequestId=Kmvb2Sz8NuDgIFJPKzLLBhuHgQGmpAjVYBMrSHDvy= │ + // │ HttpStatusCode=204 │ + // │ BackoffDelayDuration=PT0S │ + // │ AwsRequestId=KR0XZCSX │ + // └─────────────────────────────────────────────────────────────────┘ + // HttpClient + // ┌─────────────────────────────────┐ + // │ AvailableConcurrency=1 │ + // │ LeasedConcurrency=0 │ + // │ ConcurrencyAcquireDuration=PT0S │ + // │ PendingConcurrencyAcquires=0 │ + // │ MaxConcurrency=96 │ + // │ HttpClientName=Apache │ + // └─────────────────────────────────┘ + + final long[] throttling = {0}; + recurseThroughChildren(metricCollection) + .collect(Collectors.toList()) + .forEach(m -> { + counter(m, CoreMetric.RETRY_COUNT, retries -> { + collector.updateAwsRetryCount(retries); + collector.updateAwsRequestCount(retries + 1); + }); + + counter(m, HttpMetric.HTTP_STATUS_CODE, statusCode -> { + if (statusCode == HttpStatusCode.THROTTLING) { + throttling[0] += 1; + } + }); + + timing(m, CoreMetric.API_CALL_DURATION, + collector::noteAwsClientExecuteTime); + + timing(m, CoreMetric.SERVICE_CALL_DURATION, + collector::noteAwsRequestTime); + + timing(m, CoreMetric.MARSHALLING_DURATION, + collector::noteRequestMarshallTime); + + timing(m, CoreMetric.SIGNING_DURATION, + collector::noteRequestSigningTime); + + timing(m, CoreMetric.UNMARSHALLING_DURATION, + collector::noteResponseProcessingTime); + }); + + collector.updateAwsThrottleExceptionsCount(throttling[0]); + } + + @Override + public void close() { + } /** * Process a timing. - * @param timingInfo timing info - * @param subMeasurementName sub measurement + * @param collection metric collection + * @param metric metric * @param durationConsumer consumer */ private void timing( - TimingInfo timingInfo, - String subMeasurementName, + MetricCollection collection, + SdkMetric metric, Consumer durationConsumer) { - TimingInfo t1 = timingInfo.getSubMeasurement(subMeasurementName); - if (t1 != null && t1.getTimeTakenMillisIfKnown() != null) { - durationConsumer.accept(Duration.ofMillis( - t1.getTimeTakenMillisIfKnown().longValue())); - } + collection + .metricValues(metric) + .forEach(v -> durationConsumer.accept(v)); } /** * Process a counter. - * @param timingInfo timing info - * @param subMeasurementName sub measurement + * @param collection metric collection + * @param metric metric * @param consumer consumer */ private void counter( - TimingInfo timingInfo, - String subMeasurementName, + MetricCollection collection, + SdkMetric metric, LongConsumer consumer) { - Number n = timingInfo.getCounter(subMeasurementName); - if (n != null) { - consumer.accept(n.longValue()); - } + collection + .metricValues(metric) + .forEach(v -> consumer.accept(v.longValue())); + } + + /** + * Metric collections can be nested. Exposes a stream of the given + * collection and its nested children. + * @param metrics initial collection + * @return a stream of all nested metric collections + */ + private static Stream recurseThroughChildren( + MetricCollection metrics) { + return Stream.concat( + Stream.of(metrics), + metrics.children().stream() + .flatMap(c -> recurseThroughChildren(c))); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java index 4ddc5f9478bb1..ef8413ccf0a64 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java @@ -32,9 +32,8 @@ import java.util.Map; import java.util.stream.Collectors; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; @@ -55,12 +54,14 @@ import org.apache.hadoop.fs.s3a.impl.DirMarkerTracker; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool; import org.apache.hadoop.fs.shell.CommandFormat; import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.ExitUtil; + import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH; import static org.apache.hadoop.fs.s3a.Constants.BULK_DELETE_PAGE_SIZE; import static org.apache.hadoop.fs.s3a.Constants.BULK_DELETE_PAGE_SIZE_DEFAULT; @@ -784,7 +785,7 @@ long getTotalDeleteRequestDuration() { private MarkerPurgeSummary purgeMarkers( final DirMarkerTracker tracker, final int deletePageSize) - throws MultiObjectDeleteException, AmazonClientException, IOException { + throws MultiObjectDeleteException, AwsServiceException, IOException { MarkerPurgeSummary summary = new MarkerPurgeSummary(); // we get a map of surplus markers to delete. @@ -792,13 +793,13 @@ private MarkerPurgeSummary purgeMarkers( = tracker.getSurplusMarkers(); int size = markers.size(); // build a list from the strings in the map - List collect = + List collect = markers.values().stream() - .map(p -> new DeleteObjectsRequest.KeyVersion(p.getKey())) + .map(p -> ObjectIdentifier.builder().key(p.getKey()).build()) .collect(Collectors.toList()); // build an array list for ease of creating the lists of // keys in each page through the subList() method. - List markerKeys = + List markerKeys = new ArrayList<>(collect); // now randomize. Why so? if the list spans multiple S3 partitions, @@ -819,7 +820,7 @@ pages, suffix(pages), while (start < size) { // end is one past the end of the page int end = Math.min(start + deletePageSize, size); - List page = markerKeys.subList(start, + List page = markerKeys.subList(start, end); once("Remove S3 Keys", tracker.getBasePath().toString(), () -> diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java index a701f86f7b0c3..7aaec40a86805 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java @@ -21,15 +21,16 @@ import java.io.IOException; import java.util.List; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; + /** * Operations which must be offered by the store for {@link MarkerTool}. @@ -62,14 +63,14 @@ RemoteIterator listObjects( * a mistaken attempt to delete the root directory. * @throws MultiObjectDeleteException one or more of the keys could not * be deleted in a multiple object delete operation. - * @throws AmazonClientException amazon-layer failure. + * @throws AwsServiceException amazon-layer failure. * @throws IOException other IO Exception. */ @Retries.RetryMixed void removeKeys( - List keysToDelete, + List keysToDelete, boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, + throws MultiObjectDeleteException, AwsServiceException, IOException; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java index ccf80e1dde00e..d7c77feed1083 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java @@ -21,15 +21,16 @@ import java.io.IOException; import java.util.List; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; import org.apache.hadoop.fs.s3a.impl.OperationCallbacks; + /** * Implement the marker tool operations by forwarding to the * {@link OperationCallbacks} instance provided in the constructor. @@ -55,9 +56,9 @@ public RemoteIterator listObjects(final Path path, @Override public void removeKeys( - final List keysToDelete, + final List keysToDelete, final boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, IOException { + throws MultiObjectDeleteException, AwsServiceException, IOException { operationCallbacks.removeKeys(keysToDelete, deleteFakeDir ); } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index 094ea5668c05e..ea53b2e1fa9e3 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -195,7 +195,7 @@ Here are the full set of configuration options. fs.s3a.assumed.role.credentials.provider org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, - com.amazonaws.auth.EnvironmentVariableCredentialsProvider + software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider List of credential providers to authenticate with the STS endpoint and diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 9f107809eca6f..9d424bc2d8c05 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -22,7 +22,7 @@ and inside the AWS S3 SDK, immediately before the request is executed. The full architecture is covered in [Auditing Architecture](auditing_architecture.html); this document covers its use. -## Important: Auditing is disabled by default +## Important: Auditing is currently enabled Due to a memory leak from the use of `ThreadLocal` fields, this auditing feature leaked memory as S3A filesystem instances were created and deleted. @@ -32,7 +32,7 @@ See [HADOOP-18091](https://issues.apache.org/jira/browse/HADOOP-18091) _S3A audi To avoid these memory leaks, auditing was disabled by default in the hadoop 3.3.2 release. -As these memory leaks have now been fixed, auditing has been re-enabled. +As these memory leaks have now been fixed, auditing has been re-enabled in Hadoop 3.3.5+ To disable it, set `fs.s3a.audit.enabled` to `false`. @@ -77,7 +77,7 @@ ideally even identifying the process/job generating load. ## Using Auditing -Auditing is disabled by default. +Auditing is enabled by default. When auditing enabled, a Logging Auditor will annotate the S3 logs through a custom HTTP Referrer header in requests made to S3. Other auditor classes may be used instead. @@ -88,7 +88,7 @@ Other auditor classes may be used instead. |--------|---------|---------------| | `fs.s3a.audit.enabled` | Is auditing enabled? | `true` | | `fs.s3a.audit.service.classname` | Auditor classname | `org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor` | -| `fs.s3a.audit.request.handlers` | List of extra subclasses of AWS SDK RequestHandler2 to include in handler chain | `""` | +| `fs.s3a.audit.execution.interceptors` | Implementations of AWS v2 SDK `ExecutionInterceptor` to include in handler chain | `""` | | `fs.s3a.audit.referrer.enabled` | Logging auditor to publish the audit information in the HTTP Referrer header | `true` | | `fs.s3a.audit.referrer.filter` | List of audit fields to filter | `""` | | `fs.s3a.audit.reject.out.of.span.operations` | Auditor to reject operations "outside of a span" | `false` | @@ -96,14 +96,14 @@ Other auditor classes may be used instead. ### Disabling Auditing. -In this release of Hadoop, auditing is disabled. +In this release of Hadoop, auditing is enabled by default. This can be explicitly set globally or for specific buckets ```xml fs.s3a.audit.enabled - false + true ``` @@ -162,6 +162,26 @@ correlate access by S3 clients to the actual operations taking place. Note: this logging is described as "Best Effort". There's no guarantee as to when logs arrive. +### Integration with AWS SDK request processing + +The auditing component inserts itself into the AWS SDK request processing +code, so it can attach the referrer header. + +It is possible to declare extra classes to add to the processing chain, +all of which must implement the interface `software.amazon.awssdk.core.interceptor.ExecutionInterceptor`. + +The list of classes is set in the configuration option `fs.s3a.audit.execution.interceptors`. + +Any class in the list which implements `org.apache.hadoop.conf.Configurable` will have +`Configurable.setConf()` called with the filesystem configuration passed down. + +Before the upgrade to the V2 SDK, a list of extra subclasses of the AWS SDK `com.amazonaws.handlers.RequestHandler2` +class could be declared in the option `fs.s3a.audit.request.handlers`; +these would be wired up into the V1 request processing pipeline. + +This option is now ignored completely, other than printing a warning message the first time a filesystem is created with a non-empty value. + + ### Rejecting out-of-span operations The logging auditor can be configured to raise an exception whenever @@ -201,7 +221,7 @@ The HTTP referrer header is attached by the logging auditor. If the S3 Bucket is configured to log requests to another bucket, then these logs entries will include the audit information _as the referrer_. -This can be parsed (consult AWS documentation for a regular expression) +The S3 Server log entries can be parsed (consult AWS documentation for a regular expression) and the http referrer header extracted. ``` @@ -242,13 +262,15 @@ If any of the field values were `null`, the field is omitted. _Notes_ -* Thread IDs are from the current thread in the JVM, so can be compared to those in````````` +* Thread IDs are from the current thread in the JVM, so can be compared to those in Log4J logs. They are never unique. * Task Attempt/Job IDs are only ever set during operations involving the S3A committers, specifically - all operations excecuted by the committer. + all operations executed by the committer. Operations executed in the same thread as the committer's instantiation _may_ also report the IDs, even if they are unrelated to the actual task. Consider them "best effort". +Thread IDs are generated as follows: + ```java Long.toString(Thread.currentThread().getId()) ``` @@ -269,6 +291,8 @@ This is why the span ID is always passed in as part of the URL, rather than just an HTTP query parameter: even if the header is chopped, the span ID will always be present. +As of August 2023, this header is not collected in AWS CloudTrail -only S3 Server logs. + ## Privacy Implications of HTTP Referrer auditing When the S3A client makes requests of an S3 bucket, the auditor @@ -423,6 +447,12 @@ log4j.logger.org.apache.hadoop.fs.s3a.audit=TRACE This is very noisy and not recommended in normal operation. +If logging of HTTP IO is enabled then the "referer" header is printed as part of every request: +``` +log4j.logger.org.apache.http=DEBUG +log4j.logger.software.amazon.awssdk.thirdparty.org.apache.http.client.HttpClient=DEBUG +``` + ## Integration with S3A Committers Work submitted through the S3A committer will have the job (query) ID associated diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md index e649a8d76d539..138e060155c0e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -20,28 +20,42 @@ This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HAD ## Why the upgrade? - Moving to SDK V2 will provide performance benefits. -For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) +For example, the [transfer manager for SDK V2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) is built using java bindings of the AWS Common Runtime S3 client (https://github.com/awslabs/aws-crt-java) (CRT). CRT is a set of packages written in C, designed for maximising performance when interacting with AWS services such as S3. +- The V1 SDK is essentially in maintenance mode. - New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) -which S3A will benefit from are not available in SDKV1. +which S3A will benefit from are not available in SDK V1. ## What's changing? The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from [SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A. -A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes). +A complete list of the changes can be found in the +[Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes). -The major changes and how this affects S3A are listed below. +## Packaging: `aws-java-sdk-bundle-1.12.x.jar` becomes `bundle-2.x.y.jar` -### Package Change +As the module name is lost, in hadoop releases a large JAR file with +the name "bundle" is now part of the distribution. +This is the AWS V2 SDK shaded artifact. -Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes -were under `com.amazonaws`. +The new and old SDKs can co-exist; the only place that the hadoop code +may still use the original SDK is when a non-standard V1 AWS credential +provider is declared. + +Any deployment of the S3A connector must include this JAR or +the subset of non-shaded aws- JARs needed for communication +with S3 and any other services used. +As before: the exact set of dependencies used by the S3A connector +is neither defined nor comes with any commitments of stability +or compatibility of dependent libraries. -### Credential Providers + + +## Credential Provider changes and migration - Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java) has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java). @@ -49,23 +63,295 @@ has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProv changed. The change in interface will mean that custom credential providers will need to be updated to now -implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`. +implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider` instead of +`com.amazonaws.auth.AWSCredentialsProvider`. -Due to change in class names, references to SDK V1 credential providers -in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers. +### Original V1 `AWSCredentialsProvider` interface -### Delegation Tokens +Note how the interface begins with the capitalized "AWS" acronym. +The V2 interface starts with "Aws". This is a very subtle change +for developers to spot. +Compilers _will_ detect and report the type mismatch. + + +```java +package com.amazonaws.auth; + +public interface AWSCredentialsProvider { + + public AWSCredentials getCredentials(); + + public void refresh(); + +} + +``` +The interface binding also supported a factory method, `AWSCredentialsProvider instance()` which, +if available, would be invoked in preference to using any constructor. + +If the interface implemented `Closeable` or `AutoCloseable`, these would +be invoked when the provider chain was being shut down. + +### New V2 `AwsCredentialsProvider` interface + +```java +package software.amazon.awssdk.auth.credentials; + +public interface AwsCredentialsProvider { + + AwsCredentials resolveCredentials(); + +} +``` + +1. There is no `refresh()` method any more. +2. `getCredentials()` has become `resolveCredentials()`. +3. There is now the expectation in the SDK that credential resolution/lookup etc will be + performed in `resolveCredentials()`. +4. If the interface implements `Closeable` or `AutoCloseable`, these will + be invoked when the provider chain is being shut down. +5. A static method `create()` which returns an `AwsCredentialsProvider` or subclass; this will be used + in preference to a constructor + +### S3A `AWSCredentialProviderList` is now a V2 credential provider + +The class `org.apache.hadoop.fs.s3a.AWSCredentialProviderList` has moved from +being a V1 to a V2 credential provider; even if an instance can be created with +existing code, the V1 methods will not resolve: + +``` +java.lang.NoSuchMethodError: org.apache.hadoop.fs.s3a.AWSCredentialProviderList.getCredentials()Lcom/amazonaws/auth/AWSCredentials; + at org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.validateFilesystem(S3ADiagnosticsInfo.java:903) +``` + +### Migration of Credential Providers listed in `fs.s3a.aws.credentials.provider` -Custom credential providers used in delegation token binding classes will also need to be updated. -### AmazonS3 replaced by S3Client +Before: `fs.s3a.aws.credentials.provider` took a list of v1 credential providers, +This took a list containing +1. V1 credential providers implemented in the `hadoop-aws` module. +2. V1 credential providers implemented in the `aws-sdk-bundle` library. +3. Custom V1 credential providers placed onto the classpath. +4. Custom subclasses of hadoop-aws credential providers. -The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. +And here is how they change +1. All `hadoop-aws` credential providers migrated to V2. +2. Well-known `aws-sdk-bundle` credential providers _automatically remapped_ to their V2 equivalents. +3. Custom v1 providers supported if the original `aws-sdk-bundle` JAR is on the classpath. +4. Custom subclasses of hadoop-aws credential providers need manual migration. -For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2 -`S3Client`. +Because of (1) and (2), As result, standard `fs.s3a.aws.credentials.provider` configurations +should seamlessly upgrade. This also means that the same provider list, if restricted to +those classes, will work across versions. + + +### `hadoop-aws` credential providers migration to V2 + +All the fs.s3a credential providers have the same name and functionality as before. + +| Hadoop module credential provider | Authentication Mechanism | +|----------------------------------------------------------------|--------------------------------------------------| +| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider` | Session Credentials in configuration | +| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` | Simple name/secret credentials in configuration | +| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` | Anonymous Login | +| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider` | [Assumed Role credentials](./assumed_roles.html) | +| `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials | + +### Automatic `aws-sdk-bundle` credential provider remapping + +The commonly-used set of V1 credential providers are automatically remapped to V2 equivalents. + + + +| V1 Credential Provider | Remapped V2 substitute | +|-------------------------------------------------------------|----------------------------------------------------------------------------------| +| `com.amazonaws.auth.AnonymousAWSCredentials` | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` | +| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider` | `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | +| `com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper` | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | +| `com.amazonaws.auth.InstanceProfileCredentialsProvider` | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | +| `com.amazonaws.auth.profile.ProfileCredentialsProvider` | `software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider` | + +There are still a number of troublespots here: + +#### Less widely used`com.amazonaws.auth.` AWS providers + +There should be equivalents in the new SDK, but as well as being renamed +they are likely to have moved different factory/builder mechanisms. +Identify the changed classes and use their +names in the `fs.s3a.aws.credentials.provider` option. + +If a V2 equivalent is not found; provided the V1 SDK is added to the classpath, +it should still be possible to use the existing classes. + + +#### Private/third-party credential providers + +Provided the V1 SDK is added to the classpath, +it should still be possible to use the existing classes. + +Adding a V2 equivalent is the recommended long-term solution. + +#### Custom subclasses of the Hadoop credential providers + +Because all the standard hadoop credential providers have been upgraded, +any subclasses of these are not going to link or work. + +These will need to be manually migrated to being V2 Credential providers. + + +## Source code/binary integration changes + +The major changes and how this affects S3A are listed below. + +### SDK API Package Change + +* Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes +were under `com.amazonaws`. +* There is no interoperability between the old and new classes. +* All classnames are different, often in very subtle ways. It is possible to use both in the same + class, as is done in the package `org.apache.hadoop.fs.s3a.adapter`. +* All the core message classes are now automatically generated from a JSON protocol description. +* All getter methods have been renamed. +* All classes are constructed via builder methods +* Message classes are no longer Java `Serializable`. + +Most of these changes simply create what will feel to be gratuitous migration effort; +the removable of the `Serializable` nature from all message response classes can +potentially break applications -such as anything passing them between Spark workers. +See AWS SDK V2 issue [Simplify Modeled Message Marshalling #82](https://github.com/aws/aws-sdk-java-v2/issues/82), +note that it was filed in 2017, then implement your own workaround pending that issue +being resolved. + +### Compilation/Linkage Errors + +Any code making use of V1 sdk classes will fail if they +* Expect the V1 sdk classes to be on the classpath when `hadoop-aws` is declared as a dependency +* Use V1-SDK-compatible methods previously exported by the `S3AFileSystem` class and associated classes. +* Try to pass s3a classes to V1 SDK classes (e.g. credential providers). + +The sole solution to these problems is "move to the V2 SDK". + +Some `S3AUtils` methods are deleted +``` +cannot find symbol +[ERROR] symbol: method createAwsConf(org.apache.hadoop.conf.Configuration,java.lang.String) +[ERROR] location: class org.apache.hadoop.fs.s3a.S3AUtils +``` + +The signature and superclass of `AWSCredentialProviderList` has changed, which can surface in different +ways + +Signature mismatch +``` + cannot find symbol +[ERROR] symbol: method getCredentials() +[ERROR] location: variable credentials of type org.apache.hadoop.fs.s3a.AWSCredentialProviderList +``` + +It is no longer a V1 credential provider, cannot be used to pass credentials to a V1 SDK class +``` +incompatible types: org.apache.hadoop.fs.s3a.AWSCredentialProviderList cannot be converted to com.amazonaws.auth.AWSCredentialsProvider +``` + +### `AmazonS3` replaced by `S3Client`; factory and accessor changed. + +The V1 s3 client class `com.amazonaws.services.s3.AmazonS3` has been superseded by +`software.amazon.awssdk.services.s3.S3Client` + +The `S3ClientFactory` interface has been replaced by one that creates a V2 `S3Client`. +* Custom implementations will need to be updated. +* The `InconsistentS3ClientFactory` class has been deleted. + +### `S3AFileSystem` method changes: `S3AInternals`. + +The low-level s3 operations/client accessors have been moved into a new interface, +`org.apache.hadoop.fs.s3a.S3AInternals`, which must be accessed via the +`S3AFileSystem.getS3AInternals()` method. +They have also been updated to return V2 SDK classes. + +```java +@InterfaceStability.Unstable +@InterfaceAudience.LimitedPrivate("testing/diagnostics") +public interface S3AInternals { + S3Client getAmazonS3V2Client(String reason); + + @Retries.RetryTranslated + @AuditEntryPoint + String getBucketLocation() throws IOException; + + @AuditEntryPoint + @Retries.RetryTranslated + String getBucketLocation(String bucketName) throws IOException; + + @AuditEntryPoint + @Retries.RetryTranslated + HeadObjectResponse getObjectMetadata(Path path) throws IOException; + + AWSCredentialProviderList shareCredentials(final String purpose); +} +``` + + +#### `S3AFileSystem.getAmazonS3ClientForTesting(String)` moved and return type changed + +The `S3AFileSystem.getAmazonS3ClientForTesting()` method has been been deleted. + +Compilation +``` +cannot find symbol +[ERROR] symbol: method getAmazonS3ClientForTesting(java.lang.String) +[ERROR] location: variable fs of type org.apache.hadoop.fs.s3a.S3AFileSystem +``` + +It has been replaced by an `S3AInternals` equivalent which returns the V2 `S3Client` +of the filesystem instance. + +```java +((S3AFilesystem)fs).getAmazonS3ClientForTesting("testing") +``` + +```java +((S3AFilesystem)fs).getS3AInternals().getAmazonS3Client("testing") +``` + +##### `S3AFileSystem.getObjectMetadata(Path path)` moved to `S3AInternals`; return type changed + +The `getObjectMetadata(Path)` call has been moved to the `S3AInternals` interface +and an instance of the `software.amazon.awssdk.services.s3.model.HeadObjectResponse` class +returned. +The original `S3AFileSystem` method has been deleted + +Before: +```java +((S3AFilesystem)fs).getObjectMetadata(path) +``` + +After: +```java +((S3AFilesystem)fs).getS3AInternals().getObjectMetadata(path) +``` + +##### `AWSCredentialProviderList shareCredentials(String)` moved to `S3AInternals` + +The operation to share a reference-counted access to the AWS credentials used +by the S3A FS has been moved to `S3AInternals`. + +This is very much an implementation method, used to allow extension modules to share +an authentication chain into other AWS SDK client services (dynamoDB, etc.). + +### Delegation Tokens + +1. Custom credential providers used in delegation token binding classes will need to be updated +2. The return type from delegation token binding has changed to support more class + instances being returned in the future. + +`AWSCredentialProviderList` has been upgraded to the V2 API. +* It still retains a `refresh()` method but this is now a deprecated no-op. +* It is still `Closeable`; its `close()` method iterates through all entries in +the list; if they are `Closeable` or `AutoCloseable` then their `close()` method is invoked. +* Accordingly, providers may still perform background refreshes in separate threads; + the S3A client will close its provider list when the filesystem itself is closed. -The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. ### Signers @@ -74,3 +360,21 @@ has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github. The change in signers will mean the custom signers will need to be updated to implement the new interface. + +There is no support to assist in this migration. + +### S3A Auditing Extensions. + +The callbacks from the SDK have all changed, as has +the interface `org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks` + +Examine the interface and associated implementations to +see how to migrate. + +The option `fs.s3a.audit.request.handlers` to declare a list of v1 SDK +`com.amazonaws.handlers.RequestHandler2` implementations to include +in the AWS request chain is no longer supported: a warning is printed +and the value ignored. + +The V2 SDK equivalent, classes implementing `software.amazon.awssdk.core.interceptor.ExecutionInterceptor` +can be declared in the configuration option `fs.s3a.audit.execution.interceptors`. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_v2_changelog.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_v2_changelog.md new file mode 100644 index 0000000000000..162f15951f5ca --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_v2_changelog.md @@ -0,0 +1,340 @@ + + +# Upgrade S3A to AWS SDK V2: Changelog + +Note: This document is not meant to be committed as part of the final merge, and instead just serves +as a guide to help with reviewing the PR. + +This document tracks changes to S3A during the upgrade to AWS SDK V2. Once the upgrade +is complete, some of its content will be added to the existing document +[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). + +This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073). + +## Contents + +* [Client Configuration](#client-configuration) +* [Endpoint and region configuration](#endpoint-and-region-configuration) +* [List Object](#list-object) +* [EncryptionSecretOperations](#encryptionsecretoperations) +* [GetObjectMetadata](#getobjectmetadata) +* [PutObject](#putobject) +* [CopyObject](#copyobject) +* [MultipartUpload](#multipartupload) +* [GetObject](#getObject) +* [DeleteObject](#deleteobject) +* [Select](#select) +* [CredentialsProvider](#credentialsprovider) +* [Auditing](#auditing) +* [Metric Collection](#metric-collection) +* [Exception Handling](#exception-handling) +* [Failure Injection](#failure-injection) + +### Client Configuration: + +* We now have two clients, a sync S3 Client and an async S3 Client. The async s3 client is required + as the select operation is currently only supported on the async client. Once we are confident in + the current set of changes, we will also be exploring moving other operations to the async client + as this could provide potential performance benefits. However those changes are not in the scope + of this PR, and will be done separately. +* The [createAwsConf](https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java#L1190) +method is now split into: + ``` + createClientConfigBuilder // sets request timeout, user agent* + createHttpClientBuilder* // sets max connections, connection timeout, socket timeout + createProxyConfigurationBuilder // sets proxy config, defined in table below + ``` + +The table below lists the configurations S3A was using and what they now map to. + +|SDK V1 |SDK V2 | +|---|---| +|setMaxConnections |httpClientBuilder.maxConnections | +|setProtocol |The protocol is now HTTPS by default, and can only be modified by setting an HTTP endpoint on the client builder. This is done when setting the endpoint in getS3Endpoint() | +|setMaxErrorRetry |createRetryPolicyBuilder | +|setConnectionTimeout |httpClientBuilder.connectionTimeout | +|setSocketTimeout |httpClientBuilder.socketTimeout | +|setRequestTimeout |overrideConfigBuilder.apiCallAttemptTimeout | +|setSocketBufferSizeHints |Not supported | +|setSignerOverride |Not done yet | +|setProxyHost |proxyConfigBuilder.endpoint | +|setProxyPort |set when setting proxy host with .endpoint | +|setProxyUsername |proxyConfigBuilder.username | +|setProxyPassword |proxyConfigBuilder.password | +|setProxyDomain |proxyConfigBuilder.ntlmDomain, not supported in async client | +|setProxyWorkstation |proxyConfigBuilder.ntlmWorkstation, not supported in async client | +|setUserAgentPrefix |overrideConfigBuilder.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX, userAgent); | +|addHeader |overrideConfigBuilder.putHeader | +|setUseThrottleRetries |not supported | + +### Endpoint and region configuration + +Previously, if no endpoint and region was configured, fall back to using us-east-1. Set +withForceGlobalBucketAccessEnabled(true) which will allow access to buckets not in this region too. +Since the SDK V2 no longer supports cross region access, we need to set the region and endpoint of +the bucket. The behaviour has now been changed to: + +* If no endpoint is specified, use s3.amazonaws.com. +* When setting the endpoint, also set the protocol (HTTP or HTTPS) +* When setting the region, first initiate a default S3 Client with region eu-west-2. Call headBucket + using this client. If the bucket is also in eu-west-2, then this will return a successful + response. Otherwise it will throw an error with status code 301 permanently moved. This error + contains the region of the bucket in its header, which we can then use to configure the client. + +### List Object: + +There is no way to paginate the listObject V1 result, we are +doing [this](https://github.com/ahmarsuhail/hadoop/pull/23/files#diff-4050f95b7e3912145415b6e2f9cd3b0760fcf2ce96bf0980c6c30a6edad2d0fbR2745) +instead. We are trying to get pagination to listObject V1 in the SDK, but will have to use this +workaround for now. + +### EncryptionSecretOperations: + +Two new methods have been added, `getSSECustomerKey` and `getSSEAwsKMSKey`. Previously SDK V1 had +specific classes for these keys `SSECustomerKey` and `SSEAwsKeyManagementParams` . There are no such +classes with V2, and things need to be set manually. For this reason, we simply just return keys as +strings now. And will have to calculate and set md5’s ourselves when building the request. + + +### GetObjectMetadata: + +* `RequestFactory.newGetObjectMetadataRequest` is now `RequestFactory.newHeadObjectRequestBuilder`. +* In `HeaderProcessing.retrieveHeaders()`, called by `getXAttrs()`, + removed `maybeSetHeader(headers, XA_CONTENT_MD5, md.getContentMD5())` as S3 doesn’t ever actually + return an md5 header, regardless of whether you set it during a putObject. It does return + an `etag` which may or may not be an md5 depending on certain conditions. `getContentMD5()` is + always empty, there does not seem to be a need to set this header. +* `RequestFactoryImpl.setOptionalGetObjectMetadataParameters` : Method has been removed and this + logic has been moved to `RequestFactoryImpl.newHeadObjectRequestBuilder()` +* `RequestFactoryImpl.generateSSECustomerKey()` has been removed, and instead + call `EncryptionSecretOperations.createSSECustomerKey` directly in `newHeadObjectRequestBuilder()` + + + +### PutObject + +* Previously, when creating the `putObjectRequest`, you would also give it the data to be uploaded. + So it would be of the form `PutObjectRequest(bucket, key, file/inputstream)`, this is no longer + the case. Instead, the data now needs to be passed in while making the `s3Client.putObject()` + call. For this reason, the data is now part of + the `S3AFileSystem.putObject(putObjectRequest, file, listener)` + and `S3AFileSystem.putObjectDirect(putObjectRequest, putOptions, uploadData, isFile)`. +* `S3ADataBlocks`: Need to make this class public as it’s now used to pass in data + to `putObjectDirect()`, sometimes from outside the package (`MagicCommitTracker` + , `ITestS3AMiscOperations`). +* `ProgressableProgressListener`: You can no longer pass in the `Upload` while initialising the + listener + as `ProgressableProgressListener listener = new ProgressableProgressListener(this, key, upload, progress);` + The upload is now only available after initialising the listener, since the listener needs to be + initialised during creation of the Transfer Manager upload. Previously, you could create the + listener after the starting the TM upload, and attach it. +* The `Upload` is now passed into the progress listener later, + in `listener.uploadCompleted(uploadInfo.getFileUpload());`. +* `UploadInfo`: Previously, since the data to be uploaded was part of `putObjectRequest`, the + transfer manager only returned a single `Upload` type, which could be used to track the upload. + Now, depending on the upload type (eg: File or InputStream), it returns different types. This + class has been updated to return FileUpload info, as it’s only ever used for file uploads + currently. It can be extended to store different transfer types in the future. +* `WriteOperationHelper.createPutObjectRequest() `: Previously the data to be uploaded was part + of `PutObjectRequest`, and so we required two methods to create the request. One for input streams + and one for files. Since the data to be uploaded is no longer part of the request, but instead an + argument in `putObject` , we only need one method now. +* `WriteOperationHelper.newObjectMetadata()`: This method has been removed, as standard metadata, + instead of being part of the `ObjectMetadata`, is now just added while building the request, for + example `putObjectRequestBuilder.serverSideEncryption().` +* `RequestFactory`: Similar to WriteOperationHelper, there is now a single putObjectRequest, + and `newObjectMetadata` has been removed. Instead, all standard metadata is now set in the new + method `buildPutObjectRequest`. +* `RequestFactoryImpl.newObjectMetadata()`: Previously, object metadata was created + using `newObjectMetadata()` and passed into the `newPutObjectRequest()` call. This method has been + removed, as standard metadata, instead of being part of the `ObjectMetadata`, is now just added + while building the request, in `putObjectRequestBuilder.serverSideEncryption().` Content length + and content encoding set in this method is now set in `buildPutObjectRequest()` , and SSE is set + in `putEncryptionParameters()`. +* `RequestFactoryImpl.maybeSetMetadata()` : was a generic method to set user metadata on object + metadata. user metadata now gets set on the request builder, so method has been removed. +* `RequestFactoryImpl.setOptionalPutRequestParameters()` : Method has been removed, and this logic + has been moved to `putEncryptionParameters()` . + +### CopyObject + +* `RequestFactoryImpl.buildPutObjectRequest` : Destination metadata is no longer built + using `newObjectMetadata()` and instead set on the request builder. The logic has a couple of + differences: + * content encoding is set in `buildCopyObjectRequest`, + the `if (contentEncoding != null && !isDirectoryMarker)` can just + be `if (contentEncoding != null)` for copy, as for this `isDirectoryMarker` was always false. + * contentLength is not set, as this is a system defined header, and copied over automatically by + S3 during copy. +* `HeaderProcessing.cloneObjectMetadata`: This was previously also setting a lot of system defined + metadata, eg: `setHttpExpiresDate` and `setLastModified`. These have been removed as they are set + by S3 during the copy. Have tested, and can see they are set automatically regardless of the + metadataDirective (copy or replace). +* `RequestFactoryImpl. copyEncryptionParameters()` : Due to the changes + in `EncryptionSecretOperations`, source and destination encryption params have to be set manually. + +### MultipartUpload + +* `RequestFactoryImpl.newObjectMetdata()` : Metadata is now set on the request builder. For MPU, only +content encoding needs to be set, as per per previous behaviour. Encryption params are set +in ` multipartUploadEncryptionParameters`. + +### GetObject + +* Previously, GetObject returned a `S3Object` response which exposed its content in a + `S3ObjectInputStream` through the `getObjectContent()` method. In SDK v2, the response is + directly a `ResponseInputStream` with the content, while the + `GetObjectResponse` instance can be retrieved by calling `response()` on it. +* The above change simplifies managing the lifetime of the response input stream. In v1, + `S3AInputStream` had to keep a reference to the `S3Object` while holding the wrapped + `S3ObjectInputStream`. When upgraded to SDK v2, it can simply wrap the new + `ResponseInputStream`, which handles lifetime correctly. Same applies + to `SDKStreamDrainer`. Furthermore, the map in `S3ARemoteObject` associating input streams and + `S3Object` instances is no longer needed. +* The range header on a `GetObject` request is now specified as a string, rather than a + `start`-`end` pair. `S3AUtils.formatRange` was introduced to format it. + +### DeleteObject + +In SDK v1, bulk delete would throw a `com.amazonaws.services.s3.model.MultiObjectDeleteException` +in case of partial failure. In v2, instead, it returns a `DeleteObjectsResponse` containing a +list of errors. A new `MultiObjectDeleteException` class was introduced in +`org.apache.hadoop.fs.s3a` and is thrown when appropriate to reproduce the previous behaviour. +* `MultiObjectDeleteSupport.translateDeleteException` was moved into `MultiObjectDeleteException`. +* `ObjectIdentifier` replaces DeleteObjectsRequest.KeyVersion. + +### Select + +In SDK v2, Handling of select requests has changes significantly since SelectObjectContent is +only supported on the new async S3 client. In previous versions, the response to a +SelectObjectContent request exposed the results in a `SelectRecordsInputStream`, which S3A +could wrap in `SelectInputStream`. In v2, instead, the response needs to be handled by an object +implementing `SelectObjectContentResponseHandler`, which can receive an async publisher of +the "events" returned by the service (`SdkPublisher`). + +In order to adapt the new API in S3A, three new classes have been introduced in +`org.apache.hadoop.fs.s3a.select`: + +* `SelectObjectContentHelper`: wraps the `selectObjectContent()` call, provides a custom + response handler to receive the response, and exposes a `SelectEventStreamPublisher`. +* `SelectEventStreamPublisher`: a publisher of select event stream events, which handles the + future returned by the select call and wraps the original publisher. This class provides + a `toRecordsInputStream()` method which returns an input stream containing the results, + reproducing the behaviour of the old `SelectRecordsInputStream`. +* `BlockingEnumeration`: an adapter which lazily requests new elements from the publisher and + exposes them through an `Enumeration` interface. Used in + `SelectEventStreamPublisher.toRecordsInputStream()` to adapt the event publisher into + an enumeration of input streams, eventually passed to a `SequenceInputStream`. + Note that the "lazy" behaviour means that new elements are requested only on `read()` calls on + the input stream. + + + +### CredentialsProvider + +* All credential provider classes implemented in Hadoop now implement V2's `AwsCredentialProvider` +* New adapter class `org.apache.hadoop.fs.s3a.adapter.V1ToV2AwsCredentialProviderAdapter` has been + added. This converts SDK V1 credential providers to SDK V2’s which + implement `AwsCredentialsProvider`. +* `AWSCredentialProviderList` also implements `AwsCredentialProvider`. But keeps existing + constructors and add methods for V1 credential providers, and wraps V1 cred providers in the + adapter here. This means that custom binding classes in delegation tokens, as well as any custom + credential providers will continue to work. +* Added a new `getCredentials()` method in `AWSCredentialProviderList`, which ensured that custom + binding classes which are calling `AWSCredentialProviderList.getCredentials()`, continue to work. +* The following values `fs.s3a.aws.credentials.provider` are mapped: + as `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`, then map it to V2’s + +|`fs.s3a.aws.credentials.provider` value |Mapped to | +|---|---| +|`com.amazonaws.auth.EnvironmentVariableCredentialsProvider` |`software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | +|`com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper` |`org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | +|`com.amazonaws.auth.`InstanceProfileCredentialsProvider`` |`org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | + + +### Auditing + +The SDK v2 offers a new `ExecutionInterceptor` +[interface](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/core/interceptor/ExecutionInterceptor.html) +which broadly replaces the `RequestHandler2` abstract class from v1. +Switching to the new mechanism in S3A brings: + +* Simplification in `AWSAuditEventCallbacks` (and implementors) which can now extend + `ExecutionInterceptor` +* "Registering" a Span with a request has moved from `requestCreated` to `beforeExecution` + (where an `ExecutionAttributes` instance is first available) +* The ReferrerHeader is built and added to the http request in `modifyHttpRequest`, + rather than in `beforeExecution`, where no http request is yet available +* Dynamic loading of interceptors has been implemented to reproduce previous behaviour + with `RequestHandler2`s. The AWS SDK v2 offers an alternative mechanism, described + [here](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/core/interceptor/ExecutionInterceptor.html) + under "Interceptor Registration", which could make it redundant. + +In the Transfer Manager, `TransferListener` replaces `TransferStateChangeListener`. S3A code +has been updated and `AuditManagerS3A` implementations now provide an instance of the former to +switch to the active span, but registration of the new listeners is currently commented out because +it causes an incompatibility issue with the internal logger, resulting in `NoSuchMethodError`s, +at least in the current TransferManager Preview release. + + +### Metric Collection + +`AwsStatisticsCollector` has been updated to implement the new `MetricPublisher` interface +and collect the metrics from a `MetricCollection` object. +The following table maps SDK v2 metrics to their equivalent in v1: + +| v2 Metrics| com.amazonaws.util.AWSRequestMetrics.Field| Comment| +|-------------------------------------------------------------|---------------------------------------------|--------------------------------| +| CoreMetric.RETRY_COUNT| HttpClientRetryCount|| +| CoreMetric.RETRY_COUNT| RequestCount| always HttpClientRetryCount+1| +| HttpMetric.HTTP_STATUS_CODE with HttpStatusCode.THROTTLING| ThrottleException| to be confirmed| +| CoreMetric.API_CALL_DURATION| ClientExecuteTime|| +| CoreMetric.SERVICE_CALL_DURATION| HttpRequestTime|| +| CoreMetric.MARSHALLING_DURATION| RequestMarshallTime|| +| CoreMetric.SIGNING_DURATION| RequestSigningTime|| +| CoreMetric.UNMARSHALLING_DURATION| ResponseProcessingTime| to be confirmed| + +Note that none of the timing metrics (`*_DURATION`) are currently collected in S3A. + +### Exception Handling + +The code to handle exceptions thrown by the SDK has been updated to reflect the changes in v2: + +* `com.amazonaws.SdkBaseException` and `com.amazonaws.AmazonClientException` changes: + * These classes have combined and replaced with + `software.amazon.awssdk.core.exception.SdkException`. +* `com.amazonaws.SdkClientException` changes: + * This class has been replaced with `software.amazon.awssdk.core.exception.SdkClientException`. + * This class now extends `software.amazon.awssdk.core.exception.SdkException`. +* `com.amazonaws.AmazonServiceException` changes: + * This class has been replaced with + `software.amazon.awssdk.awscore.exception.AwsServiceException`. + * This class now extends `software.amazon.awssdk.core.exception.SdkServiceException`, + a new exception type that extends `software.amazon.awssdk.core.exception.SdkException`. + +See also the +[SDK changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#3-exception-changes). + + +### Failure Injection + +While using the SDK v1, failure injection was implemented in `InconsistentAmazonS3CClient`, +which extended the S3 client. In SDK v2, reproducing this approach would not be straightforward, +since the default S3 client is an internal final class. Instead, the same fault injection strategy +is now performed by a `FailureInjectionInterceptor` (see +[ExecutionInterceptor](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/core/interceptor/ExecutionInterceptor.html)) +registered on the default client by `InconsistentS3CClientFactory`. +`InconsistentAmazonS3CClient` has been removed. No changes to the user configuration are required. + diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md index 91f08bb730a50..43927723e365d 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md @@ -338,7 +338,7 @@ Here is the effective list of providers if none are declared: org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, - com.amazonaws.auth.EnvironmentVariableCredentialsProvider, + software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 052d52e4a8904..382ae36c1bf5b 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -249,56 +249,39 @@ a warning has been printed since Hadoop 2.8 whenever such a URL was used. ```xml fs.s3a.access.key - AWS access key ID. - Omit for IAM role-based or provider-based authentication. + AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication. fs.s3a.secret.key - AWS secret key. - Omit for IAM role-based or provider-based authentication. + AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication. - fs.s3a.aws.credentials.provider - - Comma-separated class names of credential provider classes which implement - com.amazonaws.auth.AWSCredentialsProvider. - - These are loaded and queried in sequence for a valid set of credentials. - Each listed class must implement one of the following means of - construction, which are attempted in order: - 1. a public constructor accepting java.net.URI and - org.apache.hadoop.conf.Configuration, - 2. a public static method named getInstance that accepts no - arguments and returns an instance of - com.amazonaws.auth.AWSCredentialsProvider, or - 3. a public default constructor. - - Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows - anonymous access to a publicly accessible S3 bucket without any credentials. - Please note that allowing anonymous access to an S3 bucket compromises - security and therefore is unsuitable for most use cases. It can be useful - for accessing public data sets without requiring AWS credentials. - - If unspecified, then the default list of credential provider classes, - queried in sequence, is: - 1. org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider: - Uses the values of fs.s3a.access.key and fs.s3a.secret.key. - 2. com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports - configuration of AWS access key ID and secret access key in - environment variables named AWS_ACCESS_KEY_ID and - AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use - of instance profile credentials if running in an EC2 VM. + fs.s3a.session.token + Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider + as one of the providers. - fs.s3a.session.token + fs.s3a.aws.credentials.provider + + org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, + software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider + - Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider - as one of the providers. + Comma-separated class names of credential provider classes which implement + software.amazon.awssdk.auth.credentials.AwsCredentialsProvider. + + When S3A delegation tokens are not enabled, this list will be used + to directly authenticate with S3 and other AWS services. + When S3A Delegation tokens are enabled, depending upon the delegation + token binding it may be used + to communicate wih the STS endpoint to request session/role + credentials. ``` @@ -350,13 +333,19 @@ credentials if they are defined. 1. The [AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment), are then looked for: these will return session or full credentials depending on which values are set. -1. An attempt is made to query the Amazon EC2 Instance Metadata Service to +1. An attempt is made to query the Amazon EC2 Instance/k8s container Metadata Service to retrieve credentials published to EC2 VMs. S3A can be configured to obtain client authentication providers from classes -which integrate with the AWS SDK by implementing the `com.amazonaws.auth.AWSCredentialsProvider` -Interface. This is done by listing the implementation classes, in order of +which integrate with the AWS SDK by implementing the +`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider` +interface. +This is done by listing the implementation classes, in order of preference, in the configuration option `fs.s3a.aws.credentials.provider`. +In previous hadoop releases, providers were required to +implement the AWS V1 SDK interface `com.amazonaws.auth.AWSCredentialsProvider`. +Consult the [Upgrading S3A to AWS SDK V2](./aws_sdk_upgrade.html) documentation +to see how to migrate credential providers. *Important*: AWS Credential Providers are distinct from _Hadoop Credential Providers_. As will be covered later, Hadoop Credential Providers allow passwords and other secrets @@ -371,21 +360,23 @@ this is advised as a more secure way to store valuable secrets. There are a number of AWS Credential Providers inside the `hadoop-aws` JAR: -| classname | description | -|-----------|-------------| -| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials | -| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials | -| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login | -| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider<`| [Assumed Role credentials](assumed_roles.html) | +| Hadoop module credential provider | Authentication Mechanism | +|----------------------------------------------------------------|--------------------------------------------------| +| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider` | Session Credentials in configuration | +| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` | Simple name/secret credentials in configuration | +| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` | Anonymous Login | +| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider` | [Assumed Role credentials](./assumed_roles.html) | +| `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials | -There are also many in the Amazon SDKs, in particular two which are automatically -set up in the authentication chain: +There are also many in the Amazon SDKs, with the common ones being. | classname | description | |-----------|-------------| -| `com.amazonaws.auth.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials | -| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables | +| `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables | +| `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials | +| `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider`| EC2/k8s Metadata Credentials | + ### EC2 IAM Metadata Authentication with `InstanceProfileCredentialsProvider` @@ -402,7 +393,7 @@ You can configure Hadoop to authenticate to AWS using a [named profile](https:// To authenticate with a named profile: -1. Declare `com.amazonaws.auth.profile.ProfileCredentialsProvider` as the provider. +1. Declare `software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider` as the provider. 1. Set your profile via the `AWS_PROFILE` environment variable. 1. Due to a [bug in version 1 of the AWS Java SDK](https://github.com/aws/aws-sdk-java/issues/803), you'll need to remove the `profile` prefix from the AWS configuration section heading. @@ -525,50 +516,9 @@ This means that the default S3A authentication chain can be defined as org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, - com.amazonaws.auth.EnvironmentVariableCredentialsProvider, + software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider - - Comma-separated class names of credential provider classes which implement - com.amazonaws.auth.AWSCredentialsProvider. - - When S3A delegation tokens are not enabled, this list will be used - to directly authenticate with S3 and other AWS services. - When S3A Delegation tokens are enabled, depending upon the delegation - token binding it may be used - to communicate with the STS endpoint to request session/role - credentials. - - These are loaded and queried in sequence for a valid set of credentials. - Each listed class must implement one of the following means of - construction, which are attempted in order: - * a public constructor accepting java.net.URI and - org.apache.hadoop.conf.Configuration, - * a public constructor accepting org.apache.hadoop.conf.Configuration, - * a public static method named getInstance that accepts no - arguments and returns an instance of - com.amazonaws.auth.AWSCredentialsProvider, or - * a public default constructor. - - Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows - anonymous access to a publicly accessible S3 bucket without any credentials. - Please note that allowing anonymous access to an S3 bucket compromises - security and therefore is unsuitable for most use cases. It can be useful - for accessing public data sets without requiring AWS credentials. - - If unspecified, then the default list of credential provider classes, - queried in sequence, is: - * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks - for session login secrets in the Hadoop configuration. - * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider: - Uses the values of fs.s3a.access.key and fs.s3a.secret.key. - * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports - configuration of AWS access key ID and secret access key in - environment variables named AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, - and AWS_SESSION_TOKEN as documented in the AWS SDK. - * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up - IAM credentials of any EC2 VM or AWS container in which the process is running. - ``` @@ -1415,7 +1365,7 @@ role information available when deployed in Amazon EC2. ```xml fs.s3a.aws.credentials.provider - com.amazonaws.auth.InstanceProfileCredentialsProvider + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider ``` @@ -2155,7 +2105,7 @@ If no custom signers are being used - this value does not need to be set. `SignerName:SignerClassName` - register a new signer with the specified name, and the class for this signer. -The Signer Class must implement `com.amazonaws.auth.Signer`. +The Signer Class must implement `software.amazon.awssdk.core.signer.Signer`. `SignerName:SignerClassName:SignerInitializerClassName` - similar time above except also allows for a custom SignerInitializer diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md index 886a2d97d246f..d18d07b9189af 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md @@ -14,7 +14,7 @@ # S3 Select -**Experimental Feature** +**Deprecated Feature** @@ -60,6 +60,20 @@ Record Readers. It's better here to directly use the Apache Spark, Hive, Impala, Flink or similar, which all use the latest ASF-supported libraries. +## Dependencies: eventstream JAR + +To use S3 Select through the S3A connector, an extra JAR MUST be added to the classpath of your application, +`eventstream-1.0.1.jar`.a +For command line tool use, this should be done by adding it to `share/hadoop/common/lib/` + +```xml + + software.amazon.eventstream + eventstream + 1.0.1 + +``` + ## Enabling/Disabling S3 Select S3 Select is enabled by default: @@ -288,10 +302,12 @@ hadoop s3guard \ ``` -## Use in MR/Analytics queries: Work in Progress +## Use in MR/Analytics queries: Partially Supported -S3 Select support in analytics queries is a work in progress. It does -not work reliably with large source files where the work is split up. +S3 Select support in analytics queries is only partially supported. +It does not work reliably with large source files where the work is split up, +and as the various query engines all assume that .csv and .json formats are splittable, +things go very wrong, fast. As a proof of concept *only*, S3 Select queries can be made through MapReduce jobs which use any Hadoop `RecordReader` @@ -663,6 +679,24 @@ to the `get()` call: do it. ## Troubleshooting +### `NoClassDefFoundError: software/amazon/eventstream/MessageDecoder` + +Select operation failing with a missing eventstream class. + +``` +java.io.IOException: java.lang.NoClassDefFoundError: software/amazon/eventstream/MessageDecoder +at org.apache.hadoop.fs.s3a.select.SelectObjectContentHelper.select(SelectObjectContentHelper.java:75) +at org.apache.hadoop.fs.s3a.WriteOperationHelper.lambda$select$10(WriteOperationHelper.java:660) +at org.apache.hadoop.fs.store.audit.AuditingFunctions.lambda$withinAuditSpan$0(AuditingFunctions.java:62) +at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122) +``` + +The eventstream JAR is not on the classpath/not in sync with the version of the full "bundle.jar" JDK + +Fix: get a compatible version of the JAR on the classpath. + +### SQL errors + Getting S3 Select code to work is hard, though those knowledgeable in SQL will find it easier. @@ -673,7 +707,6 @@ Problems can be split into: 1. Datatype casting issues 1. Bad records/data in source files. 1. Failure to configure MR jobs to work correctly. -1. Failure of MR jobs due to The exceptions here are all based on the experience during writing tests; more may surface with broader use. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index ffb2982b1ccfc..bfec94b19c101 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -1004,71 +1004,19 @@ using an absolute XInclude reference to it. ## Failure Injection -**Warning do not enable any type of failure injection in production. The -following settings are for testing only.** +S3A provides an "Inconsistent S3 Client Factory" that can be used to +simulate throttling by injecting random failures on S3 client requests. -One of the challenges with S3A integration tests was the fact that S3 was an -eventually-consistent storage system. To simulate inconsistencies more -frequently than they would normally surface, S3A supports a shim layer on top of the `AmazonS3Client` -class which artificially delays certain paths from appearing in listings. -This is implemented in the class `InconsistentAmazonS3Client`. -Now that S3 is consistent, injecting inconsistency is no longer needed -during testing. -However, it is stil useful to use the other feature of the client: -throttling simulation. +**Note** -## Simulating List Inconsistencies +In previous releases, this factory could also be used to simulate +inconsistencies during testing of S3Guard. Now that S3 is consistent, +injecting inconsistency is no longer needed during testing. -### Enabling the InconsistentAmazonS3CClient -To enable the fault-injecting client via configuration, switch the -S3A client to use the "Inconsistent S3 Client Factory" when connecting to -S3: - -```xml - - fs.s3a.s3.client.factory.impl - org.apache.hadoop.fs.s3a.InconsistentS3ClientFactory - -``` - -The inconsistent client will, on every AWS SDK request, -generate a random number, and if less than the probability, -raise a 503 exception. - -```xml - - - fs.s3a.failinject.throttle.probability - 0.05 - -``` - -These exceptions are returned to S3; they do not test the -AWS SDK retry logic. - - -### Using the `InconsistentAmazonS3CClient` in downstream integration tests - -The inconsistent client is shipped in the `hadoop-aws` JAR, so it can -be used in integration tests. - -## Testing S3Guard - -As part of the removal of S3Guard from the production code, the tests have been updated -so that - -* All S3Guard-specific tests have been deleted. -* All tests parameterized on S3Guard settings have had those test configurations removed. -* The maven profiles option to run tests with S3Guard have been removed. - -There is no need to test S3Guard -and so tests are lot faster. -(We developers are all happy) - - -## Testing Assumed Roles +## Testing Assumed Roles Tests for the AWS Assumed Role credential provider require an assumed role to request. @@ -1289,10 +1237,13 @@ time bin/hadoop fs -copyToLocal -t 10 $BUCKET/\*aws\* tmp # --------------------------------------------------- # S3 Select on Landsat +# this will fail with a ClassNotFoundException unless +# eventstore JAR is added to the classpath # --------------------------------------------------- export LANDSATGZ=s3a://landsat-pds/scene_list.gz + bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \ "SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100" diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 1ead08081f158..41351cdf1bdf0 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -70,14 +70,45 @@ These are Hadoop filesystem client classes, found in the `hadoop-aws` JAR. An exception reporting this class as missing means that this JAR is not on the classpath. -### `ClassNotFoundException: com.amazonaws.services.s3.AmazonS3Client` -(or other `com.amazonaws` class.) +### `NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest` + +The library `aws-crt.jar` is not on the classpath. Its classes +are not in the AWS `bundle.jar` file, yet may be needed by some uses made +of the SDK. + +Fix: add. + +``` +java.lang.BootstrapMethodError: java.lang.NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest +at software.amazon.awssdk.services.s3.internal.crt.S3MetaRequestPauseObservable.(S3MetaRequestPauseObservable.java:33) +at software.amazon.awssdk.transfer.s3.internal.DefaultS3TransferManager.uploadFile(DefaultS3TransferManager.java:205) +at org.apache.hadoop.fs.s3a.S3AFileSystem.putObject(S3AFileSystem.java:3064) +at org.apache.hadoop.fs.s3a.S3AFileSystem.executePut(S3AFileSystem.java:4054) + +``` +### `ClassNotFoundException: software.amazon.awssdk.services.s3.S3Client` + +(or other `software.amazon` class.) -This means that the `aws-java-sdk-bundle.jar` JAR is not on the classpath: +This means that the AWS V2 SDK `bundle.jar` JAR is not on the classpath: add it. -### `java.lang.NoSuchMethodError` referencing a `com.amazonaws` class +### `ClassNotFoundException: com.amazonaws.auth.AWSCredentials` + +(or other `com.amazonaws` class.) + +With the move to the [V2 AWS SDK](../aws_sdk_upgrade.html), +the v1 SDK classes are no longer on the classpath. + +If this happens when trying to use a custom credential provider defined +in `fs.s3a.aws.credentials.provider`, then add the `aws-sdk-bundle.jar` +JAR to the classpath. + +If this happens in your own/third-party code, then again, add the JAR, +and/or consider moving to the v2 sdk yourself. + +### `java.lang.NoSuchMethodError` referencing a `software.amazon` class This can be triggered by incompatibilities between the AWS SDK on the classpath and the version which Hadoop was compiled with. @@ -86,12 +117,14 @@ The AWS SDK JARs change their signature enough between releases that the only way to safely update the AWS SDK version is to recompile Hadoop against the later version. -The sole fix is to use the same version of the AWS SDK with which Hadoop +The fix is to use the same version of the AWS SDK with which Hadoop was built. This can also be caused by having more than one version of an AWS SDK -JAR on the classpath. If the full `aws-java-sdk-bundle<` JAR is on the -classpath, do not add any of the `aws-sdk-` JARs. +JAR on the classpath. If the full `bundle.jar` JAR is on the +classpath, do not add any of the `aws-sdk-` JARs *except* for +`aws-crt.jar` (which is required) and +`eventstream.jar` which is required when using S3 Select. ### `java.lang.NoSuchMethodError` referencing an `org.apache.hadoop` class @@ -2010,51 +2043,3 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request: When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it completely by setting it to `0`. - -## SDK Upgrade Warnings - -S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2). -For more information on the upgrade and what's changing, see -[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). - -S3A logs the following warnings for things that will be changing in the upgrade. To disable these -logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties. - -### `Directly referencing AWS SDK V1 credential provider` - -This will be logged when an AWS credential provider is referenced directly in -`fs.s3a.aws.credentials.provider`. -For example, `com.amazonaws.auth.AWSSessionCredentialsProvider` - -To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`. -Instead, use S3A's credential providers. - -### `getAmazonS3ClientForTesting() will be removed` - -This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2, -the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to -`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed. - -### -### `Custom credential providers used in delegation tokens binding classes will need to be updated` - -This will be logged when delegation tokens are used. -Delegation tokens allow the use of custom binding classes which can implement custom credential -providers. -These credential providers will currently be implementing -`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement -`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. - -### -### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated` - -This will be logged when a custom signer is used. -Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be -updated to implement `software.amazon.awssdk.core.signer.Signer`. - -### -### `getObjectMetadata() called. This operation and it's response will be changed` - -This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to -`headObject()` and will return a response of the type `HeadObjectResponse`. - diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java index a46303f339678..734bcfd9c5d30 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java @@ -20,8 +20,9 @@ import static org.apache.hadoop.fs.s3a.Constants.*; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.S3Client; import java.net.URI; @@ -32,6 +33,7 @@ import org.junit.Rule; import org.junit.rules.ExpectedException; + /** * Abstract base class for S3A unit tests using a mock S3 client and a null * metadata store. @@ -39,17 +41,20 @@ public abstract class AbstractS3AMockTest { protected static final String BUCKET = "mock-bucket"; - protected static final AmazonServiceException NOT_FOUND; - static { - NOT_FOUND = new AmazonServiceException("Not Found"); - NOT_FOUND.setStatusCode(404); - } + protected static final AwsServiceException NOT_FOUND = + AwsServiceException.builder() + .message("Not Found") + .statusCode(404) + .awsErrorDetails(AwsErrorDetails.builder() + .errorCode("") + .build()) + .build(); @Rule public ExpectedException exception = ExpectedException.none(); protected S3AFileSystem fs; - protected AmazonS3 s3; + protected S3Client s3; @Before public void setup() throws Exception { @@ -59,10 +64,9 @@ public void setup() throws Exception { // unset S3CSE property from config to avoid pathIOE. conf.unset(Constants.S3_ENCRYPTION_ALGORITHM); fs.initialize(uri, conf); - s3 = fs.getAmazonS3ClientForTesting("mocking"); + s3 = fs.getS3AInternals().getAmazonS3Client("mocking"); } - @SuppressWarnings("deprecation") public Configuration createConfiguration() { Configuration conf = new Configuration(); conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, @@ -75,9 +79,15 @@ public Configuration createConfiguration() { // assertions to be safely made without worrying // about any race conditions conf.setInt(ASYNC_DRAIN_THRESHOLD, Integer.MAX_VALUE); + // set the region to avoid the getBucketLocation on FS init. + conf.set(AWS_REGION, "eu-west-1"); return conf; } + public S3Client getS3Client() { + return s3; + } + @After public void teardown() throws Exception { if (fs != null) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index e90ad8b73efae..93f41cfaa81bb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -210,6 +210,14 @@ public S3AFileSystem getFileSystem() { return (S3AFileSystem) super.getFileSystem(); } + /** + * Get the {@link S3AInternals} internal access for the + * test filesystem. + * @return internals. + */ + public S3AInternals getS3AInternals() { + return getFileSystem().getS3AInternals(); + } /** * Describe a test in the logs. * @param text text to print diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java index 4013e9db29a3e..8d927dc957b16 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java @@ -20,7 +20,7 @@ import java.io.IOException; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.net.util.Base64; @@ -69,33 +69,33 @@ public static void assertEncrypted(S3AFileSystem fs, final S3AEncryptionMethods algorithm, final String kmsKeyArn) throws IOException { - ObjectMetadata md = fs.getObjectMetadata(path); + HeadObjectResponse md = fs.getS3AInternals().getObjectMetadata(path); String details = String.format( "file %s with encryption algorithm %s and key %s", path, - md.getSSEAlgorithm(), - md.getSSEAwsKmsKeyId()); + md.serverSideEncryptionAsString(), + md.ssekmsKeyId()); switch(algorithm) { case SSE_C: assertNull("Metadata algorithm should have been null in " + details, - md.getSSEAlgorithm()); + md.serverSideEncryptionAsString()); assertEquals("Wrong SSE-C algorithm in " + details, - SSE_C_ALGORITHM, md.getSSECustomerAlgorithm()); + SSE_C_ALGORITHM, md.sseCustomerAlgorithm()); String md5Key = convertKeyToMd5(fs); assertEquals("getSSECustomerKeyMd5() wrong in " + details, - md5Key, md.getSSECustomerKeyMd5()); + md5Key, md.sseCustomerKeyMD5()); break; case SSE_KMS: assertEquals("Wrong algorithm in " + details, - AWS_KMS_SSE_ALGORITHM, md.getSSEAlgorithm()); + AWS_KMS_SSE_ALGORITHM, md.serverSideEncryptionAsString()); assertEquals("Wrong KMS key in " + details, kmsKeyArn, - md.getSSEAwsKmsKeyId()); + md.ssekmsKeyId()); break; default: - assertEquals("AES256", md.getSSEAlgorithm()); + assertEquals("AES256", md.serverSideEncryptionAsString()); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index c13c3f48b8466..bccbe79c2a48b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -26,27 +26,30 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; +import org.assertj.core.api.Assertions; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getCSVTestPath; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; -import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.*; /** - * Integration tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic. + * Integration tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic + * through the S3A Filesystem instantiation process. */ public class ITestS3AAWSCredentialsProvider { private static final Logger LOG = @@ -55,17 +58,21 @@ public class ITestS3AAWSCredentialsProvider { @Rule public Timeout testTimeout = new Timeout(60_1000, TimeUnit.MILLISECONDS); + /** + * Expecting a wrapped ClassNotFoundException. + */ @Test - public void testBadConfiguration() throws IOException { - Configuration conf = createConf(); - conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class"); - try { - createFailingFS(conf); - } catch (IOException e) { - if (!(e.getCause() instanceof ClassNotFoundException)) { - LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e); - throw e; - } + public void testProviderClassNotFound() throws Exception { + Configuration conf = createConf("no.such.class"); + final InstantiationIOException e = + intercept(InstantiationIOException.class, "java.lang.ClassNotFoundException", () -> + createFailingFS(conf)); + if (InstantiationIOException.Kind.InstantiationFailure != e.getKind()) { + throw e; + } + if (!(e.getCause() instanceof ClassNotFoundException)) { + LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e); + throw e; } } @@ -73,105 +80,144 @@ public void testBadConfiguration() throws IOException { * A bad CredentialsProvider which has no suitable constructor. * * This class does not provide a public constructor accepting Configuration, - * or a public factory method named getInstance that accepts no arguments, + * or a public factory method named create() that accepts no arguments, * or a public default constructor. */ - static class BadCredentialsProviderConstructor - implements AWSCredentialsProvider { + public static class BadCredentialsProviderConstructor + implements AwsCredentialsProvider { @SuppressWarnings("unused") public BadCredentialsProviderConstructor(String fsUri, Configuration conf) { } @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials("dummy_key", "dummy_secret"); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("dummy_key", "dummy_secret"); } - @Override - public void refresh() { - } } @Test public void testBadCredentialsConstructor() throws Exception { - Configuration conf = createConf(); - conf.set(AWS_CREDENTIALS_PROVIDER, - BadCredentialsProviderConstructor.class.getName()); - try { - createFailingFS(conf); - } catch (IOException e) { - GenericTestUtils.assertExceptionContains(CONSTRUCTOR_EXCEPTION, e); + Configuration conf = createConf(BadCredentialsProviderConstructor.class); + final InstantiationIOException ex = + intercept(InstantiationIOException.class, CONSTRUCTOR_EXCEPTION, () -> + createFailingFS(conf)); + if (InstantiationIOException.Kind.UnsupportedConstructor != ex.getKind()) { + throw ex; } } - protected Configuration createConf() { + /** + * Create a configuration bonded to the given provider classname. + * @param provider provider to bond to + * @return a configuration + */ + protected Configuration createConf(String provider) { Configuration conf = new Configuration(); removeBaseAndBucketOverrides(conf, DELEGATION_TOKEN_BINDING, AWS_CREDENTIALS_PROVIDER); + conf.set(AWS_CREDENTIALS_PROVIDER, provider); + conf.set(DELEGATION_TOKEN_BINDING, ""); return conf; } + /** + * Create a configuration bonded to the given provider class. + * @param provider provider to bond to + * @return a configuration + */ + protected Configuration createConf(Class provider) { + return createConf(provider.getName()); + } + /** * Create a filesystem, expect it to fail by raising an IOException. * Raises an assertion exception if in fact the FS does get instantiated. + * The FS is always deleted. * @param conf configuration * @throws IOException an expected exception. */ private void createFailingFS(Configuration conf) throws IOException { - S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf); - fs.listStatus(new Path("/")); - fail("Expected exception - got " + fs); + try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) { + fs.listStatus(new Path("/")); + fail("Expected exception - got " + fs); + } } - static class BadCredentialsProvider implements AWSCredentialsProvider { + /** + * Returns an invalid set of credentials. + */ + public static class BadCredentialsProvider implements AwsCredentialsProvider { @SuppressWarnings("unused") public BadCredentialsProvider(Configuration conf) { } @Override - public AWSCredentials getCredentials() { - return new BasicAWSCredentials("bad_key", "bad_secret"); + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("bad_key", "bad_secret"); } - @Override - public void refresh() { - } } @Test public void testBadCredentials() throws Exception { - Configuration conf = new Configuration(); - conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProvider.class.getName()); - try { - createFailingFS(conf); - } catch (AccessDeniedException e) { - // expected - } catch (AWSServiceIOException e) { - GenericTestUtils.assertExceptionContains( - "UnrecognizedClientException", e); - // expected - } + Configuration conf = createConf(BadCredentialsProvider.class); + intercept(AccessDeniedException.class, "", () -> + createFailingFS(conf)); } + /** + * Test using the anonymous credential provider with the public csv + * test file; if the test file path is unset then it will be skipped. + */ @Test - @SuppressWarnings("deprecation") public void testAnonymousProvider() throws Exception { - Configuration conf = new Configuration(); - conf.set(AWS_CREDENTIALS_PROVIDER, - AnonymousAWSCredentialsProvider.class.getName()); + Configuration conf = createConf(AnonymousAWSCredentialsProvider.class); Path testFile = getCSVTestPath(conf); try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) { - assertNotNull("S3AFileSystem instance must not be null", fs); - assertTrue("FileSystem must be the instance of S3AFileSystem", fs instanceof S3AFileSystem); + Assertions.assertThat(fs) + .describedAs("Filesystem") + .isNotNull(); FileStatus stat = fs.getFileStatus(testFile); - assertNotNull("FileStatus with qualified path must not be null", stat); assertEquals( "The qualified path returned by getFileStatus should be same as the original file", testFile, stat.getPath()); } } + /** + * Create credentials via the create() method. + * They are invalid credentials, so IO will fail as access denied. + */ + @Test + public void testCredentialsWithCreateMethod() throws Exception { + Configuration conf = createConf(CredentialsProviderWithCreateMethod.class); + intercept(AccessDeniedException.class, "", () -> + createFailingFS(conf)); + } + + /** + * Credentials via the create() method. + */ + public static final class CredentialsProviderWithCreateMethod implements AwsCredentialsProvider { + + public static AwsCredentialsProvider create() { + LOG.info("creating CredentialsProviderWithCreateMethod"); + return new CredentialsProviderWithCreateMethod(); + } + + /** Private: cannot be created directly. */ + private CredentialsProviderWithCreateMethod() { + } + + @Override + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("bad_key", "bad_secret"); + } + + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java index 9485202f64cb4..38c4685eb137e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java @@ -36,8 +36,10 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED; +import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A; import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -124,12 +126,17 @@ public static void expectUnknownStore( private Configuration createConfigurationWithProbe(final int probe) { Configuration conf = new Configuration(getFileSystem().getConf()); S3ATestUtils.disableFilesystemCaching(conf); + removeBaseAndBucketOverrides(conf, + S3A_BUCKET_PROBE, + ENDPOINT, + AWS_REGION); conf.setInt(S3A_BUCKET_PROBE, probe); + conf.set(AWS_REGION, EU_WEST_1); return conf; } @Test - public void testBucketProbingV1() throws Exception { + public void testBucketProbing() throws Exception { describe("Test the V1 bucket probe"); Configuration configuration = createConfigurationWithProbe(1); expectUnknownStore( @@ -137,18 +144,24 @@ public void testBucketProbingV1() throws Exception { } @Test - public void testBucketProbingV2() throws Exception { - describe("Test the V2 bucket probe"); + public void testBucketProbing2() throws Exception { + describe("Test the bucket probe with probe value set to 2"); Configuration configuration = createConfigurationWithProbe(2); + expectUnknownStore( () -> FileSystem.get(uri, configuration)); - /* - * Bucket probing should also be done when value of - * S3A_BUCKET_PROBE is greater than 2. - */ - configuration.setInt(S3A_BUCKET_PROBE, 3); - expectUnknownStore( - () -> FileSystem.get(uri, configuration)); + } + + @Test + public void testBucketProbing3() throws Exception { + describe("Test the bucket probe with probe value set to 3"); + Configuration configuration = createConfigurationWithProbe(3); + fs = FileSystem.get(uri, configuration); + Path root = new Path(uri); + + assertTrue("root path should always exist", fs.exists(root)); + assertTrue("getFileStatus on root should always return a directory", + fs.getFileStatus(root).isDirectory()); } @Test @@ -162,8 +175,8 @@ public void testBucketProbingParameterValidation() throws Exception { } @Test - public void testAccessPointProbingV2() throws Exception { - describe("Test V2 bucket probing using an AccessPoint ARN"); + public void testAccessPointProbing2() throws Exception { + describe("Test bucket probing using probe value 2, and an AccessPoint ARN"); Configuration configuration = createArnConfiguration(); String accessPointArn = "arn:aws:s3:eu-west-1:123456789012:accesspoint/" + randomBucket; configuration.set(String.format(InternalConstants.ARN_BUCKET_OPTION, randomBucket), @@ -175,7 +188,7 @@ public void testAccessPointProbingV2() throws Exception { @Test public void testAccessPointRequired() throws Exception { - describe("Test V2 bucket probing with 'fs.s3a.accesspoint.required' property."); + describe("Test bucket probing with 'fs.s3a.accesspoint.required' property."); Configuration configuration = createArnConfiguration(); configuration.set(AWS_S3_ACCESSPOINT_REQUIRED, "true"); intercept(PathIOException.class, @@ -197,7 +210,7 @@ public void testAccessPointRequired() throws Exception { */ private Configuration createArnConfiguration() { Configuration configuration = createConfigurationWithProbe(2); - configuration.set(AWS_REGION, "eu-west-1"); + configuration.set(AWS_REGION, EU_WEST_1); return configuration; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java index 1071582cc67d2..1c6fb9c3ab6a3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java @@ -20,11 +20,13 @@ import java.util.List; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.Grant; -import com.amazonaws.services.s3.model.GroupGrantee; -import com.amazonaws.services.s3.model.Permission; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectAclRequest; +import software.amazon.awssdk.services.s3.model.GetObjectAclResponse; +import software.amazon.awssdk.services.s3.model.Grant; +import software.amazon.awssdk.services.s3.model.Grantee; +import software.amazon.awssdk.services.s3.model.Permission; +import software.amazon.awssdk.services.s3.model.Type; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; @@ -55,7 +57,6 @@ protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); removeBaseAndBucketOverrides(conf, CANNED_ACL); - conf.set(CANNED_ACL, LOG_DELIVERY_WRITE); // needed because of direct calls made conf.setBoolean(S3AAuditConstants.REJECT_OUT_OF_SPAN_OPERATIONS, false); @@ -89,18 +90,26 @@ private void assertObjectHasLoggingGrant(Path path, boolean isFile) { S3AFileSystem fs = getFileSystem(); StoreContext storeContext = fs.createStoreContext(); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("acls"); + S3Client s3 = getS3AInternals().getAmazonS3Client("acls"); String key = storeContext.pathToKey(path); if (!isFile) { key = key + "/"; } - AccessControlList acl = s3.getObjectAcl(storeContext.getBucket(), - key); - List grants = acl.getGrantsAsList(); + GetObjectAclResponse acl = s3.getObjectAcl(GetObjectAclRequest.builder() + .bucket(storeContext.getBucket()) + .key(key) + .build()); + List grants = acl.grants(); for (Grant grant : grants) { LOG.info("{}", grant.toString()); } - Grant loggingGrant = new Grant(GroupGrantee.LogDelivery, Permission.Write); + Grant loggingGrant = Grant.builder() + .grantee(Grantee.builder() + .type(Type.GROUP) + .uri("http://acs.amazonaws.com/groups/s3/LogDelivery") + .build()) + .permission(Permission.WRITE) + .build(); Assertions.assertThat(grants) .describedAs("ACL grants of object %s", path) .contains(loggingGrant); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionKms.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionKms.java index bcc37c8bfbbba..4f1dcdfd5238b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionKms.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionKms.java @@ -21,11 +21,11 @@ import java.io.IOException; import java.util.Map; -import com.amazonaws.services.s3.Headers; import org.assertj.core.api.Assertions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.impl.AWSHeaders; import org.apache.hadoop.fs.s3a.impl.HeaderProcessing; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; @@ -69,14 +69,14 @@ protected void assertEncrypted(Path path) throws IOException { // Assert KeyWrap Algo assertEquals("Key wrap algo isn't same as expected", KMS_KEY_WRAP_ALGO, processHeader(fsXAttrs, - xAttrPrefix + Headers.CRYPTO_KEYWRAP_ALGORITHM)); + xAttrPrefix + AWSHeaders.CRYPTO_KEYWRAP_ALGORITHM)); // Assert content encryption algo for KMS, is present in the // materials description and KMS key ID isn't. String keyId = getS3EncryptionKey(getTestBucketName(getConfiguration()), getConfiguration()); Assertions.assertThat(processHeader(fsXAttrs, - xAttrPrefix + Headers.MATERIALS_DESCRIPTION)) + xAttrPrefix + AWSHeaders.MATERIALS_DESCRIPTION)) .describedAs("Materials Description should contain the content " + "encryption algo and should not contain the KMS keyID.") .contains(KMS_CONTENT_ENCRYPTION_ALGO) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index ff75f6e26138d..5570efe641004 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,10 +18,28 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.S3ClientOptions; +import java.io.File; +import java.net.URI; +import java.nio.file.AccessDeniedException; +import java.security.PrivilegedExceptionAction; +import org.assertj.core.api.Assertions; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.client.config.SdkClientConfiguration; +import software.amazon.awssdk.core.client.config.SdkClientOption; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.StsException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -29,31 +47,19 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.auth.STSClientFactory; import org.apache.hadoop.fs.s3native.S3xLoginHelper; -import org.apache.hadoop.test.GenericTestUtils; - -import org.assertj.core.api.Assertions; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.File; -import java.net.URI; -import java.security.PrivilegedExceptionAction; - import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.http.HttpStatus; -import org.junit.rules.TemporaryFolder; +import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.EU_WEST_1; import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -84,6 +90,23 @@ public class ITestS3AConfiguration { @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); + /** + * Get the S3 client of the active filesystem. + * @param reason why? + * @return the client + */ + private S3Client getS3Client(String reason) { + return requireNonNull(getS3AInternals().getAmazonS3Client(reason)); + } + + /** + * Get the internals of the active filesystem. + * @return the internals + */ + private S3AInternals getS3AInternals() { + return fs.getS3AInternals(); + } + /** * Test if custom endpoint is picked up. *

    @@ -111,7 +134,6 @@ public void testEndpoint() throws Exception { } else { conf.set(Constants.ENDPOINT, endpoint); fs = S3ATestUtils.createTestFileSystem(conf); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("test endpoint"); String endPointRegion = ""; // Differentiate handling of "s3-" and "s3." based endpoint identifiers String[] endpointParts = StringUtils.split(endpoint, '.'); @@ -122,8 +144,9 @@ public void testEndpoint() throws Exception { } else { fail("Unexpected endpoint"); } + String region = getS3AInternals().getBucketLocation(); assertEquals("Endpoint config setting and bucket location differ: ", - endPointRegion, s3.getBucketLocation(fs.getUri().getHost())); + endPointRegion, region); } } @@ -149,7 +172,7 @@ protected void useFailFastConfiguration() { } /** - * Expect a filesystem to not be created from a configuration + * Expect a filesystem to not be created from a configuration. * @return the exception intercepted * @throws Exception any other exception */ @@ -348,22 +371,24 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() try { fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("configuration"); - assertNotNull(s3); - S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class, - "clientOptions"); + S3Client s3 = getS3Client("configuration"); + + SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class, + "clientConfiguration"); + S3Configuration s3Configuration = + (S3Configuration)clientConfiguration.option(SdkClientOption.SERVICE_CONFIGURATION); assertTrue("Expected to find path style access to be switched on!", - clientOptions.isPathStyleAccess()); + s3Configuration.pathStyleAccessEnabled()); byte[] file = ContractTestUtils.toAsciiByteArray("test file"); ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length, (int) conf.getLongBytes(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true); - } catch (final AWSS3IOException e) { + } catch (final AWSRedirectException e) { LOG.error("Caught exception: ", e); // Catch/pass standard path style access behaviour when live bucket // isn't in the same region as the s3 client default. See // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html - assertEquals(HttpStatus.SC_MOVED_PERMANENTLY, e.getStatusCode()); + assertEquals(HttpStatus.SC_MOVED_PERMANENTLY, e.statusCode()); } catch (final IllegalArgumentException e) { // Path style addressing does not work with AP ARNs if (!fs.getBucket().contains("arn:")) { @@ -380,12 +405,12 @@ public void testDefaultUserAgent() throws Exception { conf = new Configuration(); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("User Agent"); - assertNotNull(s3); - ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, + S3Client s3 = getS3Client("User Agent"); + SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class, "clientConfiguration"); - assertEquals("Hadoop " + VersionInfo.getVersion(), - awsConf.getUserAgentPrefix()); + Assertions.assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT)) + .describedAs("User Agent prefix") + .startsWith("Hadoop " + VersionInfo.getVersion()); } @Test @@ -394,12 +419,12 @@ public void testCustomUserAgent() throws Exception { conf.set(Constants.USER_AGENT_PREFIX, "MyApp"); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("User agent"); - assertNotNull(s3); - ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, + S3Client s3 = getS3Client("User agent"); + SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class, "clientConfiguration"); - assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(), - awsConf.getUserAgentPrefix()); + Assertions.assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT)) + .describedAs("User Agent prefix") + .startsWith("MyApp, Hadoop " + VersionInfo.getVersion()); } @Test @@ -407,21 +432,21 @@ public void testRequestTimeout() throws Exception { conf = new Configuration(); conf.set(REQUEST_TIMEOUT, "120"); fs = S3ATestUtils.createTestFileSystem(conf); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("Request timeout (ms)"); - ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, + S3Client s3 = getS3Client("Request timeout (ms)"); + SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class, "clientConfiguration"); assertEquals("Configured " + REQUEST_TIMEOUT + " is different than what AWS sdk configuration uses internally", - 120000, awsConf.getRequestTimeout()); + 120000, + clientConfiguration.option(SdkClientOption.API_CALL_ATTEMPT_TIMEOUT).toMillis()); } @Test - @SuppressWarnings("deprecation") public void testCloseIdempotent() throws Throwable { conf = new Configuration(); fs = S3ATestUtils.createTestFileSystem(conf); AWSCredentialProviderList credentials = - fs.shareCredentials("testCloseIdempotent"); + getS3AInternals().shareCredentials("testCloseIdempotent"); credentials.close(); fs.close(); assertTrue("Closing FS didn't close credentials " + credentials, @@ -514,64 +539,73 @@ public void testConfOptionPropagationToFS() throws Exception { } @Test(timeout = 10_000L) - public void testConnectTtlPropagation() throws Exception { - Configuration config = new Configuration(false); - ClientConfiguration awsConf = new ClientConfiguration(); - initConnectionSettings(config, awsConf); - Assertions.assertThat(awsConf.getConnectionTTL()) - .describedAs("connection ttl should be set to default value as" + - " %s is not set", CONNECTION_TTL) - .isEqualTo(DEFAULT_CONNECTION_TTL); - long connectionTtlTestVal = 1000; - config.setLong(CONNECTION_TTL, connectionTtlTestVal); - initConnectionSettings(config, awsConf); - Assertions.assertThat(awsConf.getConnectionTTL()) - .describedAs("%s not propagated to aws conf", CONNECTION_TTL) - .isEqualTo(connectionTtlTestVal); - - long connectionTtlTestVal1 = -1; - config.setLong(CONNECTION_TTL, connectionTtlTestVal1); - initConnectionSettings(config, awsConf); - Assertions.assertThat(awsConf.getConnectionTTL()) - .describedAs("%s not propagated to aws conf", CONNECTION_TTL) - .isEqualTo(connectionTtlTestVal1); - - long connectionTtlTestVal2 = -100; - config.setLong(CONNECTION_TTL, connectionTtlTestVal2); - intercept(IllegalArgumentException.class, () -> initConnectionSettings(config, awsConf)); + public void testS3SpecificSignerOverride() throws Exception { + Configuration config = new Configuration(); + removeBaseAndBucketOverrides(config, + CUSTOM_SIGNERS, SIGNING_ALGORITHM_S3, SIGNING_ALGORITHM_STS, AWS_REGION); + + config.set(CUSTOM_SIGNERS, + "CustomS3Signer:" + CustomS3Signer.class.getName() + + ",CustomSTSSigner:" + CustomSTSSigner.class.getName()); + + config.set(SIGNING_ALGORITHM_S3, "CustomS3Signer"); + config.set(SIGNING_ALGORITHM_STS, "CustomSTSSigner"); + + config.set(AWS_REGION, EU_WEST_1); + fs = S3ATestUtils.createTestFileSystem(config); + + S3Client s3Client = getS3Client("testS3SpecificSignerOverride"); + + final String bucket = fs.getBucket(); + StsClient stsClient = + STSClientFactory.builder(config, bucket, new AnonymousAWSCredentialsProvider(), "", + "").build(); + + intercept(StsException.class, "", () -> + stsClient.getSessionToken()); + + intercept(AccessDeniedException.class, "", () -> + Invoker.once("head", bucket, () -> + s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()))); + + Assertions.assertThat(CustomS3Signer.isS3SignerCalled()) + .describedAs("Custom S3 signer not called").isTrue(); + + Assertions.assertThat(CustomSTSSigner.isSTSSignerCalled()) + .describedAs("Custom STS signer not called").isTrue(); } - @Test(timeout = 10_000L) - public void testS3SpecificSignerOverride() throws IOException { - ClientConfiguration clientConfiguration = null; - Configuration config; - - String signerOverride = "testSigner"; - String s3SignerOverride = "testS3Signer"; - - // Default SIGNING_ALGORITHM, overridden for S3 only - config = new Configuration(); - config.set(SIGNING_ALGORITHM_S3, s3SignerOverride); - clientConfiguration = S3AUtils - .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3); - Assert.assertEquals(s3SignerOverride, - clientConfiguration.getSignerOverride()); - clientConfiguration = S3AUtils - .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_STS); - Assert.assertNull(clientConfiguration.getSignerOverride()); - - // Configured base SIGNING_ALGORITHM, overridden for S3 only - config = new Configuration(); - config.set(SIGNING_ALGORITHM, signerOverride); - config.set(SIGNING_ALGORITHM_S3, s3SignerOverride); - clientConfiguration = S3AUtils - .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3); - Assert.assertEquals(s3SignerOverride, - clientConfiguration.getSignerOverride()); - clientConfiguration = S3AUtils - .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_STS); - Assert - .assertEquals(signerOverride, clientConfiguration.getSignerOverride()); + public static final class CustomS3Signer implements Signer { + + private static boolean s3SignerCalled = false; + + @Override + public SdkHttpFullRequest sign(SdkHttpFullRequest request, + ExecutionAttributes executionAttributes) { + LOG.debug("Custom S3 signer called"); + s3SignerCalled = true; + return request; + } + + public static boolean isS3SignerCalled() { + return s3SignerCalled; + } } + public static final class CustomSTSSigner implements Signer { + + private static boolean stsSignerCalled = false; + + @Override + public SdkHttpFullRequest sign(SdkHttpFullRequest request, + ExecutionAttributes executionAttributes) { + LOG.debug("Custom STS signer called"); + stsSignerCalled = true; + return request; + } + + public static boolean isSTSSignerCalled() { + return stsSignerCalled; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java index 64e37bf832b87..45b0c6c206f2d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java @@ -63,7 +63,7 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { private static final String SERVICE_AMAZON_S3_STATUS_CODE_403 - = "Service: Amazon S3; Status Code: 403;"; + = "Service: S3, Status Code: 403"; private static final String KEY_1 = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs="; private static final String KEY_2 diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java index 68ab5bd9e8c19..7e399f347100f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java @@ -20,7 +20,7 @@ import java.io.IOException; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -51,9 +51,9 @@ protected S3AEncryptionMethods getSSEAlgorithm() { @Override protected void assertEncrypted(Path path) throws IOException { - ObjectMetadata md = getFileSystem().getObjectMetadata(path); + HeadObjectResponse md = getS3AInternals().getObjectMetadata(path); assertEquals("SSE Algorithm", EncryptionTestUtils.AWS_KMS_SSE_ALGORITHM, - md.getSSEAlgorithm()); - assertThat(md.getSSEAwsKmsKeyId(), containsString("arn:aws:kms:")); + md.serverSideEncryptionAsString()); + assertThat(md.ssekmsKeyId(), containsString("arn:aws:kms:")); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java index 56ce9300dc4e7..1b25846fafddf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java @@ -118,7 +118,8 @@ public void testEncryptionOverRename() throws Throwable { S3AFileSystem fs = getFileSystem(); Path path = path(getMethodName() + "find-encryption-algo"); ContractTestUtils.touch(fs, path); - String sseAlgorithm = fs.getObjectMetadata(path).getSSEAlgorithm(); + String sseAlgorithm = getS3AInternals().getObjectMetadata(path) + .serverSideEncryptionAsString(); if(StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) { skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index add6502d7da71..e5e109ad91b50 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -21,21 +21,24 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.util.AwsHostNameUtils; import org.assertj.core.api.Assertions; import org.junit.Test; +import software.amazon.awssdk.awscore.AwsExecutionAttribute; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; -import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CENTRAL_REGION; -import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AWS_REGION_SYSPROP; +import static org.apache.hadoop.fs.s3a.Statistic.STORE_REGION_PROBE; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -44,114 +47,98 @@ */ public class ITestS3AEndpointRegion extends AbstractS3ATestBase { - private static final String AWS_REGION_TEST = "test-region"; private static final String AWS_ENDPOINT_TEST = "test-endpoint"; - private static final String AWS_ENDPOINT_TEST_WITH_REGION = - "test-endpoint.some-region.amazonaws.com"; - public static final String MARS_NORTH_2 = "mars-north-2"; - /** - * Test to verify that setting a region with the config would bypass the - * construction of region from endpoint. - */ - @Test - public void testWithRegionConfig() { - getFileSystem().getConf().set(AWS_REGION, AWS_REGION_TEST); - - //Creating an endpoint config with a custom endpoint. - AwsClientBuilder.EndpointConfiguration epr = createEpr(AWS_ENDPOINT_TEST, - getFileSystem().getConf().getTrimmed(AWS_REGION)); - //Checking if setting region config bypasses the endpoint region. - Assertions.assertThat(epr.getSigningRegion()) - .describedAs("There is a region mismatch") - .isEqualTo(getFileSystem().getConf().get(AWS_REGION)); - } /** - * Test to verify that not setting the region config, would lead to using - * endpoint to construct the region. + * Test to verify that not setting the region config, will lead to the client factory making + * a HEAD bucket call to configure the correct region. If an incorrect region is set, the HEAD + * bucket call in this test will raise an exception. */ @Test - public void testWithoutRegionConfig() { - getFileSystem().getConf().unset(AWS_REGION); - - //Creating an endpoint config with a custom endpoint containing a region. - AwsClientBuilder.EndpointConfiguration eprRandom = - createEpr(AWS_ENDPOINT_TEST_WITH_REGION, - getFileSystem().getConf().getTrimmed(AWS_REGION)); - String regionFromEndpoint = - AwsHostNameUtils - .parseRegionFromAwsPartitionPattern(AWS_ENDPOINT_TEST_WITH_REGION); - //Checking if not setting region config leads to constructing the region - // from endpoint. - Assertions.assertThat(eprRandom.getSigningRegion()) - .describedAs("There is a region mismatch") - .isNotEqualTo(getFileSystem().getConf().get(AWS_REGION)) - .isEqualTo(regionFromEndpoint); - } + public void testWithoutRegionConfig() throws IOException { + Configuration conf = getConfiguration(); + String bucket = getFileSystem().getBucket(); + conf.unset(String.format("fs.s3a.bucket.%s.endpoint.region", bucket)); + conf.unset(AWS_REGION); - /** - * Method to create EndpointConfiguration using an endpoint. - * - * @param endpoint the endpoint to be used for EndpointConfiguration creation. - * @return an instance of EndpointConfiguration. - */ - private AwsClientBuilder.EndpointConfiguration createEpr(String endpoint, - String awsRegion) { - return DefaultS3ClientFactory.createEndpointConfiguration(endpoint, - new ClientConfiguration(), awsRegion); + S3AFileSystem fs = new S3AFileSystem(); + fs.initialize(getFileSystem().getUri(), conf); + + fs.getS3AInternals().getBucketMetadata(); + + Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) + .describedAs("Region is not configured, region probe should have been made").isEqualTo(1); } @Test - public void testInvalidRegionDefaultEndpoint() throws Throwable { - describe("Create a client with an invalid region and the default endpoint"); + public void testWithRegionConfig() throws IOException, URISyntaxException { Configuration conf = getConfiguration(); - // we are making a big assumption about the timetable for AWS - // region rollout. - // if this test ever fails because this region now exists - // -congratulations! - conf.set(AWS_REGION, MARS_NORTH_2); - createMarsNorth2Client(conf); + conf.set(AWS_REGION, "us-east-2"); + + S3AFileSystem fs = new S3AFileSystem(); + fs.initialize(new URI("s3a://landsat-pds"), conf); + + Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) + .describedAs("Region is configured, region probe should not have been made").isEqualTo(0); } @Test - public void testUnsetRegionDefaultEndpoint() throws Throwable { - describe("Create a client with no region and the default endpoint"); + public void testRegionCache() throws IOException, URISyntaxException { Configuration conf = getConfiguration(); conf.unset(AWS_REGION); - createS3Client(conf, DEFAULT_ENDPOINT, AWS_S3_CENTRAL_REGION); + conf.unset("fs.s3a.bucket.landsat-pds.endpoint.region"); + S3AFileSystem fs = new S3AFileSystem(); + + fs.initialize(new URI("s3a://landsat-pds"), conf); + + Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) + .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(1); + + fs.initialize(new URI("s3a://landsat-pds"), conf); + + // value should already be cached. + Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) + .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(0); } - /** - * By setting the system property {@code "aws.region"} we can - * guarantee that the SDK region resolution chain will always succeed - * (and fast). - * Clearly there is no validation of the region during the build process. - */ @Test - public void testBlankRegionTriggersSDKResolution() throws Throwable { - describe("Create a client with a blank region and the default endpoint." - + " This will trigger the SDK Resolution chain"); + public void testEndpointOverride() throws Throwable { + describe("Create a client with no region and the default endpoint"); Configuration conf = getConfiguration(); - conf.set(AWS_REGION, ""); - System.setProperty(AWS_REGION_SYSPROP, MARS_NORTH_2); - try { - createMarsNorth2Client(conf); - } finally { - System.clearProperty(AWS_REGION_SYSPROP); - } + + S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST); + + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); } - /** - * Create an S3 client bonded to an invalid region; - * verify that calling {@code getRegion()} triggers - * a failure. - * @param conf configuration to use in the building. - */ - private void createMarsNorth2Client(Configuration conf) throws Exception { - AmazonS3 client = createS3Client(conf, DEFAULT_ENDPOINT, MARS_NORTH_2); - intercept(IllegalArgumentException.class, MARS_NORTH_2, client::getRegion); + + class RegionInterceptor implements ExecutionInterceptor { + private boolean endpointOverridden; + + RegionInterceptor(boolean endpointOverridden) { + this.endpointOverridden = endpointOverridden; + } + + @Override + public void beforeExecution(Context.BeforeExecution context, + ExecutionAttributes executionAttributes) { + + if (endpointOverridden) { + Assertions.assertThat( + executionAttributes.getAttribute(AwsExecutionAttribute.ENDPOINT_OVERRIDDEN)) + .describedAs("Endpoint not overridden").isTrue(); + + Assertions.assertThat( + executionAttributes.getAttribute(AwsExecutionAttribute.CLIENT_ENDPOINT).toString()) + .describedAs("There is an endpoint mismatch").isEqualTo("https://" + AWS_ENDPOINT_TEST); + } + + // We don't actually want to make a request, so exit early. + throw AwsServiceException.builder().message("Exception thrown by interceptor").build(); + } } /** @@ -160,16 +147,23 @@ private void createMarsNorth2Client(Configuration conf) throws Exception { * value. * @param conf configuration to use. * @param endpoint endpoint. - * @param expectedRegion expected region * @return the client. * @throws URISyntaxException parse problems. * @throws IOException IO problems */ @SuppressWarnings("deprecation") - private AmazonS3 createS3Client(Configuration conf, - String endpoint, - String expectedRegion) - throws URISyntaxException, IOException { + private S3Client createS3Client(Configuration conf, + String endpoint) + throws IOException { + + boolean endpointOverridden = false; + + if (endpoint != null && !endpoint.isEmpty()) { + endpointOverridden = true; + } + + List interceptors = new ArrayList<>(); + interceptors.add(new RegionInterceptor(endpointOverridden)); DefaultS3ClientFactory factory = new DefaultS3ClientFactory(); @@ -177,16 +171,14 @@ private AmazonS3 createS3Client(Configuration conf, S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(new AnonymousAWSCredentialsProvider()) - .withPathUri(new URI("s3a://localhost/")) .withEndpoint(endpoint) .withMetrics(new EmptyS3AStatisticsContext() - .newStatisticsFromAwsSdk()); - AmazonS3 client = factory.createS3Client( - new URI("s3a://localhost/"), + .newStatisticsFromAwsSdk()) + .withExecutionInterceptors(interceptors); + + S3Client client = factory.createS3Client( + getFileSystem().getUri(), parameters); - Assertions.assertThat(client.getRegionName()) - .describedAs("Client region name") - .isEqualTo(expectedRegion); return client; } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index c0f6a4b23226b..a741b11b0ce47 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -18,7 +18,8 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.S3Error; import org.assertj.core.api.Assertions; import org.junit.Assume; @@ -27,6 +28,7 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; import org.apache.hadoop.fs.statistics.StoreStatisticNames; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -34,10 +36,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.nio.file.AccessDeniedException; +import java.util.stream.Collectors; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles; @@ -115,12 +119,12 @@ private void removeKeys(S3AFileSystem fileSystem, String... keys) } } - private List buildDeleteRequest( + private List buildDeleteRequest( final String[] keys) { - List request = new ArrayList<>( + List request = new ArrayList<>( keys.length); for (String key : keys) { - request.add(new DeleteObjectsRequest.KeyVersion(key)); + request.add(ObjectIdentifier.builder().key(key).build()); } return request; } @@ -156,12 +160,26 @@ public void testMultiObjectDeleteNoPermissions() throws Throwable { // create a span, expect it to be activated. fs.getAuditSpanSource().createSpan(StoreStatisticNames.OP_DELETE, csvPath.toString(), null); - List keys + List keys = buildDeleteRequest( new String[]{ fs.pathToKey(csvPath), "missing-key.csv" }); + MultiObjectDeleteException ex = intercept( + MultiObjectDeleteException.class, + () -> fs.removeKeys(keys, false)); + final List undeleted = ex.errors().stream() + .map(S3Error::key) + .map(fs::keyToQualifiedPath) + .collect(Collectors.toList()); + final String undeletedFiles = undeleted.stream() + .map(Path::toString) + .collect(Collectors.joining(", ")); + failIf(undeleted.size() != 2, + "undeleted list size wrong: " + undeletedFiles, + ex); + assertTrue("no CSV in " +undeletedFiles, undeleted.contains(csvPath)); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java index 1a944ec29968a..4aa9c6a52eaa2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -25,19 +25,22 @@ import java.nio.charset.StandardCharsets; import java.nio.file.AccessDeniedException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.GetBucketEncryptionResult; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetBucketEncryptionRequest; +import software.amazon.awssdk.services.s3.model.GetBucketEncryptionResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import org.assertj.core.api.Assertions; import org.junit.Assume; import org.junit.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.api.RequestFactory; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; +import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.store.EtagChecksum; import org.apache.hadoop.test.LambdaTestUtils; @@ -106,15 +109,15 @@ public void testCreateNonRecursiveSuccess() throws IOException { public void testPutObjectDirect() throws Throwable { final S3AFileSystem fs = getFileSystem(); try (AuditSpan span = span()) { - ObjectMetadata metadata = fs.newObjectMetadata(-1); - metadata.setContentLength(-1); + RequestFactory factory = RequestFactoryImpl.builder().withBucket(fs.getBucket()).build(); Path path = path("putDirect"); - final PutObjectRequest put = new PutObjectRequest(fs.getBucket(), - path.toUri().getPath(), - new ByteArrayInputStream("PUT".getBytes()), - metadata); + PutObjectRequest.Builder putObjectRequestBuilder = + factory.newPutObjectRequestBuilder(path.toUri().getPath(), null, -1, false); + putObjectRequestBuilder.contentLength(-1L); LambdaTestUtils.intercept(IllegalStateException.class, - () -> fs.putObjectDirect(put, PutObjectOptions.keepingDirs(), null)); + () -> fs.putObjectDirect(putObjectRequestBuilder.build(), PutObjectOptions.keepingDirs(), + new S3ADataBlocks.BlockUploadData(new ByteArrayInputStream("PUT".getBytes())), + false, null)); assertPathDoesNotExist("put object was created", path); } } @@ -406,13 +409,15 @@ private static T verifyNoTrailingSlash(String role, T o) { * Gets default encryption settings for the bucket or returns null if default * encryption is disabled. */ - private GetBucketEncryptionResult getDefaultEncryption() throws IOException { + private GetBucketEncryptionResponse getDefaultEncryption() throws IOException { S3AFileSystem fs = getFileSystem(); - AmazonS3 s3 = fs.getAmazonS3ClientForTesting("check default encryption"); - try { + S3Client s3 = getS3AInternals().getAmazonS3Client("check default encryption"); + try (AuditSpan s = span()){ return Invoker.once("getBucketEncryption()", fs.getBucket(), - () -> s3.getBucketEncryption(fs.getBucket())); + () -> s3.getBucketEncryption(GetBucketEncryptionRequest.builder() + .bucket(fs.getBucket()) + .build())); } catch (FileNotFoundException e) { return null; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java index 818d2fc889c17..197811f39fb9b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java @@ -18,13 +18,14 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.MultipartUpload; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.junit.Test; + import java.io.IOException; import java.util.HashSet; import java.util.Set; @@ -114,7 +115,7 @@ private void assertUploadsPresent(MultipartUtils.UploadIterator list, } private MultipartTestUtils.IdKey toIdKey(MultipartUpload mu) { - return new MultipartTestUtils.IdKey(mu.getKey(), mu.getUploadId()); + return new MultipartTestUtils.IdKey(mu.key(), mu.uploadId()); } private Path getPartFilename(int index) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java index c58f13efbf267..b88d0b4aab617 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java @@ -107,7 +107,7 @@ public void testRequesterPaysDisabledFails() throws Throwable { try (FileSystem fs = requesterPaysPath.getFileSystem(conf)) { intercept( AccessDeniedException.class, - "403 Forbidden", + "403", "Expected requester pays bucket to fail without header set", () -> fs.open(requesterPaysPath).close() ); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java index 7c56f8d2ea050..6ccb7ac26040b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java @@ -37,7 +37,6 @@ import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER; import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER_ARRAY; import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER_DISK; -import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BYTEBUFFER; import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS; import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER; import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_REDUCED_REDUNDANCY; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 0778662542d88..290a4d995c757 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -25,10 +25,9 @@ import java.time.OffsetDateTime; import java.util.concurrent.TimeUnit; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.services.securitytoken.AWSSecurityTokenService; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; -import com.amazonaws.services.securitytoken.model.Credentials; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.Credentials; import org.hamcrest.Matchers; import org.junit.Test; import org.slf4j.Logger; @@ -71,7 +70,6 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class); - @SuppressWarnings("deprecation") private static final String TEMPORARY_AWS_CREDENTIALS = TemporaryAWSCredentialsProvider.NAME; @@ -117,10 +115,10 @@ protected Configuration createConfiguration() { public void testSTS() throws IOException { Configuration conf = getContract().getConf(); S3AFileSystem testFS = getFileSystem(); - credentials = testFS.shareCredentials("testSTS"); + credentials = getS3AInternals().shareCredentials("testSTS"); String bucket = testFS.getBucket(); - AWSSecurityTokenServiceClientBuilder builder = STSClientFactory.builder( + StsClientBuilder builder = STSClientFactory.builder( conf, bucket, credentials, @@ -154,7 +152,7 @@ public void testSTS() throws IOException { // now create an invalid set of credentials by changing the session // token - conf2.set(SESSION_TOKEN, "invalid-" + sessionCreds.getSessionToken()); + conf2.set(SESSION_TOKEN, "invalid-" + sessionCreds.sessionToken()); try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf2)) { createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE); fail("Expected an access exception, but file access to " @@ -176,14 +174,13 @@ protected String getStsRegion(final Configuration conf) { } @Test - @SuppressWarnings("deprecation") public void testTemporaryCredentialValidation() throws Throwable { Configuration conf = new Configuration(); conf.set(ACCESS_KEY, "accesskey"); conf.set(SECRET_KEY, "secretkey"); conf.set(SESSION_TOKEN, ""); LambdaTestUtils.intercept(CredentialInitializationException.class, - () -> new TemporaryAWSCredentialsProvider(conf).getCredentials()); + () -> new TemporaryAWSCredentialsProvider(conf).resolveCredentials()); } /** @@ -360,25 +357,24 @@ public void testSessionCredentialsEndpointNoRegion() throws Throwable { * @return the caught exception. * @throws Exception any unexpected exception. */ - @SuppressWarnings("deprecation") public E expectedSessionRequestFailure( final Class clazz, final String endpoint, final String region, final String exceptionText) throws Exception { try(AWSCredentialProviderList parentCreds = - getFileSystem().shareCredentials("test"); + getS3AInternals().shareCredentials("test"); DurationInfo ignored = new DurationInfo(LOG, "requesting credentials")) { Configuration conf = new Configuration(getContract().getConf()); - ClientConfiguration awsConf = - S3AUtils.createAwsConf(conf, null, AWS_SERVICE_IDENTIFIER_STS); + return intercept(clazz, exceptionText, () -> { - AWSSecurityTokenService tokenService = + StsClient tokenService = STSClientFactory.builder(parentCreds, - awsConf, + conf, endpoint, - region) + region, + getFileSystem().getBucket()) .build(); Invoker invoker = new Invoker(new S3ARetryPolicy(conf), LOG_AT_ERROR); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java index 40857373fb808..b7e55f01a371e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java @@ -21,9 +21,9 @@ import java.io.IOException; import java.net.URI; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.services.s3.AmazonS3; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.S3Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +51,7 @@ import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.util.Progressable; + import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.noopAuditor; import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTrackerFactory; import static org.apache.hadoop.util.Preconditions.checkNotNull; @@ -116,9 +117,7 @@ public MockS3AFileSystem(S3AFileSystem mock, root = new Path(FS_URI.toString()); } - private static T prepareRequest(T t) { - return t; - } + private static void prepareRequest(SdkRequest.Builder t) {} @Override public RequestFactory getRequestFactory() { @@ -210,7 +209,7 @@ public boolean isMultipartUploadEnabled() { * @param client client. */ @Override - public void setAmazonS3Client(AmazonS3 client) { + public void setAmazonS3Client(S3Client client) { LOG.debug("Setting S3 client to {}", client); super.setAmazonS3Client(client); } @@ -353,13 +352,13 @@ public long getDefaultBlockSize() { void deleteObjectAtPath(Path f, String key, boolean isFile) - throws AmazonClientException, IOException { + throws SdkException, IOException { deleteObject(key); } @Override protected void maybeCreateFakeParentDirectory(Path path) - throws IOException, AmazonClientException { + throws IOException, SdkException { // no-op } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java index 3240309aef971..0c61caacd055b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java @@ -23,32 +23,47 @@ import java.net.URI; import java.util.ArrayList; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.MultipartUploadListing; -import com.amazonaws.services.s3.model.Region; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest; +import software.amazon.awssdk.services.s3.model.GetBucketLocationResponse; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse; +import software.amazon.awssdk.transfer.s3.S3TransferManager; /** - * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3} + * An {@link S3ClientFactory} that returns Mockito mocks of the {@link S3Client} * interface suitable for unit testing. */ -@SuppressWarnings("deprecation") public class MockS3ClientFactory implements S3ClientFactory { + @Override - public AmazonS3 createS3Client(URI uri, - final S3ClientCreationParameters parameters) { - AmazonS3 s3 = mock(AmazonS3.class); - String bucket = uri.getHost(); - when(s3.doesBucketExist(bucket)).thenReturn(true); - when(s3.doesBucketExistV2(bucket)).thenReturn(true); + public S3Client createS3Client(URI uri, final S3ClientCreationParameters parameters) { + S3Client s3 = mock(S3Client.class); // this listing is used in startup if purging is enabled, so // return a stub value - MultipartUploadListing noUploads = new MultipartUploadListing(); - noUploads.setMultipartUploads(new ArrayList<>(0)); - when(s3.listMultipartUploads(any())) - .thenReturn(noUploads); - when(s3.getBucketLocation(anyString())) - .thenReturn(Region.US_West.toString()); + ListMultipartUploadsResponse noUploads = ListMultipartUploadsResponse.builder() + .uploads(new ArrayList<>(0)) + .isTruncated(false) + .build(); + when(s3.listMultipartUploads((ListMultipartUploadsRequest) any())).thenReturn(noUploads); + when(s3.getBucketLocation((GetBucketLocationRequest) any())).thenReturn( + GetBucketLocationResponse.builder().locationConstraint(Region.US_WEST_2.toString()) + .build()); + return s3; + } + + @Override + public S3AsyncClient createS3AsyncClient(URI uri, final S3ClientCreationParameters parameters) { + S3AsyncClient s3 = mock(S3AsyncClient.class); return s3; } + + @Override + public S3TransferManager createS3TransferManager(S3AsyncClient s3AsyncClient) { + S3TransferManager tm = mock(S3TransferManager.class); + return tm; + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java index 1ddff3c4cd5e9..2b7620ddbddfc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java @@ -18,9 +18,10 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.PartETag; -import com.amazonaws.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -33,6 +34,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Date; import java.util.List; import java.util.Objects; import java.util.Set; @@ -80,10 +82,11 @@ public static IdKey createPartUpload(S3AFileSystem fs, String key, int len, byte[] data = dataset(len, 'a', 'z'); InputStream in = new ByteArrayInputStream(data); String uploadId = writeHelper.initiateMultiPartUpload(key, PutObjectOptions.keepingDirs()); - UploadPartRequest req = writeHelper.newUploadPartRequest(key, uploadId, - partNo, len, in, null, 0L); - PartETag partEtag = writeHelper.uploadPart(req, null).getPartETag(); - LOG.debug("uploaded part etag {}, upid {}", partEtag.getETag(), uploadId); + UploadPartRequest req = writeHelper.newUploadPartRequestBuilder(key, uploadId, + partNo, len).build(); + RequestBody body = RequestBody.fromInputStream(in, len); + UploadPartResponse response = writeHelper.uploadPart(req, body, null); + LOG.debug("uploaded part etag {}, upid {}", response.eTag(), uploadId); return new IdKey(key, uploadId); } } @@ -99,10 +102,10 @@ public static void clearAnyUploads(S3AFileSystem fs, Path path) { = fs.getWriteOperationHelper(); while (uploads.hasNext()) { MultipartUpload upload = uploads.next(); - LOG.debug("Cleaning up upload: {} {}", upload.getKey(), - truncatedUploadId(upload.getUploadId())); - helper.abortMultipartUpload(upload.getKey(), - upload.getUploadId(), true, LOG_EVENT); + LOG.debug("Cleaning up upload: {} {}", upload.key(), + truncatedUploadId(upload.uploadId())); + helper.abortMultipartUpload(upload.key(), + upload.uploadId(), true, LOG_EVENT); } } catch (IOException ioe) { LOG.info("Ignoring exception: ", ioe); @@ -118,8 +121,8 @@ public static void assertNoUploadsAt(S3AFileSystem fs, Path path) throws MultipartUtils.UploadIterator uploads = fs.listUploads(key); while (uploads.hasNext()) { MultipartUpload upload = uploads.next(); - Assert.fail("Found unexpected upload " + upload.getKey() + " " + - truncatedUploadId(upload.getUploadId())); + Assert.fail("Found unexpected upload " + upload.key() + " " + + truncatedUploadId(upload.uploadId())); } } @@ -149,9 +152,9 @@ public static List listMultipartUploads(S3AFileSystem fs, return fs .listMultipartUploads(prefix).stream() .map(upload -> String.format("Upload to %s with ID %s; initiated %s", - upload.getKey(), - upload.getUploadId(), - S3ATestUtils.LISTING_FORMAT.format(upload.getInitiated()))) + upload.key(), + upload.uploadId(), + S3ATestUtils.LISTING_FORMAT.format(Date.from(upload.initiated())))) .collect(Collectors.toList()); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index a6269c437665a..246d111d14b8d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -251,4 +251,9 @@ public interface S3ATestConstants { * Value: {@value}. */ String PROJECT_BUILD_DIRECTORY_PROPERTY = "project.build.directory"; + + /** + * AWS ireland region. + */ + String EU_WEST_1 = "eu-west-1"; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 469562f9b33b9..239e52a7264a0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -61,7 +61,7 @@ import org.apache.hadoop.util.functional.CallableRaisingIOE; import org.apache.hadoop.util.functional.FutureIO; -import com.amazonaws.auth.AWSCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Assume; @@ -435,17 +435,22 @@ public static Path getLandsatCSVPath(Configuration conf) { * @param clazz the expected exception class * @param ex the exception caught * @return the exception, if it is of the expected class - * @throws Exception the exception passed in. + * @throws AssertionError if the exception is {@code null}. + * @throws Exception the exception passed in if it is of a different type */ public static E verifyExceptionClass(Class clazz, Exception ex) throws Exception { + Assertions.assertThat(ex) + .describedAs("Exception expected of class %s", clazz) + .isNotNull(); if (!(ex.getClass().equals(clazz))) { throw ex; } return (E)ex; } + /** * Turn off FS Caching: use if a filesystem with different options from * the default is required. @@ -612,8 +617,7 @@ public static void unsetHadoopCredentialProviders(final Configuration conf) { * @return a set of credentials * @throws IOException on a failure */ - @SuppressWarnings("deprecation") - public static AWSCredentialsProvider buildAwsCredentialsProvider( + public static AwsCredentialsProvider buildAwsCredentialsProvider( final Configuration conf) throws IOException { assumeSessionTestsEnabled(conf); @@ -668,13 +672,14 @@ public static MarshalledCredentials requestSessionCredentials( MarshalledCredentials sc = MarshalledCredentialBinding .requestSessionCredentials( buildAwsCredentialsProvider(conf), - S3AUtils.createAwsConf(conf, bucket, AWS_SERVICE_IDENTIFIER_STS), + conf, conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT, DEFAULT_ASSUMED_ROLE_STS_ENDPOINT), conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT_REGION, ASSUMED_ROLE_STS_ENDPOINT_REGION_DEFAULT), duration, - new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT)); + new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT), + bucket); sc.validate("requested session credentials: ", MarshalledCredentials.CredentialTypeRequired.SessionOnly); return sc; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java index c881aac35d94d..28d0ca56f0eae 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java @@ -18,12 +18,13 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.regions.Regions; +import software.amazon.awssdk.regions.Region; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.test.HadoopTestBase; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -47,9 +48,9 @@ public void parseAccessPointFromArn() throws IllegalArgumentException { String accessPoint = "testAp"; String[][] regionPartitionEndpoints = new String[][] { - {Regions.EU_WEST_1.getName(), "aws"}, - {Regions.US_GOV_EAST_1.getName(), "aws-us-gov"}, - {Regions.CN_NORTH_1.getName(), "aws-cn"}, + {Region.EU_WEST_1.id(), "aws"}, + {Region.US_GOV_EAST_1.id(), "aws-us-gov"}, + {Region.CN_NORTH_1.id(), "aws-cn"}, }; for (String[] testPair : regionPartitionEndpoints) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java index 35199f4092790..0ac49812e4cb6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java @@ -22,14 +22,14 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.net.SocketTimeoutException; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkBaseException; -import com.amazonaws.SdkClientException; -import com.amazonaws.services.s3.model.AmazonS3Exception; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.model.S3Exception; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -38,6 +38,7 @@ import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.net.ConnectTimeoutException; + import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.Invoker.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.verifyExceptionClass; @@ -98,9 +99,11 @@ public class TestInvoker extends Assert { private int retryCount; private Invoker invoker = new Invoker(RETRY_POLICY, (text, e, retries, idempotent) -> retryCount++); - private static final AmazonClientException CLIENT_TIMEOUT_EXCEPTION = - new AmazonClientException(new Local.ConnectTimeoutException("timeout")); - private static final AmazonServiceException BAD_REQUEST = serviceException( + private static final SdkException CLIENT_TIMEOUT_EXCEPTION = + SdkException.builder() + .cause(new Local.ConnectTimeoutException("timeout")) + .build(); + private static final AwsServiceException BAD_REQUEST = serviceException( AWSBadRequestException.STATUS_CODE, "bad request"); @@ -109,24 +112,26 @@ public void setup() { resetCounters(); } - private static AmazonServiceException serviceException(int code, + private static AwsServiceException serviceException(int code, String text) { - AmazonServiceException ex = new AmazonServiceException(text); - ex.setStatusCode(code); - return ex; + return AwsServiceException.builder() + .message(text) + .statusCode(code) + .build(); } - private static AmazonS3Exception createS3Exception(int code) { + private static S3Exception createS3Exception(int code) { return createS3Exception(code, "", null); } - private static AmazonS3Exception createS3Exception(int code, + private static S3Exception createS3Exception(int code, String message, Throwable inner) { - AmazonS3Exception ex = new AmazonS3Exception(message); - ex.setStatusCode(code); - ex.initCause(inner); - return ex; + return (S3Exception) S3Exception.builder() + .message(message) + .statusCode(code) + .cause(inner) + .build(); } protected void verifyTranslated( @@ -136,7 +141,7 @@ protected void verifyTranslated( } private static E verifyTranslated(Class clazz, - SdkBaseException exception) throws Exception { + SdkException exception) throws Exception { return verifyExceptionClass(clazz, translateException("test", "/", exception)); } @@ -157,16 +162,22 @@ public void testS3500isStatus500Exception() throws Exception { @Test public void test500isStatus500Exception() throws Exception { - AmazonServiceException ex = new AmazonServiceException(""); - ex.setStatusCode(500); + AwsServiceException ex = AwsServiceException.builder() + .message("") + .statusCode(500) + .build(); verifyTranslated(AWSStatus500Exception.class, ex); } @Test public void testExceptionsWithTranslatableMessage() throws Exception { - SdkBaseException xmlParsing = new SdkBaseException(EOF_MESSAGE_IN_XML_PARSER); - SdkBaseException differentLength = new SdkBaseException(EOF_READ_DIFFERENT_LENGTH); + SdkException xmlParsing = SdkException.builder() + .message(EOF_MESSAGE_IN_XML_PARSER) + .build(); + SdkException differentLength = SdkException.builder() + .message(EOF_READ_DIFFERENT_LENGTH) + .build(); verifyTranslated(EOFException.class, xmlParsing); verifyTranslated(EOFException.class, differentLength); @@ -178,7 +189,9 @@ public void testSdkDifferentLengthExceptionIsTranslatable() throws Throwable { final AtomicInteger counter = new AtomicInteger(0); invoker.retry("test", null, false, () -> { if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) { - throw new SdkClientException(EOF_READ_DIFFERENT_LENGTH); + throw SdkClientException.builder() + .message(EOF_READ_DIFFERENT_LENGTH) + .build(); } }); @@ -190,7 +203,9 @@ public void testSdkXmlParsingExceptionIsTranslatable() throws Throwable { final AtomicInteger counter = new AtomicInteger(0); invoker.retry("test", null, false, () -> { if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) { - throw new SdkClientException(EOF_MESSAGE_IN_XML_PARSER); + throw SdkClientException.builder() + .message(EOF_MESSAGE_IN_XML_PARSER) + .build(); } }); @@ -201,14 +216,36 @@ public void testSdkXmlParsingExceptionIsTranslatable() throws Throwable { public void testExtractConnectTimeoutException() throws Throwable { throw extractException("", "", new ExecutionException( - new AmazonClientException(LOCAL_CONNECTION_TIMEOUT_EX))); + SdkException.builder() + .cause(LOCAL_CONNECTION_TIMEOUT_EX) + .build())); } @Test(expected = SocketTimeoutException.class) public void testExtractSocketTimeoutException() throws Throwable { throw extractException("", "", new ExecutionException( - new AmazonClientException(SOCKET_TIMEOUT_EX))); + SdkException.builder() + .cause(SOCKET_TIMEOUT_EX) + .build())); + } + + @Test(expected = org.apache.hadoop.net.ConnectTimeoutException.class) + public void testExtractConnectTimeoutExceptionFromCompletionException() throws Throwable { + throw extractException("", "", + new CompletionException( + SdkException.builder() + .cause(LOCAL_CONNECTION_TIMEOUT_EX) + .build())); + } + + @Test(expected = SocketTimeoutException.class) + public void testExtractSocketTimeoutExceptionFromCompletionException() throws Throwable { + throw extractException("", "", + new CompletionException( + SdkException.builder() + .cause(SOCKET_TIMEOUT_EX) + .build())); } /** @@ -259,7 +296,7 @@ public void testRetryThrottled() throws Throwable { ex, retries, false); } - protected AmazonServiceException newThrottledException() { + protected AwsServiceException newThrottledException() { return serviceException( AWSServiceThrottledException.STATUS_CODE, "throttled"); } @@ -354,7 +391,9 @@ public void testUnshadedConnectionTimeoutExceptionMatching() // connection timeout exceptions are special, but as AWS shades // theirs, we need to string match them verifyTranslated(ConnectTimeoutException.class, - new AmazonClientException(HTTP_CONNECTION_TIMEOUT_EX)); + SdkException.builder() + .cause(HTTP_CONNECTION_TIMEOUT_EX) + .build()); } @Test @@ -362,14 +401,18 @@ public void testShadedConnectionTimeoutExceptionMatching() throws Throwable { // connection timeout exceptions are special, but as AWS shades // theirs, we need to string match them verifyTranslated(ConnectTimeoutException.class, - new AmazonClientException(LOCAL_CONNECTION_TIMEOUT_EX)); + SdkException.builder() + .cause(LOCAL_CONNECTION_TIMEOUT_EX) + .build()); } @Test public void testShadedConnectionTimeoutExceptionNotMatching() throws Throwable { InterruptedIOException ex = verifyTranslated(InterruptedIOException.class, - new AmazonClientException(new Local.NotAConnectTimeoutException())); + SdkException.builder() + .cause(new Local.NotAConnectTimeoutException()) + .build()); if (ex instanceof ConnectTimeoutException) { throw ex; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 6456cb5e12a7f..4b06d596a5661 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -30,31 +30,45 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.annotation.Nullable; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import org.apache.hadoop.util.Sets; -import org.junit.Rule; +import org.assertj.core.api.Assertions; import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; +import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.io.retry.RetryPolicy; +import org.apache.hadoop.util.Sets; -import static org.apache.hadoop.fs.s3a.Constants.*; -import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; -import static org.apache.hadoop.fs.s3a.S3AUtils.*; +import static org.apache.hadoop.fs.s3a.Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER; +import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_CSVTEST_FILE; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.authenticationContains; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.buildClassListString; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getCSVTestPath; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.STANDARD_AWS_PROVIDERS; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.DOES_NOT_IMPLEMENT; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * Unit tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic. @@ -67,19 +81,18 @@ public class TestS3AAWSCredentialsProvider { private static final URI TESTFILE_URI = new Path( DEFAULT_CSVTEST_FILE).toUri(); - @Rule - public ExpectedException exception = ExpectedException.none(); + private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class); @Test public void testProviderWrongClass() throws Exception { expectProviderInstantiationFailure(this.getClass(), - NOT_AWS_PROVIDER); + DOES_NOT_IMPLEMENT + " software.amazon.awssdk.auth.credentials.AwsCredentialsProvider"); } @Test public void testProviderAbstractClass() throws Exception { expectProviderInstantiationFailure(AbstractProvider.class, - ABSTRACT_PROVIDER); + InstantiationIOException.ABSTRACT_PROVIDER); } @Test @@ -92,18 +105,17 @@ public void testProviderNotAClass() throws Exception { public void testProviderConstructorError() throws Exception { expectProviderInstantiationFailure( ConstructorSignatureErrorProvider.class, - CONSTRUCTOR_EXCEPTION); + InstantiationIOException.CONSTRUCTOR_EXCEPTION); } @Test public void testProviderFailureError() throws Exception { expectProviderInstantiationFailure( ConstructorFailureProvider.class, - INSTANTIATION_EXCEPTION); + InstantiationIOException.INSTANTIATION_EXCEPTION); } @Test - @SuppressWarnings("deprecation") public void testInstantiationChain() throws Throwable { Configuration conf = new Configuration(false); conf.set(AWS_CREDENTIALS_PROVIDER, @@ -112,7 +124,7 @@ public void testInstantiationChain() throws Throwable { + " ,\n " + AnonymousAWSCredentialsProvider.NAME); Path testFile = getCSVTestPath(conf); - AWSCredentialProviderList list = createAWSCredentialProviderSet( + AWSCredentialProviderList list = createAWSCredentialProviderList( testFile.toUri(), conf); List> expectedClasses = Arrays.asList( @@ -123,15 +135,14 @@ public void testInstantiationChain() throws Throwable { } @Test - @SuppressWarnings("deprecation") public void testDefaultChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); // use the default credential provider chain conf.unset(AWS_CREDENTIALS_PROVIDER); - AWSCredentialProviderList list1 = createAWSCredentialProviderSet( + AWSCredentialProviderList list1 = createAWSCredentialProviderList( uri1, conf); - AWSCredentialProviderList list2 = createAWSCredentialProviderSet( + AWSCredentialProviderList list2 = createAWSCredentialProviderList( uri2, conf); List> expectedClasses = STANDARD_AWS_PROVIDERS; assertCredentialProviders(expectedClasses, list1); @@ -144,30 +155,29 @@ public void testDefaultChainNoURI() throws Exception { // use the default credential provider chain conf.unset(AWS_CREDENTIALS_PROVIDER); assertCredentialProviders(STANDARD_AWS_PROVIDERS, - createAWSCredentialProviderSet(null, conf)); + createAWSCredentialProviderList(null, conf)); } @Test - @SuppressWarnings("deprecation") public void testConfiguredChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); List> expectedClasses = Arrays.asList( - EnvironmentVariableCredentialsProvider.class, - InstanceProfileCredentialsProvider.class, - AnonymousAWSCredentialsProvider.class); + IAMInstanceCredentialsProvider.class, + AnonymousAWSCredentialsProvider.class, + EnvironmentVariableCredentialsProvider.class + ); Configuration conf = createProviderConfiguration(buildClassListString(expectedClasses)); - AWSCredentialProviderList list1 = createAWSCredentialProviderSet( + AWSCredentialProviderList list1 = createAWSCredentialProviderList( uri1, conf); - AWSCredentialProviderList list2 = createAWSCredentialProviderSet( + AWSCredentialProviderList list2 = createAWSCredentialProviderList( uri2, conf); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); } @Test - @SuppressWarnings("deprecation") public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -175,9 +185,9 @@ public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { Arrays.asList( InstanceProfileCredentialsProvider.class); conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); - AWSCredentialProviderList list1 = createAWSCredentialProviderSet( + AWSCredentialProviderList list1 = createAWSCredentialProviderList( uri1, conf); - AWSCredentialProviderList list2 = createAWSCredentialProviderSet( + AWSCredentialProviderList list2 = createAWSCredentialProviderList( uri2, conf); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); @@ -194,54 +204,75 @@ public void testFallbackToDefaults() throws Throwable { EnvironmentVariableCredentialsProvider.class), Sets.newHashSet()); assertTrue("empty credentials", credentials.size() > 0); + } + + @Test + public void testProviderConstructor() throws Throwable { + final AWSCredentialProviderList list = new AWSCredentialProviderList("name", + new AnonymousAWSCredentialsProvider(), + new ErrorProvider(TESTFILE_URI, new Configuration())); + Assertions.assertThat(list.getProviders()) + .describedAs("provider list in %s", list) + .hasSize(2); + final AwsCredentials credentials = list.resolveCredentials(); + Assertions.assertThat(credentials) + .isInstanceOf(AwsBasicCredentials.class); + assertCredentialResolution(credentials, null, null); + } + + public static void assertCredentialResolution(AwsCredentials creds, String key, String secret) { + Assertions.assertThat(creds.accessKeyId()) + .describedAs("access key of %s", creds) + .isEqualTo(key); + Assertions.assertThat(creds.secretAccessKey()) + .describedAs("secret key of %s", creds) + .isEqualTo(secret); + } + + private String buildClassList(Class... classes) { + return Arrays.stream(classes) + .map(Class::getCanonicalName) + .collect(Collectors.joining(",")); + } + private String buildClassList(String... classes) { + return Arrays.stream(classes) + .collect(Collectors.joining(",")); } /** * A credential provider declared as abstract, so it cannot be instantiated. */ - static abstract class AbstractProvider implements AWSCredentialsProvider { + static abstract class AbstractProvider implements AwsCredentialsProvider { + + @Override + public AwsCredentials resolveCredentials() { + return null; + } } /** * A credential provider whose constructor signature doesn't match. */ protected static class ConstructorSignatureErrorProvider - implements AWSCredentialsProvider { + extends AbstractProvider { @SuppressWarnings("unused") public ConstructorSignatureErrorProvider(String str) { } - - @Override - public AWSCredentials getCredentials() { - return null; - } - - @Override - public void refresh() { - } } /** * A credential provider whose constructor raises an NPE. */ protected static class ConstructorFailureProvider - implements AWSCredentialsProvider { + extends AbstractProvider { @SuppressWarnings("unused") public ConstructorFailureProvider() { throw new NullPointerException("oops"); } - @Override - public AWSCredentials getCredentials() { - return null; - } - - @Override - public void refresh() { - } } @Test @@ -254,33 +285,23 @@ public void testAWSExceptionTranslation() throws Throwable { } } - protected static class AWSExceptionRaisingFactory implements AWSCredentialsProvider { + protected static class AWSExceptionRaisingFactory extends AbstractProvider { public static final String NO_AUTH = "No auth"; - public static AWSCredentialsProvider getInstance() { + public static AwsCredentialsProvider create() { throw new NoAuthWithAWSException(NO_AUTH); } - - @Override - public AWSCredentials getCredentials() { - return null; - } - - @Override - public void refresh() { - - } } @Test public void testFactoryWrongType() throws Throwable { expectProviderInstantiationFailure( FactoryOfWrongType.class, - CONSTRUCTOR_EXCEPTION); + InstantiationIOException.CONSTRUCTOR_EXCEPTION); } - static class FactoryOfWrongType implements AWSCredentialsProvider { + static class FactoryOfWrongType extends AbstractProvider { public static final String NO_AUTH = "No auth"; @@ -289,14 +310,10 @@ public static String getInstance() { } @Override - public AWSCredentials getCredentials() { + public AwsCredentials resolveCredentials() { return null; } - @Override - public void refresh() { - - } } /** @@ -309,7 +326,7 @@ public void refresh() { private IOException expectProviderInstantiationFailure(String option, String expectedErrorText) throws Exception { return intercept(IOException.class, expectedErrorText, - () -> createAWSCredentialProviderSet( + () -> createAWSCredentialProviderList( TESTFILE_URI, createProviderConfiguration(option))); } @@ -359,12 +376,14 @@ private static void assertCredentialProviders( List> expectedClasses, AWSCredentialProviderList list) { assertNotNull(list); - List providers = list.getProviders(); - assertEquals(expectedClasses.size(), providers.size()); + List providers = list.getProviders(); + Assertions.assertThat(providers) + .describedAs("providers") + .hasSize(expectedClasses.size()); for (int i = 0; i < expectedClasses.size(); ++i) { Class expectedClass = expectedClasses.get(i); - AWSCredentialsProvider provider = providers.get(i); + AwsCredentialsProvider provider = providers.get(i); assertNotNull( String.format("At position %d, expected class is %s, but found null.", i, expectedClass), provider); @@ -380,7 +399,6 @@ private static void assertCredentialProviders( * @see S3ATestUtils#authenticationContains(Configuration, String). */ @Test - @SuppressWarnings("deprecation") public void testAuthenticationContainsProbes() { Configuration conf = new Configuration(false); assertFalse("found AssumedRoleCredentialProvider", @@ -398,7 +416,7 @@ public void testExceptionLogic() throws Throwable { // verify you can't get credentials from it NoAuthWithAWSException noAuth = intercept(NoAuthWithAWSException.class, AWSCredentialProviderList.NO_AWS_CREDENTIAL_PROVIDERS, - () -> providers.getCredentials()); + () -> providers.resolveCredentials()); // but that it closes safely providers.close(); @@ -447,11 +465,10 @@ public void testRefCounting() throws Throwable { providers.close(); assertEquals("Ref count after close() for " + providers, 0, providers.getRefCount()); - providers.refresh(); intercept(NoAuthWithAWSException.class, AWSCredentialProviderList.CREDENTIALS_REQUESTED_WHEN_CLOSED, - () -> providers.getCredentials()); + () -> providers.resolveCredentials()); } /** @@ -470,35 +487,17 @@ public void testIOEInConstructorPropagation() throws Throwable { /** * Credential provider which raises an IOE when constructed. */ - protected static class IOERaisingProvider implements AWSCredentialsProvider { + protected static class IOERaisingProvider extends AbstractProvider { public IOERaisingProvider(URI uri, Configuration conf) throws IOException { throw new InterruptedIOException("expected"); } - @Override - public AWSCredentials getCredentials() { - return null; - } - - @Override - public void refresh() { - - } } - private static final AWSCredentials EXPECTED_CREDENTIALS = new AWSCredentials() { - @Override - public String getAWSAccessKeyId() { - return "expectedAccessKey"; - } - - @Override - public String getAWSSecretKey() { - return "expectedSecret"; - } - }; + private static final AwsCredentials EXPECTED_CREDENTIALS = + AwsBasicCredentials.create("expectedAccessKey", "expectedSecret"); /** * Credential provider that takes a long time. @@ -510,7 +509,7 @@ public SlowProvider(@Nullable URI uri, Configuration conf) { } @Override - protected AWSCredentials createCredentials(Configuration config) throws IOException { + protected AwsCredentials createCredentials(Configuration config) throws IOException { // yield to other callers to induce race condition Thread.yield(); return EXPECTED_CREDENTIALS; @@ -524,13 +523,13 @@ public void testConcurrentAuthentication() throws Throwable { Configuration conf = createProviderConfiguration(SlowProvider.class.getName()); Path testFile = getCSVTestPath(conf); - AWSCredentialProviderList list = createAWSCredentialProviderSet(testFile.toUri(), conf); + AWSCredentialProviderList list = createAWSCredentialProviderList(testFile.toUri(), conf); SlowProvider provider = (SlowProvider) list.getProviders().get(0); ExecutorService pool = Executors.newFixedThreadPool(CONCURRENT_THREADS); - List> results = new ArrayList<>(); + List> results = new ArrayList<>(); try { assertFalse( @@ -546,15 +545,15 @@ public void testConcurrentAuthentication() throws Throwable { } for (int i = 0; i < CONCURRENT_THREADS; i++) { - results.add(pool.submit(() -> list.getCredentials())); + results.add(pool.submit(() -> list.resolveCredentials())); } - for (Future result : results) { - AWSCredentials credentials = result.get(); + for (Future result : results) { + AwsCredentials credentials = result.get(); assertEquals("Access key from credential provider", - "expectedAccessKey", credentials.getAWSAccessKeyId()); + "expectedAccessKey", credentials.accessKeyId()); assertEquals("Secret key from credential provider", - "expectedSecret", credentials.getAWSSecretKey()); + "expectedSecret", credentials.secretAccessKey()); } } finally { pool.awaitTermination(10, TimeUnit.SECONDS); @@ -563,7 +562,7 @@ public void testConcurrentAuthentication() throws Throwable { assertTrue( "Provider initialized without errors. isInitialized should be true", - provider.isInitialized()); + provider.isInitialized()); assertTrue( "Provider initialized without errors. hasCredentials should be true", provider.hasCredentials()); @@ -584,7 +583,7 @@ public ErrorProvider(@Nullable URI uri, Configuration conf) { } @Override - protected AWSCredentials createCredentials(Configuration config) throws IOException { + protected AwsCredentials createCredentials(Configuration config) throws IOException { throw new IOException("expected error"); } } @@ -594,12 +593,12 @@ public void testConcurrentAuthenticationError() throws Throwable { Configuration conf = createProviderConfiguration(ErrorProvider.class.getName()); Path testFile = getCSVTestPath(conf); - AWSCredentialProviderList list = createAWSCredentialProviderSet(testFile.toUri(), conf); + AWSCredentialProviderList list = createAWSCredentialProviderList(testFile.toUri(), conf); ErrorProvider provider = (ErrorProvider) list.getProviders().get(0); ExecutorService pool = Executors.newFixedThreadPool(CONCURRENT_THREADS); - List> results = new ArrayList<>(); + List> results = new ArrayList<>(); try { assertFalse("Provider not initialized. isInitialized should be false", @@ -613,10 +612,10 @@ public void testConcurrentAuthenticationError() throws Throwable { } for (int i = 0; i < CONCURRENT_THREADS; i++) { - results.add(pool.submit(() -> list.getCredentials())); + results.add(pool.submit(() -> list.resolveCredentials())); } - for (Future result : results) { + for (Future result : results) { interceptFuture(CredentialInitializationException.class, "expected error", result @@ -637,4 +636,39 @@ public void testConcurrentAuthenticationError() throws Throwable { "Provider initialization failed. getInitializationException should contain the error", provider.getInitializationException().getMessage().contains("expected error")); } + + + /** + * V2 Credentials whose factory method raises ClassNotFoundException. + * This will fall back to an attempted v1 load which will fail because it + * is the wrong type. + * The exception raised will be from the v2 instantiation attempt, + * not the v1 attempt. + */ + @Test + public void testV2ClassNotFound() throws Throwable { + InstantiationIOException expected = intercept(InstantiationIOException.class, + "simulated v2 CNFE", + () -> createAWSCredentialProviderList( + TESTFILE_URI, + createProviderConfiguration(V2CredentialProviderDoesNotInstantiate.class.getName()))); + // print for the curious + LOG.info("{}", expected.toString()); + } + + /** + * V2 credentials which raises an instantiation exception in + * the factory method. + */ + public static final class V2CredentialProviderDoesNotInstantiate + extends AbstractProvider { + + private V2CredentialProviderDoesNotInstantiate() { + } + + public static AwsCredentialsProvider create() throws ClassNotFoundException { + throw new ClassNotFoundException("simulated v2 CNFE"); + } + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index ffa2c81e58adf..4f329afe7ad51 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -30,7 +30,6 @@ import org.junit.Before; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -105,16 +104,14 @@ public void testWriteOperationHelperPartLimits() throws Throwable { noopAuditor(conf), AuditTestSupport.NOOP_SPAN, new MinimalWriteOperationHelperCallbacks()); - ByteArrayInputStream inputStream = new ByteArrayInputStream( - "a".getBytes()); // first one works String key = "destKey"; - woh.newUploadPartRequest(key, - "uploadId", 1, 1024, inputStream, null, 0L); + woh.newUploadPartRequestBuilder(key, + "uploadId", 1, 1024); // but ask past the limit and a PathIOE is raised intercept(PathIOException.class, key, - () -> woh.newUploadPartRequest(key, - "uploadId", 50000, 1024, inputStream, null, 0L)); + () -> woh.newUploadPartRequestBuilder(key, + "uploadId", 50000, 1024)); } static class StreamClosedException extends IOException {} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java index 62a99d7209263..a4162f212179b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java @@ -27,9 +27,9 @@ import java.net.URI; import java.util.Date; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -37,6 +37,7 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; + /** * deleteOnExit test for S3A. */ @@ -74,25 +75,25 @@ public void testDeleteOnExit() throws Exception { // unset S3CSE property from config to avoid pathIOE. conf.unset(Constants.S3_ENCRYPTION_ALGORITHM); testFs.initialize(uri, conf); - AmazonS3 testS3 = testFs.getAmazonS3ClientForTesting("mocking"); + S3Client testS3 = testFs.getS3AInternals().getAmazonS3Client("mocking"); Path path = new Path("/file"); String key = path.toUri().getPath().substring(1); - ObjectMetadata meta = new ObjectMetadata(); - meta.setContentLength(1L); - meta.setLastModified(new Date(2L)); - when(testS3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) - .thenReturn(meta); + HeadObjectResponse objectMetadata = + HeadObjectResponse.builder().contentLength(1L).lastModified(new Date(2L).toInstant()) + .build(); + when(testS3.headObject(argThat(correctGetMetadataRequest(BUCKET, key)))) + .thenReturn(objectMetadata); testFs.deleteOnExit(path); testFs.close(); assertEquals(0, testFs.getDeleteOnDnExitCount()); } - private ArgumentMatcher correctGetMetadataRequest( + private ArgumentMatcher correctGetMetadataRequest( String bucket, String key) { return request -> request != null - && request.getBucketName().equals(bucket) - && request.getKey().equals(key); + && request.bucket().equals(bucket) + && request.key().equals(key); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java index fd649c436bf59..c4d9ca7bcc04b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java @@ -18,30 +18,38 @@ package org.apache.hadoop.fs.s3a; +import static org.apache.hadoop.fs.s3a.AWSCredentialProviderList.maybeTranslateCredentialException; import static org.apache.hadoop.fs.s3a.Constants.*; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.verifyExceptionClass; import static org.apache.hadoop.fs.s3a.S3AUtils.*; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404; +import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*; import static org.junit.Assert.*; import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; -import java.net.SocketTimeoutException; import java.nio.file.AccessDeniedException; -import java.util.Collections; -import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.model.AmazonS3Exception; +import org.assertj.core.api.Assertions; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; import org.junit.Test; +import org.apache.hadoop.fs.s3a.api.UnsupportedRequestException; +import org.apache.hadoop.fs.s3a.audit.AuditFailureException; +import org.apache.hadoop.fs.s3a.audit.AuditOperationRejectedException; import org.apache.hadoop.fs.s3a.impl.ErrorTranslation; + import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; /** @@ -51,28 +59,27 @@ @SuppressWarnings("ThrowableNotThrown") public class TestS3AExceptionTranslation { - private static final org.apache.http.conn.ConnectTimeoutException - HTTP_CONNECTION_TIMEOUT_EX - = new org.apache.http.conn.ConnectTimeoutException("apache"); - private static final SocketTimeoutException SOCKET_TIMEOUT_EX - = new SocketTimeoutException("socket"); - @Test - public void test301ContainsEndpoint() throws Exception { - String bucket = "bucket.s3-us-west-2.amazonaws.com"; - int sc301 = 301; - AmazonS3Exception s3Exception = createS3Exception("wrong endpoint", sc301, - Collections.singletonMap(S3AUtils.ENDPOINT_KEY, - bucket)); + public void test301ContainsRegion() throws Exception { + String region = "us-west-1"; + + AwsErrorDetails redirectError = AwsErrorDetails.builder() + .sdkHttpResponse( + SdkHttpResponse.builder().putHeader(BUCKET_REGION_HEADER, region).build()) + .build(); + + S3Exception s3Exception = createS3Exception("wrong region", + SC_301_MOVED_PERMANENTLY, + redirectError); AWSRedirectException ex = verifyTranslated( AWSRedirectException.class, s3Exception); - assertStatusCode(sc301, ex); + assertStatusCode(SC_301_MOVED_PERMANENTLY, ex); assertNotNull(ex.getMessage()); - assertContained(ex.getMessage(), bucket); - assertContained(ex.getMessage(), ENDPOINT); - assertExceptionContains(ENDPOINT, ex, "endpoint"); - assertExceptionContains(bucket, ex, "bucket name"); + assertContained(ex.getMessage(), region); + assertContained(ex.getMessage(), AWS_REGION); + assertExceptionContains(AWS_REGION, ex, "region"); + assertExceptionContains(region, ex, "region name"); } protected void assertContained(String text, String contained) { @@ -88,17 +95,17 @@ protected void verifyTranslated( @Test public void test400isBad() throws Exception { - verifyTranslated(400, AWSBadRequestException.class); + verifyTranslated(SC_400_BAD_REQUEST, AWSBadRequestException.class); } @Test public void test401isNotPermittedFound() throws Exception { - verifyTranslated(401, AccessDeniedException.class); + verifyTranslated(SC_401_UNAUTHORIZED, AccessDeniedException.class); } @Test public void test403isNotPermittedFound() throws Exception { - verifyTranslated(403, AccessDeniedException.class); + verifyTranslated(SC_403_FORBIDDEN, AccessDeniedException.class); } /** @@ -106,7 +113,7 @@ public void test403isNotPermittedFound() throws Exception { */ @Test public void test404isNotFound() throws Exception { - verifyTranslated(SC_404, FileNotFoundException.class); + verifyTranslated(SC_404_NOT_FOUND, FileNotFoundException.class); } /** @@ -114,8 +121,11 @@ public void test404isNotFound() throws Exception { */ @Test public void testUnknownBucketException() throws Exception { - AmazonS3Exception ex404 = createS3Exception(SC_404); - ex404.setErrorCode(ErrorTranslation.AwsErrorCodes.E_NO_SUCH_BUCKET); + S3Exception ex404 = createS3Exception(b -> b + .statusCode(SC_404_NOT_FOUND) + .awsErrorDetails(AwsErrorDetails.builder() + .errorCode(ErrorTranslation.AwsErrorCodes.E_NO_SUCH_BUCKET) + .build())); verifyTranslated( UnknownStoreException.class, ex404); @@ -123,12 +133,12 @@ public void testUnknownBucketException() throws Exception { @Test public void test410isNotFound() throws Exception { - verifyTranslated(410, FileNotFoundException.class); + verifyTranslated(SC_410_GONE, FileNotFoundException.class); } @Test public void test416isEOF() throws Exception { - verifyTranslated(416, EOFException.class); + verifyTranslated(SC_416_RANGE_NOT_SATISFIABLE, EOFException.class); } @Test @@ -143,19 +153,21 @@ public void testGenericS3Exception() throws Exception { @Test public void testGenericServiceS3Exception() throws Exception { // service exception of no known type - AmazonServiceException ase = new AmazonServiceException("unwind"); - ase.setStatusCode(500); + AwsServiceException ase = AwsServiceException.builder() + .message("unwind") + .statusCode(SC_500_INTERNAL_SERVER_ERROR) + .build(); AWSServiceIOException ex = verifyTranslated( AWSStatus500Exception.class, ase); - assertStatusCode(500, ex); + assertStatusCode(SC_500_INTERNAL_SERVER_ERROR, ex); } protected void assertStatusCode(int expected, AWSServiceIOException ex) { assertNotNull("Null exception", ex); - if (expected != ex.getStatusCode()) { + if (expected != ex.statusCode()) { throw new AssertionError("Expected status code " + expected - + "but got " + ex.getStatusCode(), + + "but got " + ex.statusCode(), ex); } } @@ -164,23 +176,35 @@ protected void assertStatusCode(int expected, AWSServiceIOException ex) { public void testGenericClientException() throws Exception { // Generic Amazon exception verifyTranslated(AWSClientIOException.class, - new AmazonClientException("")); + SdkException.builder().message("").build()); } - private static AmazonS3Exception createS3Exception(int code) { - return createS3Exception("", code, null); + private static S3Exception createS3Exception( + Consumer consumer) { + S3Exception.Builder builder = S3Exception.builder() + .awsErrorDetails(AwsErrorDetails.builder() + .build()); + consumer.accept(builder); + return (S3Exception) builder.build(); } - private static AmazonS3Exception createS3Exception(String message, int code, - Map additionalDetails) { - AmazonS3Exception source = new AmazonS3Exception(message); - source.setStatusCode(code); - source.setAdditionalDetails(additionalDetails); + private static S3Exception createS3Exception(int code) { + return createS3Exception(b -> b.message("").statusCode(code)); + } + + private static S3Exception createS3Exception(String message, int code, + AwsErrorDetails additionalDetails) { + + S3Exception source = (S3Exception) S3Exception.builder() + .message(message) + .statusCode(code) + .awsErrorDetails(additionalDetails) + .build(); return source; } private static E verifyTranslated(Class clazz, - AmazonClientException exception) throws Exception { + SdkException exception) throws Exception { // Verifying that the translated exception have the correct error message. IOException ioe = translateException("test", "/", exception); assertExceptionContains(exception.getMessage(), ioe, @@ -212,16 +236,98 @@ public void testInterruptExceptionDetecting() throws Throwable { public void testExtractInterrupted() throws Throwable { throw extractException("", "", new ExecutionException( - new AmazonClientException( - new InterruptedException("")))); + SdkException.builder() + .cause(new InterruptedException("")) + .build())); } @Test(expected = InterruptedIOException.class) public void testExtractInterruptedIO() throws Throwable { throw extractException("", "", new ExecutionException( - new AmazonClientException( - new InterruptedIOException("")))); + SdkException.builder() + .cause(new InterruptedIOException("")) + .build())); + } + + private SdkClientException sdkClientException(String message, Throwable cause) { + return SdkClientException.builder() + .message(message) + .cause(cause) + .build(); + } + @Test + public void testTranslateCredentialException() throws Throwable { + verifyExceptionClass(AccessDeniedException.class, + maybeTranslateCredentialException("/", + new CredentialInitializationException("Credential initialization failed"))); + } + + @Test + public void testTranslateNestedCredentialException() throws Throwable { + final AccessDeniedException ex = + verifyExceptionClass(AccessDeniedException.class, + maybeTranslateCredentialException("/", + sdkClientException("", + new CredentialInitializationException("Credential initialization failed")))); + // unwrap and verify that the initial client exception has been stripped + final Throwable cause = ex.getCause(); + Assertions.assertThat(cause) + .isInstanceOf(CredentialInitializationException.class); + CredentialInitializationException cie = (CredentialInitializationException) cause; + Assertions.assertThat(cie.retryable()) + .describedAs("Retryable flag") + .isFalse(); + } + + + @Test + public void testTranslateNonCredentialException() throws Throwable { + Assertions.assertThat( + maybeTranslateCredentialException("/", + sdkClientException("not a credential exception", null))) + .isNull(); + Assertions.assertThat( + maybeTranslateCredentialException("/", + sdkClientException("", sdkClientException("not a credential exception", null)))) + .isNull(); + } + + @Test + public void testTranslateAuditException() throws Throwable { + verifyExceptionClass(AccessDeniedException.class, + maybeTranslateAuditException("/", + new AuditFailureException("failed"))); + } + + @Test + public void testTranslateNestedAuditException() throws Throwable { + verifyExceptionClass(AccessDeniedException.class, + maybeTranslateAuditException("/", + sdkClientException("", new AuditFailureException("failed")))); + } + + + @Test + public void testTranslateNestedAuditRejectedException() throws Throwable { + final UnsupportedRequestException ex = + verifyExceptionClass(UnsupportedRequestException.class, + maybeTranslateAuditException("/", + sdkClientException("", new AuditOperationRejectedException("rejected")))); + Assertions.assertThat(ex.getCause()) + .isInstanceOf(AuditOperationRejectedException.class); + } + + @Test + public void testTranslateNonAuditException() throws Throwable { + Assertions.assertThat( + maybeTranslateAuditException("/", + sdkClientException("not an audit exception", null))) + .isNull(); + Assertions.assertThat( + maybeTranslateAuditException("/", + sdkClientException("", sdkClientException("not an audit exception", null)))) + .isNull(); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java index 34a275b580f25..1a2a21a6e5111 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java @@ -21,21 +21,22 @@ import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.FileNotFoundException; +import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import software.amazon.awssdk.services.s3.model.CommonPrefix; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Object; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -44,6 +45,7 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; + /** * S3A tests for getFileStatus using mock S3 client. */ @@ -53,17 +55,16 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { public void testFile() throws Exception { Path path = new Path("/file"); String key = path.toUri().getPath().substring(1); - ObjectMetadata meta = new ObjectMetadata(); - meta.setContentLength(1L); - meta.setLastModified(new Date(2L)); - when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) - .thenReturn(meta); + HeadObjectResponse objectMetadata = + HeadObjectResponse.builder().contentLength(1L).lastModified(new Date(2L).toInstant()) + .build(); + when(s3.headObject(argThat(correctGetMetadataRequest(BUCKET, key)))).thenReturn(objectMetadata); FileStatus stat = fs.getFileStatus(path); assertNotNull(stat); assertEquals(fs.makeQualified(path), stat.getPath()); assertTrue(stat.isFile()); - assertEquals(meta.getContentLength(), stat.getLen()); - assertEquals(meta.getLastModified().getTime(), stat.getModificationTime()); + assertEquals(objectMetadata.contentLength().longValue(), stat.getLen()); + assertEquals(Date.from(objectMetadata.lastModified()).getTime(), stat.getModificationTime()); ContractTestUtils.assertNotErasureCoded(fs, path); assertTrue(path + " should have erasure coding unset in " + "FileStatus#toString(): " + stat, @@ -74,17 +75,16 @@ public void testFile() throws Exception { public void testFakeDirectory() throws Exception { Path path = new Path("/dir"); String key = path.toUri().getPath().substring(1); - when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) + when(s3.headObject(argThat(correctGetMetadataRequest(BUCKET, key)))) .thenThrow(NOT_FOUND); String keyDir = key + "/"; - ListObjectsV2Result listResult = new ListObjectsV2Result(); - S3ObjectSummary objectSummary = new S3ObjectSummary(); - objectSummary.setKey(keyDir); - objectSummary.setSize(0L); - listResult.getObjectSummaries().add(objectSummary); + List s3Objects = new ArrayList<>(1); + s3Objects.add(S3Object.builder().key(keyDir).size(0L).build()); + ListObjectsV2Response listObjectsV2Response = + ListObjectsV2Response.builder().contents(s3Objects).build(); when(s3.listObjectsV2(argThat( matchListV2Request(BUCKET, keyDir)) - )).thenReturn(listResult); + )).thenReturn(listObjectsV2Response); FileStatus stat = fs.getFileStatus(path); assertNotNull(stat); assertEquals(fs.makeQualified(path), stat.getPath()); @@ -95,12 +95,13 @@ public void testFakeDirectory() throws Exception { public void testImplicitDirectory() throws Exception { Path path = new Path("/dir"); String key = path.toUri().getPath().substring(1); - when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) + when(s3.headObject(argThat(correctGetMetadataRequest(BUCKET, key)))) .thenThrow(NOT_FOUND); - when(s3.getObjectMetadata(argThat( + when(s3.headObject(argThat( correctGetMetadataRequest(BUCKET, key + "/")) )).thenThrow(NOT_FOUND); - setupListMocks(Collections.singletonList("dir/"), Collections.emptyList()); + setupListMocks(Collections.singletonList(CommonPrefix.builder().prefix("dir/").build()), + Collections.emptyList()); FileStatus stat = fs.getFileStatus(path); assertNotNull(stat); assertEquals(fs.makeQualified(path), stat.getPath()); @@ -115,9 +116,9 @@ public void testImplicitDirectory() throws Exception { public void testRoot() throws Exception { Path path = new Path("/"); String key = path.toUri().getPath().substring(1); - when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) + when(s3.headObject(argThat(correctGetMetadataRequest(BUCKET, key)))) .thenThrow(NOT_FOUND); - when(s3.getObjectMetadata(argThat( + when(s3.headObject(argThat( correctGetMetadataRequest(BUCKET, key + "/") ))).thenThrow(NOT_FOUND); setupListMocks(Collections.emptyList(), Collections.emptyList()); @@ -132,9 +133,9 @@ public void testRoot() throws Exception { public void testNotFound() throws Exception { Path path = new Path("/dir"); String key = path.toUri().getPath().substring(1); - when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) + when(s3.headObject(argThat(correctGetMetadataRequest(BUCKET, key)))) .thenThrow(NOT_FOUND); - when(s3.getObjectMetadata(argThat( + when(s3.headObject(argThat( correctGetMetadataRequest(BUCKET, key + "/") ))).thenThrow(NOT_FOUND); setupListMocks(Collections.emptyList(), Collections.emptyList()); @@ -142,36 +143,38 @@ public void testNotFound() throws Exception { fs.getFileStatus(path); } - private void setupListMocks(List prefixes, - List summaries) { - + private void setupListMocks(List prefixes, + List s3Objects) { // V1 list API mock - ObjectListing objects = mock(ObjectListing.class); - when(objects.getCommonPrefixes()).thenReturn(prefixes); - when(objects.getObjectSummaries()).thenReturn(summaries); - when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects); + ListObjectsResponse v1Response = ListObjectsResponse.builder() + .commonPrefixes(prefixes) + .contents(s3Objects) + .build(); + when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(v1Response); // V2 list API mock - ListObjectsV2Result v2Result = mock(ListObjectsV2Result.class); - when(v2Result.getCommonPrefixes()).thenReturn(prefixes); - when(v2Result.getObjectSummaries()).thenReturn(summaries); - when(s3.listObjectsV2(any(ListObjectsV2Request.class))) - .thenReturn(v2Result); + ListObjectsV2Response v2Result = ListObjectsV2Response.builder() + .commonPrefixes(prefixes) + .contents(s3Objects) + .build(); + when(s3.listObjectsV2( + any(software.amazon.awssdk.services.s3.model.ListObjectsV2Request.class))).thenReturn( + v2Result); } - private ArgumentMatcher correctGetMetadataRequest( + private ArgumentMatcher correctGetMetadataRequest( String bucket, String key) { return request -> request != null - && request.getBucketName().equals(bucket) - && request.getKey().equals(key); + && request.bucket().equals(bucket) + && request.key().equals(key); } private ArgumentMatcher matchListV2Request( String bucket, String key) { return (ListObjectsV2Request request) -> { return request != null - && request.getBucketName().equals(bucket) - && request.getPrefix().equals(key); + && request.bucket().equals(bucket) + && request.prefix().equals(key); }; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java index c62bf5daca3a4..da1284343da9f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java @@ -19,16 +19,18 @@ package org.apache.hadoop.fs.s3a; import javax.net.ssl.SSLException; +import java.io.FilterInputStream; import java.io.IOException; import java.net.SocketException; import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; -import com.amazonaws.SdkClientException; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import org.junit.Test; import org.apache.commons.io.IOUtils; @@ -37,6 +39,7 @@ import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.util.functional.CallableRaisingIOE; + import static java.lang.Math.min; import static org.apache.hadoop.util.functional.FutureIO.eval; import static org.junit.Assert.assertArrayEquals; @@ -121,13 +124,22 @@ private S3AInputStream getMockedS3AInputStream() { * @return mocked object. */ private S3AInputStream.InputStreamCallbacks getMockedInputStreamCallback() { - return new S3AInputStream.InputStreamCallbacks() { + GetObjectResponse objectResponse = GetObjectResponse.builder() + .eTag("test-etag") + .build(); - private final S3Object mockedS3Object = getMockedS3Object(); + ResponseInputStream[] responseInputStreams = + new ResponseInputStream[] { + getMockedInputStream(objectResponse, true), + getMockedInputStream(objectResponse, true), + getMockedInputStream(objectResponse, false) + }; + + return new S3AInputStream.InputStreamCallbacks() { private Integer mockedS3ObjectIndex = 0; @Override - public S3Object getObject(GetObjectRequest request) { + public ResponseInputStream getObject(GetObjectRequest request) { // Set s3 client to return mocked s3object with defined read behavior. mockedS3ObjectIndex++; // open() -> lazySeek() -> reopen() @@ -144,14 +156,17 @@ public S3Object getObject(GetObjectRequest request) { // -> getObjectContent(objectInputStreamGood)-> objectInputStreamGood // -> wrappedStream.read if (mockedS3ObjectIndex == 3) { - throw new SdkClientException("Failed to get S3Object"); + throw AwsServiceException.builder() + .message("Failed to get S3Object") + .awsErrorDetails(AwsErrorDetails.builder().errorCode("test-code").build()) + .build(); } - return mockedS3Object; + return responseInputStreams[min(mockedS3ObjectIndex, responseInputStreams.length) - 1]; } @Override - public GetObjectRequest newGetRequest(String key) { - return new GetObjectRequest(fs.getBucket(), key); + public GetObjectRequest.Builder newGetRequestBuilder(String key) { + return GetObjectRequest.builder().bucket(fs.getBucket()).key(key); } @Override @@ -166,70 +181,41 @@ public void close() { } /** - * Get mocked S3Object that returns bad input stream on the initial of - * getObjectContent calls. - * - * @return mocked object. - */ - private S3Object getMockedS3Object() { - S3ObjectInputStream objectInputStreamBad1 = getMockedInputStream(true); - S3ObjectInputStream objectInputStreamBad2 = getMockedInputStream(true); - S3ObjectInputStream objectInputStreamGood = getMockedInputStream(false); - - return new S3Object() { - private final S3ObjectInputStream[] inputStreams = - {objectInputStreamBad1, objectInputStreamBad2, objectInputStreamGood}; - - private Integer inputStreamIndex = 0; - - @Override - public S3ObjectInputStream getObjectContent() { - // Set getObjectContent behavior: - // Returns bad stream twice, and good stream afterwards. - inputStreamIndex++; - return inputStreams[min(inputStreamIndex, inputStreams.length) - 1]; - } - - @Override - public ObjectMetadata getObjectMetadata() { - // Set getObjectMetadata behavior: returns dummy metadata - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("ETag", "test-etag"); - return metadata; - } - }; - } - - /** - * Get mocked S3ObjectInputStream where we can trigger IOException to + * Get mocked ResponseInputStream where we can trigger IOException to * simulate the read failure. * * @param triggerFailure true when a failure injection is enabled. * @return mocked object. */ - private S3ObjectInputStream getMockedInputStream(boolean triggerFailure) { - return new S3ObjectInputStream(IOUtils.toInputStream(INPUT, StandardCharsets.UTF_8), null) { - - private final IOException exception = - new SSLException(new SocketException("Connection reset")); - - @Override - public int read() throws IOException { - int result = super.read(); - if (triggerFailure) { - throw exception; - } - return result; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int result = super.read(b, off, len); - if (triggerFailure) { - throw exception; - } - return result; - } - }; + private ResponseInputStream getMockedInputStream( + GetObjectResponse objectResponse, boolean triggerFailure) { + + FilterInputStream inputStream = + new FilterInputStream(IOUtils.toInputStream(INPUT, StandardCharsets.UTF_8)) { + + private final IOException exception = + new SSLException(new SocketException("Connection reset")); + + @Override + public int read() throws IOException { + int result = super.read(); + if (triggerFailure) { + throw exception; + } + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = super.read(b, off, len); + if (triggerFailure) { + throw exception; + } + return result; + } + }; + + return new ResponseInputStream(objectResponse, + AbortableInputStream.create(inputStream, () -> {})); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java index e05ee25adfa74..0982c8cbd4761 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java @@ -20,18 +20,17 @@ import java.io.IOException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.Protocol; import org.assertj.core.api.Assertions; import org.junit.Test; +import software.amazon.awssdk.http.apache.ProxyConfiguration; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.impl.AWSClientConfig; import org.apache.hadoop.test.AbstractHadoopTestBase; import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST; import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT; import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED; -import static org.apache.hadoop.fs.s3a.S3AUtils.initProxySupport; /** * Tests to verify {@link S3AUtils} translates the proxy configurations @@ -79,11 +78,16 @@ public void testProxyDefault() throws IOException { private void verifyProxy(Configuration proxyConfig, boolean isExpectedSecured) throws IOException { - ClientConfiguration awsConf = new ClientConfiguration(); - initProxySupport(proxyConfig, "test-bucket", awsConf); - Assertions.assertThat(awsConf.getProxyProtocol()) + ProxyConfiguration config = + AWSClientConfig.createProxyConfiguration(proxyConfig, "testBucket"); + ProxyConfiguration asyncConfig = + AWSClientConfig.createProxyConfiguration(proxyConfig, "testBucket"); + Assertions.assertThat(config.scheme()) .describedAs("Proxy protocol not as expected") - .isEqualTo(isExpectedSecured ? Protocol.HTTPS : Protocol.HTTP); + .isEqualTo(isExpectedSecured ? "https" : "http"); + Assertions.assertThat(asyncConfig.scheme()) + .describedAs("Proxy protocol not as expected") + .isEqualTo(isExpectedSecured ? "https" : "http"); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java index 0e105c25c3a45..643db02087b46 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java @@ -18,17 +18,22 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.junit.Test; + import java.io.IOException; -import java.util.Date; +import java.io.InputStream; +import java.time.Instant; import static org.junit.Assert.assertEquals; @@ -40,10 +45,10 @@ import static org.mockito.Mockito.when; /** - * Uses mocks to check that the {@link S3ObjectInputStream} is closed when - * {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer} is called. Unlike the - * other unbuffer tests, this specifically tests that the underlying S3 object - * stream is closed. + * Uses mocks to check that the {@link ResponseInputStream} is + * closed when {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer} is called. + * Unlike the other unbuffer tests, this specifically tests that the underlying + * S3 object stream is closed. */ public class TestS3AUnbuffer extends AbstractS3AMockTest { @@ -51,22 +56,27 @@ public class TestS3AUnbuffer extends AbstractS3AMockTest { public void testUnbuffer() throws IOException { // Create mock ObjectMetadata for getFileStatus() Path path = new Path("/file"); - ObjectMetadata meta = mock(ObjectMetadata.class); - when(meta.getContentLength()).thenReturn(1L); - when(meta.getLastModified()).thenReturn(new Date(2L)); - when(meta.getETag()).thenReturn("mock-etag"); - when(s3.getObjectMetadata(any())).thenReturn(meta); + HeadObjectResponse objectMetadata = HeadObjectResponse.builder() + .contentLength(1L) + .lastModified(Instant.ofEpochMilli(2L)) + .eTag("mock-etag") + .build(); + when(s3.headObject((HeadObjectRequest) any())).thenReturn(objectMetadata); - // Create mock S3ObjectInputStream and S3Object for open() - S3ObjectInputStream objectStream = mock(S3ObjectInputStream.class); + // Create mock ResponseInputStream and GetObjectResponse for open() + GetObjectResponse objectResponse = GetObjectResponse.builder() + .contentLength(1L) + .lastModified(Instant.ofEpochMilli(2L)) + .eTag("mock-etag") + .build(); + InputStream objectStream = mock(InputStream.class); when(objectStream.read()).thenReturn(-1); when(objectStream.read(any(byte[].class))).thenReturn(-1); when(objectStream.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1); - - S3Object s3Object = mock(S3Object.class); - when(s3Object.getObjectContent()).thenReturn(objectStream); - when(s3Object.getObjectMetadata()).thenReturn(meta); - when(s3.getObject(any())).thenReturn(s3Object); + ResponseInputStream getObjectResponseInputStream = + new ResponseInputStream(objectResponse, + AbortableInputStream.create(objectStream, () -> {})); + when(s3.getObject((GetObjectRequest) any())).thenReturn(getObjectResponseInputStream); // Call read and then unbuffer FSDataInputStream stream = fs.open(path); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java index 42de7cdffc80e..66d9032e858eb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java @@ -18,18 +18,18 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.s3.Headers; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.transfer.model.CopyResult; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.CopyObjectResult; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; @@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.CHANGE_DETECTED; import static org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.createPolicy; import static org.apache.hadoop.fs.s3a.impl.ChangeTracker.CHANGE_REPORTED_BY_S3; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_412_PRECONDITION_FAILED; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -68,7 +69,7 @@ public void testVersionCheckingHandlingNoVersions() throws Throwable { ChangeDetectionPolicy.Source.VersionId, false); assertFalse("Tracker should not have applied contraints " + tracker, - tracker.maybeApplyConstraint(newGetObjectRequest())); + tracker.maybeApplyConstraint(newGetObjectRequestBuilder())); tracker.processResponse( newResponse(null, null), "", 0); @@ -96,7 +97,7 @@ public void testEtagCheckingWarn() throws Throwable { ChangeDetectionPolicy.Source.ETag, false); assertFalse("Tracker should not have applied constraints " + tracker, - tracker.maybeApplyConstraint(newGetObjectRequest())); + tracker.maybeApplyConstraint(newGetObjectRequestBuilder())); tracker.processResponse( newResponse("e1", null), "", 0); @@ -122,13 +123,13 @@ public void testVersionCheckingOnClient() throws Throwable { ChangeDetectionPolicy.Source.VersionId, false); assertFalse("Tracker should not have applied constraints " + tracker, - tracker.maybeApplyConstraint(newGetObjectRequest())); + tracker.maybeApplyConstraint(newGetObjectRequestBuilder())); tracker.processResponse( newResponse(null, "rev1"), "", 0); assertTrackerMismatchCount(tracker, 0); assertRevisionId(tracker, "rev1"); - GetObjectRequest request = newGetObjectRequest(); + GetObjectRequest request = newGetObjectRequestBuilder().build(); expectChangeException(tracker, newResponse(null, "rev2"), "change detected"); // mismatch was noted (so gets to FS stats) @@ -149,14 +150,14 @@ public void testVersionCheckingOnServer() throws Throwable { ChangeDetectionPolicy.Source.VersionId, false); assertFalse("Tracker should not have applied contraints " + tracker, - tracker.maybeApplyConstraint(newGetObjectRequest())); + tracker.maybeApplyConstraint(newGetObjectRequestBuilder())); tracker.processResponse( newResponse(null, "rev1"), "", 0); assertTrackerMismatchCount(tracker, 0); assertRevisionId(tracker, "rev1"); - GetObjectRequest request = newGetObjectRequest(); - assertConstraintApplied(tracker, request); + GetObjectRequest.Builder builder = newGetObjectRequestBuilder(); + assertConstraintApplied(tracker, builder); // now, the tracker expects a null response expectChangeException(tracker, null, CHANGE_REPORTED_BY_S3); assertTrackerMismatchCount(tracker, 1); @@ -249,31 +250,33 @@ public void testCopyVersionMismatch() throws Throwable { // 412 is translated to RemoteFileChangedException // note: this scenario is never currently hit due to // https://github.com/aws/aws-sdk-java/issues/1644 - AmazonServiceException awsException = - new AmazonServiceException("aws exception"); - awsException.setStatusCode(ChangeTracker.SC_PRECONDITION_FAILED); + AwsServiceException awsException = + AwsServiceException.builder() + .message("aws exception") + .statusCode(SC_412_PRECONDITION_FAILED) + .build(); expectChangeException(tracker, awsException, "copy", RemoteFileChangedException.PRECONDITIONS_FAILED); // processing another type of exception does nothing - tracker.processException(new SdkBaseException("foo"), "copy"); + tracker.processException(SdkException.builder().message("foo").build(), "copy"); } protected void assertConstraintApplied(final ChangeTracker tracker, - final GetObjectRequest request) { + final GetObjectRequest.Builder builder) { assertTrue("Tracker should have applied contraints " + tracker, - tracker.maybeApplyConstraint(request)); + tracker.maybeApplyConstraint(builder)); } protected void assertConstraintApplied(final ChangeTracker tracker, - final CopyObjectRequest request) throws PathIOException { + final CopyObjectRequest.Builder requestBuilder) throws PathIOException { assertTrue("Tracker should have applied contraints " + tracker, - tracker.maybeApplyConstraint(request)); + tracker.maybeApplyConstraint(requestBuilder)); } protected RemoteFileChangedException expectChangeException( final ChangeTracker tracker, - final S3Object response, + final GetObjectResponse response, final String message) throws Exception { return expectException(tracker, response, message, RemoteFileChangedException.class); @@ -281,7 +284,7 @@ protected RemoteFileChangedException expectChangeException( protected RemoteFileChangedException expectChangeException( final ChangeTracker tracker, - final SdkBaseException exception, + final SdkException exception, final String operation, final String message) throws Exception { return expectException(tracker, exception, operation, message, @@ -290,7 +293,7 @@ protected RemoteFileChangedException expectChangeException( protected PathIOException expectNoVersionAttributeException( final ChangeTracker tracker, - final S3Object response, + final GetObjectResponse response, final String message) throws Exception { return expectException(tracker, response, message, NoVersionAttributeException.class); @@ -298,7 +301,7 @@ protected PathIOException expectNoVersionAttributeException( protected PathIOException expectNoVersionAttributeException( final ChangeTracker tracker, - final CopyResult response, + final CopyObjectResponse response, final String message) throws Exception { return expectException(tracker, response, message, NoVersionAttributeException.class); @@ -306,7 +309,7 @@ protected PathIOException expectNoVersionAttributeException( protected T expectException( final ChangeTracker tracker, - final S3Object response, + final GetObjectResponse response, final String message, final Class clazz) throws Exception { return intercept( @@ -320,7 +323,7 @@ protected T expectException( protected T expectException( final ChangeTracker tracker, - final CopyResult response, + final CopyObjectResponse response, final String message, final Class clazz) throws Exception { return intercept( @@ -334,7 +337,7 @@ protected T expectException( protected T expectException( final ChangeTracker tracker, - final SdkBaseException exception, + final SdkException exception, final String operation, final String message, final Class clazz) throws Exception { @@ -389,48 +392,36 @@ protected ChangeTracker newTracker(final ChangeDetectionPolicy.Mode mode, if (objectAttributes.getVersionId() == null && objectAttributes.getETag() == null) { assertFalse("Tracker should not have applied constraints " + tracker, - tracker.maybeApplyConstraint(newGetObjectRequest())); + tracker.maybeApplyConstraint(newGetObjectRequestBuilder())); } return tracker; } - private GetObjectRequest newGetObjectRequest() { - return new GetObjectRequest(BUCKET, OBJECT); + private GetObjectRequest.Builder newGetObjectRequestBuilder() { + return GetObjectRequest.builder().bucket(BUCKET).key(OBJECT); } - private CopyObjectRequest newCopyObjectRequest() { - return new CopyObjectRequest(BUCKET, OBJECT, BUCKET, DEST_OBJECT); + private CopyObjectRequest.Builder newCopyObjectRequest() { + return CopyObjectRequest.builder().sourceBucket(BUCKET).sourceKey(OBJECT) + .destinationBucket(BUCKET).destinationKey(DEST_OBJECT); } - private CopyResult newCopyResult(String eTag, String versionId) { - CopyResult copyResult = new CopyResult(); - copyResult.setSourceBucketName(BUCKET); - copyResult.setSourceKey(OBJECT); - copyResult.setDestinationBucketName(BUCKET); - copyResult.setDestinationKey(DEST_OBJECT); - copyResult.setETag(eTag); - copyResult.setVersionId(versionId); - return copyResult; + private CopyObjectResponse newCopyResult(String eTag, String versionId) { + CopyObjectResponse.Builder copyObjectResponseBuilder = CopyObjectResponse.builder(); + + return copyObjectResponseBuilder.versionId(versionId) + .copyObjectResult(CopyObjectResult.builder().eTag(eTag).build()).build(); } - private S3Object newResponse(String etag, String versionId) { - ObjectMetadata md = new ObjectMetadata(); + private GetObjectResponse newResponse(String etag, String versionId) { + GetObjectResponse.Builder builder = GetObjectResponse.builder(); if (etag != null) { - md.setHeader(Headers.ETAG, etag); + builder.eTag(etag); } if (versionId != null) { - md.setHeader(Headers.S3_VERSION_ID, versionId); + builder.versionId(versionId); } - S3Object response = emptyResponse(); - response.setObjectMetadata(md); - return response; - } - - private S3Object emptyResponse() { - S3Object response = new S3Object(); - response.setBucketName(BUCKET); - response.setKey(OBJECT); - return response; + return builder.build(); } private S3ObjectAttributes objectAttributes( diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestWildflyAndOpenSSLBinding.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestWildflyAndOpenSSLBinding.java index a2b013f468a79..9e903fd85ff49 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestWildflyAndOpenSSLBinding.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestWildflyAndOpenSSLBinding.java @@ -20,10 +20,9 @@ import java.io.IOException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.Protocol; import org.junit.Before; import org.junit.Test; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; @@ -74,7 +73,7 @@ public void testUnknownMode() throws Throwable { Configuration conf = new Configuration(false); conf.set(SSL_CHANNEL_MODE, "no-such-mode "); intercept(IllegalArgumentException.class, () -> - bindSSLChannelMode(conf, new ClientConfiguration())); + bindSSLChannelMode(conf, ApacheHttpClient.builder())); } @Test @@ -143,9 +142,7 @@ private DelegatingSSLSocketFactory.SSLChannelMode bindSocketFactory( DelegatingSSLSocketFactory.resetDefaultFactory(); Configuration conf = new Configuration(false); conf.set(SSL_CHANNEL_MODE, channelMode.name()); - ClientConfiguration awsConf = new ClientConfiguration(); - awsConf.setProtocol(Protocol.HTTPS); - bindSSLChannelMode(conf, awsConf); + bindSSLChannelMode(conf, ApacheHttpClient.builder()); return DelegatingSSLSocketFactory.getDefaultFactory().getChannelMode(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java new file mode 100644 index 0000000000000..b0e1b57d75471 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/adapter/TestV1CredentialsProvider.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.adapter; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; +import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider; +import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; + +import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.DEFAULT_CSVTEST_FILE; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ANONYMOUS_CREDENTIALS_V1; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.EC2_CONTAINER_CREDENTIALS_V1; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.ENVIRONMENT_CREDENTIALS_V1; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for v1 to v2 credential provider logic. + */ +public class TestV1CredentialsProvider { + + /** + * URI of the landsat images. + */ + private static final URI TESTFILE_URI = new Path( + DEFAULT_CSVTEST_FILE).toUri(); + + private static final Logger LOG = LoggerFactory.getLogger(TestV1CredentialsProvider.class); + + + @Test + public void testV1V2Mapping() throws Exception { + URI uri1 = new URI("s3a://bucket1"); + + List> expectedClasses = + Arrays.asList( + IAMInstanceCredentialsProvider.class, + AnonymousAWSCredentialsProvider.class, + EnvironmentVariableCredentialsProvider.class); + Configuration conf = + createProviderConfiguration(buildClassList( + EC2_CONTAINER_CREDENTIALS_V1, + ANONYMOUS_CREDENTIALS_V1, + ENVIRONMENT_CREDENTIALS_V1)); + AWSCredentialProviderList list1 = createAWSCredentialProviderList( + uri1, conf); + assertCredentialProviders(expectedClasses, list1); + } + + @Test + public void testV1Wrapping() throws Exception { + URI uri1 = new URI("s3a://bucket1"); + + List> expectedClasses = + Arrays.asList( + V1ToV2AwsCredentialProviderAdapter.class, + V1ToV2AwsCredentialProviderAdapter.class); + Configuration conf = + createProviderConfiguration(buildClassList( + LegacyV1CredentialProvider.class.getName(), + LegacyV1CredentialProviderWithConf.class.getName())); + AWSCredentialProviderList list1 = createAWSCredentialProviderList( + uri1, conf); + assertCredentialProviders(expectedClasses, list1); + } + + private String buildClassList(String... classes) { + return Arrays.stream(classes) + .collect(Collectors.joining(",")); + } + + + /** + * Expect a provider to raise an exception on failure. + * @param option aws provider option string. + * @param expectedErrorText error text to expect + * @return the exception raised + * @throws Exception any unexpected exception thrown. + */ + private IOException expectProviderInstantiationFailure(String option, + String expectedErrorText) throws Exception { + return intercept(IOException.class, expectedErrorText, + () -> createAWSCredentialProviderList( + TESTFILE_URI, + createProviderConfiguration(option))); + } + + /** + * Create a configuration with a specific provider. + * @param providerOption option for the aws credential provider option. + * @return a configuration to use in test cases + */ + private Configuration createProviderConfiguration( + final String providerOption) { + Configuration conf = new Configuration(false); + conf.set(AWS_CREDENTIALS_PROVIDER, providerOption); + return conf; + } + + /** + * Asserts expected provider classes in list. + * @param expectedClasses expected provider classes + * @param list providers to check + */ + private static void assertCredentialProviders( + List> expectedClasses, + AWSCredentialProviderList list) { + assertNotNull(list); + List providers = list.getProviders(); + Assertions.assertThat(providers) + .describedAs("providers") + .hasSize(expectedClasses.size()); + for (int i = 0; i < expectedClasses.size(); ++i) { + Class expectedClass = + expectedClasses.get(i); + AwsCredentialsProvider provider = providers.get(i); + assertNotNull( + String.format("At position %d, expected class is %s, but found null.", + i, expectedClass), provider); + assertTrue( + String.format("At position %d, expected class is %s, but found %s.", + i, expectedClass, provider.getClass()), + expectedClass.isAssignableFrom(provider.getClass())); + } + } + + + public static class LegacyV1CredentialProvider implements AWSCredentialsProvider { + + public LegacyV1CredentialProvider() { + } + + @Override + public AWSCredentials getCredentials() { + return null; + } + + @Override + public void refresh() { + + } + } + + /** + * V1 credentials with a configuration constructor. + */ + public static final class LegacyV1CredentialProviderWithConf + extends LegacyV1CredentialProvider { + + public LegacyV1CredentialProviderWithConf(Configuration conf) { + } + } + + /** + * V1 Credentials whose factory method raises ClassNotFoundException. + * Expect this to fail rather than trigger recursive recovery; + * exception will be wrapped with something intended to be informative. + */ + @Test + public void testV1InstantiationFailurePropagation() throws Throwable { + InstantiationIOException expected = intercept(InstantiationIOException.class, + "simulated CNFE", + () -> createAWSCredentialProviderList( + TESTFILE_URI, + createProviderConfiguration(V1CredentialProviderDoesNotInstantiate.class.getName()))); + // print for the curious + LOG.info("{}", expected.toString()); + } + + + /** + * V1 credentials which raises an instantiation exception. + */ + public static final class V1CredentialProviderDoesNotInstantiate + extends LegacyV1CredentialProvider { + + private V1CredentialProviderDoesNotInstantiate() { + } + + public static AWSCredentialsProvider getInstance() throws ClassNotFoundException { + throw new ClassNotFoundException("simulated CNFE"); + } + } + + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index 298c1444bb9b3..e2297e37e50c4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -21,13 +21,23 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Map; -import java.util.function.Consumer; import java.util.stream.Collectors; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; + +import software.amazon.awssdk.awscore.AwsExecutionAttribute; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; + import org.junit.After; import org.junit.Before; import org.slf4j.Logger; @@ -137,22 +147,58 @@ protected AuditSpanS3A activeSpan() { /** * Create a head request and pass it through the manager's beforeExecution() * callback. + * * @return a processed request. */ - protected GetObjectMetadataRequest head() { - return manager.beforeExecution( - requestFactory.newGetObjectMetadataRequest("/")); + protected SdkHttpRequest head() { + HeadObjectRequest.Builder headObjectRequestBuilder = + requestFactory.newHeadObjectRequestBuilder("/"); + manager.requestCreated(headObjectRequestBuilder); + HeadObjectRequest headObjectRequest = headObjectRequestBuilder.build(); + ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build(); + InterceptorContext context = InterceptorContext.builder() + .request(headObjectRequest) + .httpRequest(SdkHttpRequest.builder() + .uri(URI.create("https://test")) + .method(SdkHttpMethod.HEAD) + .build()) + .build(); + manager.beforeExecution(context, executionAttributes); + return manager.modifyHttpRequest(context, executionAttributes); } /** - * Create a GetObject request and modify it before passing it through auditor. - * @param modifyRequest Consumer Interface for changing the request before passing to the auditor - * @return the request + * Create a get request and pass it through the manager's beforeExecution() + * callback. + * + * @return a processed request. */ - protected GetObjectRequest get(Consumer modifyRequest) { - GetObjectRequest req = requestFactory.newGetObjectRequest("/"); - modifyRequest.accept(req); - return manager.beforeExecution(req); + protected SdkHttpRequest get(String range) { + GetObjectRequest.Builder getObjectRequestBuilder = + requestFactory.newGetObjectRequestBuilder("/"); + + SdkHttpRequest.Builder httpRequestBuilder = + SdkHttpRequest.builder().uri(URI.create("https://test")).method(SdkHttpMethod.GET); + + if (!range.isEmpty()) { + getObjectRequestBuilder.range(range); + List rangeHeader = new ArrayList<>(); + rangeHeader.add(range); + Map> headers = new HashMap<>(); + headers.put("Range", rangeHeader); + httpRequestBuilder.headers(headers); + } + + manager.requestCreated(getObjectRequestBuilder); + GetObjectRequest getObjectRequest = getObjectRequestBuilder.build(); + ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build().putAttribute( + AwsExecutionAttribute.OPERATION_NAME, "GetObject"); + InterceptorContext context = InterceptorContext.builder() + .request(getObjectRequest) + .httpRequest(httpRequestBuilder.build()) + .build(); + manager.beforeExecution(context, executionAttributes); + return manager.modifyHttpRequest(context, executionAttributes); } /** @@ -244,15 +290,31 @@ protected void assertMapNotContains(final Map params, final Stri * @param keys keys to be provided in the bulk delete request. * @return a processed request. */ - protected DeleteObjectsRequest headForBulkDelete(String... keys) { + protected SdkHttpRequest headForBulkDelete(String... keys) { if (keys == null || keys.length == 0) { return null; } - List keysToDelete = Arrays + + List keysToDelete = Arrays .stream(keys) - .map(DeleteObjectsRequest.KeyVersion::new) + .map(key -> ObjectIdentifier.builder().key(key).build()) .collect(Collectors.toList()); - return manager.beforeExecution(requestFactory.newBulkDeleteRequest(keysToDelete)); + + ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build(); + + SdkHttpRequest.Builder httpRequestBuilder = + SdkHttpRequest.builder().uri(URI.create("https://test")).method(SdkHttpMethod.POST); + + DeleteObjectsRequest deleteObjectsRequest = + requestFactory.newBulkDeleteRequestBuilder(keysToDelete).build(); + + InterceptorContext context = InterceptorContext.builder() + .request(deleteObjectsRequest) + .httpRequest(httpRequestBuilder.build()) + .build(); + + manager.beforeExecution(context, executionAttributes); + return manager.modifyHttpRequest(context, executionAttributes); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java index ad72d75081b27..1520e588e544e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java @@ -30,7 +30,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION; import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_SPAN_CREATION; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_SERVICE_CLASSNAME; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.LOGGING_AUDIT_SERVICE; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.NOOP_AUDIT_SERVICE; @@ -119,7 +119,7 @@ public static Configuration resetAuditOptions(Configuration conf) { S3ATestUtils.removeBaseAndBucketOverrides(conf, REFERRER_HEADER_ENABLED, REJECT_OUT_OF_SPAN_OPERATIONS, - AUDIT_REQUEST_HANDLERS, + AUDIT_EXECUTION_INTERCEPTORS, AUDIT_SERVICE_CLASSNAME, AUDIT_ENABLED); return conf; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java index 9e6d82ce6ac75..ea7a1a34da735 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java @@ -33,6 +33,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.enableLoggingAuditor; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.resetAuditOptions; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter; @@ -57,8 +58,9 @@ public Configuration createConfiguration() { Configuration conf = super.createConfiguration(); resetAuditOptions(conf); enableLoggingAuditor(conf); - conf.set(AUDIT_REQUEST_HANDLERS, - SimpleAWSRequestHandler.CLASS); + conf.set(AUDIT_EXECUTION_INTERCEPTORS, + SimpleAWSExecutionInterceptor.CLASS); + conf.set(AUDIT_REQUEST_HANDLERS, "not-valid-class"); return conf; } @@ -114,22 +116,26 @@ public void testInvokeOutOfSpanRejected() throws Throwable { } @Test - public void testRequestHandlerBinding() throws Throwable { - describe("Verify that extra request handlers can be added and that they" + public void testExecutionInterceptorBinding() throws Throwable { + describe("Verify that extra ExecutionInterceptor can be added and that they" + " will be invoked during request execution"); - final long baseCount = SimpleAWSRequestHandler.getInvocationCount(); + final long baseCount = SimpleAWSExecutionInterceptor.getInvocationCount(); final S3AFileSystem fs = getFileSystem(); final long exec0 = lookupCounterStatistic(iostats(), AUDIT_REQUEST_EXECUTION.getSymbol()); // API call to a known path, `getBucketLocation()` does not always result in an API call. fs.listStatus(path("/")); // which MUST have ended up calling the extension request handler - Assertions.assertThat(SimpleAWSRequestHandler.getInvocationCount()) + Assertions.assertThat(SimpleAWSExecutionInterceptor.getInvocationCount()) .describedAs("Invocation count of plugged in request handler") .isGreaterThan(baseCount); assertThatStatisticCounter(iostats(), AUDIT_REQUEST_EXECUTION.getSymbol()) .isGreaterThan(exec0); assertThatStatisticCounter(iostats(), AUDIT_FAILURE.getSymbol()) .isZero(); + Assertions.assertThat(SimpleAWSExecutionInterceptor.getStaticConf()) + .describedAs("configuratin of SimpleAWSExecutionInterceptor") + .isNotNull() + .isSameAs(fs.getConf()); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSRequestHandler.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java similarity index 56% rename from hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSRequestHandler.java rename to hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java index 6f5a0445a92f7..bf9b90bcdf31e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSRequestHandler.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSExecutionInterceptor.java @@ -20,28 +20,35 @@ import java.util.concurrent.atomic.AtomicLong; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.handlers.RequestHandler2; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; /** - * Simple AWS handler to verify dynamic loading of extra request - * handlers during auditing setup. + * Simple AWS interceptor to verify dynamic loading of extra + * execution interceptors during auditing setup. * The invocation counter tracks the count of calls to - * {@link #beforeExecution(AmazonWebServiceRequest)}. + * {@link #beforeExecution}. */ -public final class SimpleAWSRequestHandler extends RequestHandler2 { +public final class SimpleAWSExecutionInterceptor extends Configured + implements ExecutionInterceptor { public static final String CLASS - = "org.apache.hadoop.fs.s3a.audit.SimpleAWSRequestHandler"; + = "org.apache.hadoop.fs.s3a.audit.SimpleAWSExecutionInterceptor"; + + private static Configuration staticConf; /** Count of invocations. */ private static final AtomicLong INVOCATIONS = new AtomicLong(0); @Override - public AmazonWebServiceRequest beforeExecution( - final AmazonWebServiceRequest request) { + public void beforeExecution(Context.BeforeExecution context, + ExecutionAttributes executionAttributes) { INVOCATIONS.incrementAndGet(); - return request; + staticConf = getConf(); } /** @@ -51,4 +58,14 @@ public AmazonWebServiceRequest beforeExecution( public static long getInvocationCount() { return INVOCATIONS.get(); } + + /** + * get the static conf, which is set the config of the + * last executor invoked. + * @return the static configuration. + */ + + public static Configuration getStaticConf() { + return staticConf; + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditIntegration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditIntegration.java index 7cdab4c4b75e0..4f476604332b1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditIntegration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditIntegration.java @@ -22,9 +22,12 @@ import java.nio.file.AccessDeniedException; import java.util.List; -import com.amazonaws.DefaultRequest; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import org.assertj.core.api.Assertions; import org.junit.Test; @@ -39,13 +42,15 @@ import org.apache.hadoop.service.Service; import org.apache.hadoop.test.AbstractHadoopTestBase; + import static org.apache.hadoop.fs.s3a.S3AUtils.translateException; import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.attachSpanToRequest; import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.retrieveAttachedSpan; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.createIOStatisticsStoreForAuditing; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.noopAuditConfig; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_EXECUTION_INTERCEPTORS; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_SERVICE_CLASSNAME; +import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_EXECUTION_ATTRIBUTE; import static org.apache.hadoop.service.ServiceAssert.assertServiceStateStarted; import static org.apache.hadoop.service.ServiceAssert.assertServiceStateStopped; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -159,30 +164,50 @@ public void testAuditManagerLifecycle() throws Throwable { } @Test - public void testSingleRequestHandler() throws Throwable { + public void testSingleExecutionInterceptor() throws Throwable { AuditManagerS3A manager = AuditIntegration.createAndStartAuditManager( noopAuditConfig(), ioStatistics); - List handlers - = manager.createRequestHandlers(); - assertThat(handlers) + List interceptors + = manager.createExecutionInterceptors(); + assertThat(interceptors) .hasSize(1); - RequestHandler2 handler = handlers.get(0); + ExecutionInterceptor interceptor = interceptors.get(0); + RequestFactory requestFactory = RequestFactoryImpl.builder() .withBucket("bucket") .build(); + HeadObjectRequest.Builder requestBuilder = + requestFactory.newHeadObjectRequestBuilder("/"); + + assertThat(interceptor instanceof AWSAuditEventCallbacks).isTrue(); + ((AWSAuditEventCallbacks)interceptor).requestCreated(requestBuilder); + + HeadObjectRequest request = requestBuilder.build(); + SdkHttpRequest httpRequest = SdkHttpRequest.builder() + .protocol("https") + .host("test") + .method(SdkHttpMethod.HEAD) + .build(); + + ExecutionAttributes attributes = ExecutionAttributes.builder().build(); + InterceptorContext context = InterceptorContext.builder() + .request(request) + .httpRequest(httpRequest) + .build(); + // test the basic pre-request sequence while avoiding // the complexity of recreating the full sequence // (and probably getting it wrong) - GetObjectMetadataRequest r - = requestFactory.newGetObjectMetadataRequest("/"); - DefaultRequest dr = new DefaultRequest(r, "S3"); - assertThat(handler.beforeMarshalling(r)) - .isNotNull(); - assertThat(handler.beforeExecution(r)) - .isNotNull(); - handler.beforeRequest(dr); - + interceptor.beforeExecution(context, attributes); + interceptor.modifyRequest(context, attributes); + interceptor.beforeMarshalling(context, attributes); + interceptor.afterMarshalling(context, attributes); + interceptor.modifyHttpRequest(context, attributes); + interceptor.beforeTransmission(context, attributes); + AuditSpanS3A span = attributes.getAttribute(AUDIT_SPAN_EXECUTION_ATTRIBUTE); + assertThat(span).isNotNull(); + assertThat(span.isValidSpan()).isFalse(); } /** @@ -192,14 +217,14 @@ public void testSingleRequestHandler() throws Throwable { public void testRequestHandlerLoading() throws Throwable { Configuration conf = noopAuditConfig(); conf.setClassLoader(this.getClass().getClassLoader()); - conf.set(AUDIT_REQUEST_HANDLERS, - SimpleAWSRequestHandler.CLASS); + conf.set(AUDIT_EXECUTION_INTERCEPTORS, + SimpleAWSExecutionInterceptor.CLASS); AuditManagerS3A manager = AuditIntegration.createAndStartAuditManager( conf, ioStatistics); - assertThat(manager.createRequestHandlers()) + assertThat(manager.createExecutionInterceptors()) .hasSize(2) - .hasAtLeastOneElementOfType(SimpleAWSRequestHandler.class); + .hasAtLeastOneElementOfType(SimpleAWSExecutionInterceptor.class); } @Test @@ -216,8 +241,8 @@ public void testLoggingAuditorBinding() throws Throwable { @Test public void testNoopAuditManager() throws Throwable { AuditManagerS3A manager = AuditIntegration.stubAuditManager(); - assertThat(manager.createStateChangeListener()) - .describedAs("transfer state change listener") + assertThat(manager.createTransferListener()) + .describedAs("transfer listener") .isNotNull(); } @@ -226,11 +251,10 @@ public void testSpanAttachAndRetrieve() throws Throwable { AuditManagerS3A manager = AuditIntegration.stubAuditManager(); AuditSpanS3A span = manager.createSpan("op", null, null); - GetObjectMetadataRequest request = - new GetObjectMetadataRequest("bucket", "key"); - attachSpanToRequest(request, span); - AWSAuditEventCallbacks callbacks = retrieveAttachedSpan(request); - assertThat(callbacks).isSameAs(span); + ExecutionAttributes attributes = ExecutionAttributes.builder().build(); + attachSpanToRequest(attributes, span); + AuditSpanS3A retrievedSpan = retrieveAttachedSpan(attributes); + assertThat(retrievedSpan).isSameAs(span); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditSpanLifecycle.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditSpanLifecycle.java index 608667d9dfed8..e5e4afc434c8e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditSpanLifecycle.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditSpanLifecycle.java @@ -20,13 +20,14 @@ import java.util.List; -import com.amazonaws.handlers.RequestHandler2; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import org.junit.Before; import org.junit.Test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.store.audit.AuditSpan; + import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.noopAuditConfig; import static org.assertj.core.api.Assertions.assertThat; @@ -56,10 +57,10 @@ public void testStop() throws Throwable { } @Test - public void testCreateRequestHandlers() throws Throwable { - List handlers - = getManager().createRequestHandlers(); - assertThat(handlers).isNotEmpty(); + public void testCreateExecutionInterceptors() throws Throwable { + List interceptors + = getManager().createExecutionInterceptors(); + assertThat(interceptors).isNotEmpty(); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index b772e6dfc06fc..7f8dd043261b2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -20,12 +20,11 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.util.List; import java.util.Map; import java.util.regex.Matcher; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.http.SdkHttpRequest; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -38,6 +37,7 @@ import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; import org.apache.hadoop.security.UserGroupInformation; + import static org.apache.hadoop.fs.audit.AuditConstants.DELETE_KEYS_SIZE; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; @@ -97,13 +97,16 @@ protected Configuration createConfig() { public void testHttpReferrerPatchesTheRequest() throws Throwable { AuditSpan span = span(); long ts = span.getTimestamp(); - GetObjectMetadataRequest request = head(); - Map headers - = request.getCustomRequestHeaders(); + SdkHttpRequest request = head(); + Map> headers = request.headers(); assertThat(headers) .describedAs("Custom headers") .containsKey(HEADER_REFERRER); - String header = headers.get(HEADER_REFERRER); + List headerValues = headers.get(HEADER_REFERRER); + assertThat(headerValues) + .describedAs("Multiple referrer headers") + .hasSize(1); + String header = headerValues.get(0); LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); @@ -305,13 +308,16 @@ public void testStripWrappedQuotes() throws Throwable { @Test public void testGetObjectRange() throws Throwable { AuditSpan span = span(); - GetObjectRequest request = get(getObjectRequest -> getObjectRequest.setRange(100, 200)); - Map headers - = request.getCustomRequestHeaders(); + SdkHttpRequest request = get("bytes=100-200"); + Map> headers = request.headers(); assertThat(headers) - .describedAs("Custom headers") - .containsKey(HEADER_REFERRER); - String header = headers.get(HEADER_REFERRER); + .describedAs("Custom headers") + .containsKey(HEADER_REFERRER); + List headerValues = headers.get(HEADER_REFERRER); + assertThat(headerValues) + .describedAs("Multiple referrer headers") + .hasSize(1); + String header = headerValues.get(0); LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); @@ -324,13 +330,16 @@ public void testGetObjectRange() throws Throwable { @Test public void testGetObjectWithoutRange() throws Throwable { AuditSpan span = span(); - GetObjectRequest request = get(getObjectRequest -> {}); - Map headers - = request.getCustomRequestHeaders(); + SdkHttpRequest request = get(""); + Map> headers = request.headers(); assertThat(headers) .describedAs("Custom headers") .containsKey(HEADER_REFERRER); - String header = headers.get(HEADER_REFERRER); + List headerValues = headers.get(HEADER_REFERRER); + assertThat(headerValues) + .describedAs("Multiple referrer headers") + .hasSize(1); + String header = headerValues.get(0); LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); @@ -341,16 +350,20 @@ public void testGetObjectWithoutRange() throws Throwable { public void testHttpReferrerForBulkDelete() throws Throwable { AuditSpan span = span(); long ts = span.getTimestamp(); - DeleteObjectsRequest request = headForBulkDelete( + SdkHttpRequest request = headForBulkDelete( "key_01", "key_02", "key_03"); - Map headers - = request.getCustomRequestHeaders(); + Map> headers + = request.headers(); assertThat(headers) .describedAs("Custom headers") .containsKey(HEADER_REFERRER); - String header = headers.get(HEADER_REFERRER); + List headerValues = headers.get(HEADER_REFERRER); + assertThat(headerValues) + .describedAs("Multiple referrer headers") + .hasSize(1); + String header = headerValues.get(0); LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestLoggingAuditor.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestLoggingAuditor.java index 8d37b432acb56..0059e5b6c5392 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestLoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestLoggingAuditor.java @@ -18,9 +18,12 @@ package org.apache.hadoop.fs.s3a.audit; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CopyPartRequest; -import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest; +import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -30,6 +33,7 @@ import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor; import org.apache.hadoop.fs.store.audit.AuditSpan; + import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; import static org.assertj.core.api.Assertions.assertThat; @@ -131,8 +135,23 @@ public void testLoggingSpan() throws Throwable { */ @Test public void testCopyOutsideSpanAllowed() throws Throwable { - getManager().beforeExecution(new CopyPartRequest()); - getManager().beforeExecution(new CompleteMultipartUploadRequest()); + getManager().beforeExecution( + InterceptorContext.builder() + .request(UploadPartCopyRequest.builder().build()) + .build(), + ExecutionAttributes.builder().build()); + getManager().beforeExecution( + InterceptorContext.builder() + .request(GetBucketLocationRequest.builder().build()) + .build(), + ExecutionAttributes.builder().build()); + getManager().beforeExecution( + InterceptorContext.builder() + .request(CompleteMultipartUploadRequest.builder() + .multipartUpload(u -> {}) + .build()) + .build(), + ExecutionAttributes.builder().build()); } /** @@ -141,9 +160,9 @@ public void testCopyOutsideSpanAllowed() throws Throwable { */ @Test public void testTransferStateListenerOutsideSpan() throws Throwable { - TransferStateChangeListener listener - = getManager().createStateChangeListener(); - listener.transferStateChanged(null, null); + TransferListener listener + = getManager().createTransferListener(); + listener.transferInitiated(null); assertHeadUnaudited(); } @@ -158,15 +177,15 @@ public void testTransferStateListenerInSpan() throws Throwable { AuditSpan span = span(); // create the listener in the span - TransferStateChangeListener listener - = getManager().createStateChangeListener(); + TransferListener listener + = getManager().createTransferListener(); span.deactivate(); // head calls fail assertHeadUnaudited(); // until the state change switches this thread back to the span - listener.transferStateChanged(null, null); + listener.transferInitiated(null); // which can be probed assertActiveSpan(span); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 658c81cd8f2d1..5534bb77c0ddb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -26,14 +26,15 @@ import java.util.List; import java.util.stream.IntStream; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.services.sts.model.StsException; import com.fasterxml.jackson.core.JsonProcessingException; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -51,13 +52,14 @@ import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; -import static org.apache.hadoop.fs.s3a.S3AUtils.*; +import static org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.E_FORBIDDEN_AWS_PROVIDER; import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.*; import static org.apache.hadoop.fs.s3a.auth.RoleModel.*; import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*; @@ -140,7 +142,6 @@ private E expectFileSystemCreateFailure( } @Test - @SuppressWarnings("deprecation") public void testCreateCredentialProvider() throws IOException { describe("Create the credential provider"); @@ -148,13 +149,12 @@ public void testCreateCredentialProvider() throws IOException { try (AssumedRoleCredentialProvider provider = new AssumedRoleCredentialProvider(uri, conf)) { LOG.info("Provider is {}", provider); - AWSCredentials credentials = provider.getCredentials(); + AwsCredentials credentials = provider.resolveCredentials(); assertNotNull("Null credentials from " + provider, credentials); } } @Test - @SuppressWarnings("deprecation") public void testCreateCredentialProviderNoURI() throws IOException { describe("Create the credential provider"); @@ -162,7 +162,7 @@ public void testCreateCredentialProviderNoURI() throws IOException { try (AssumedRoleCredentialProvider provider = new AssumedRoleCredentialProvider(null, conf)) { LOG.info("Provider is {}", provider); - AWSCredentials credentials = provider.getCredentials(); + AwsCredentials credentials = provider.resolveCredentials(); assertNotNull("Null credentials from " + provider, credentials); } } @@ -172,7 +172,6 @@ public void testCreateCredentialProviderNoURI() throws IOException { * @return a configuration set to use to the role ARN. * @throws JsonProcessingException problems working with JSON policies. */ - @SuppressWarnings("deprecation") protected Configuration createValidRoleConf() throws JsonProcessingException { String roleARN = getAssumedRoleARN(); @@ -186,13 +185,17 @@ protected Configuration createValidRoleConf() throws JsonProcessingException { } @Test - @SuppressWarnings("deprecation") public void testAssumedInvalidRole() throws Throwable { Configuration conf = new Configuration(); conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE); - interceptClosing(AWSSecurityTokenServiceException.class, + interceptClosing(StsException.class, "", - () -> new AssumedRoleCredentialProvider(uri, conf)); + () -> { + AssumedRoleCredentialProvider p = + new AssumedRoleCredentialProvider(uri, conf); + p.resolveCredentials(); + return p; + }); } @Test @@ -204,7 +207,6 @@ public void testAssumeRoleFSBadARN() throws Exception { } @Test - @SuppressWarnings("deprecation") public void testAssumeRoleNoARN() throws Exception { describe("Attemnpt to create the FS with no ARN"); Configuration conf = createAssumedRoleConfig(); @@ -237,7 +239,6 @@ public void testAssumeRoleFSBadPolicy2() throws Exception { } @Test - @SuppressWarnings("deprecation") public void testAssumeRoleCannotAuthAssumedRole() throws Exception { describe("Assert that you can't use assumed roles to auth assumed roles"); @@ -246,12 +247,11 @@ public void testAssumeRoleCannotAuthAssumedRole() throws Exception { conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.NAME); expectFileSystemCreateFailure(conf, - IOException.class, + InstantiationIOException.class, E_FORBIDDEN_AWS_PROVIDER); } @Test - @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth() throws Exception { describe("Try to authenticate with a keypair with spaces"); @@ -267,7 +267,6 @@ public void testAssumeRoleBadInnerAuth() throws Exception { } @Test - @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth2() throws Exception { describe("Try to authenticate with an invalid keypair"); @@ -351,7 +350,6 @@ private Configuration createAssumedRoleConfig(String roleARN) { } @Test - @SuppressWarnings("deprecation") public void testAssumeRoleUndefined() throws Throwable { describe("Verify that you cannot instantiate the" + " AssumedRoleCredentialProvider without a role ARN"); @@ -363,12 +361,11 @@ public void testAssumeRoleUndefined() throws Throwable { } @Test - @SuppressWarnings("deprecation") public void testAssumedIllegalDuration() throws Throwable { describe("Expect the constructor to fail if the session is to short"); Configuration conf = new Configuration(); conf.set(ASSUMED_ROLE_SESSION_DURATION, "30s"); - interceptClosing(AWSSecurityTokenServiceException.class, "", + interceptClosing(StsException.class, "", () -> new AssumedRoleCredentialProvider(uri, conf)); } @@ -534,7 +531,6 @@ public Path methodPath() throws IOException { * don't break. */ @Test - @SuppressWarnings("deprecation") public void testAssumedRoleRetryHandler() throws Throwable { try(AssumedRoleCredentialProvider provider = new AssumedRoleCredentialProvider(getFileSystem().getUri(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java index a829d470e7a66..ad7d59a7319cf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java @@ -25,12 +25,12 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import com.amazonaws.SignableRequest; -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.arn.Arn; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.Signer; -import com.amazonaws.services.s3.internal.AWSS3V4Signer; +import software.amazon.awssdk.arns.Arn; +import software.amazon.awssdk.auth.signer.Aws4Signer; +import software.amazon.awssdk.auth.signer.AwsS3V4Signer; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.SdkHttpFullRequest; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; @@ -152,7 +152,7 @@ private Configuration createTestConfig(String identifier) { } private String determineRegion(String bucketName) throws IOException { - return getFileSystem().getBucketLocation(bucketName); + return getS3AInternals().getBucketLocation(bucketName); } @Private @@ -183,14 +183,15 @@ public CustomSigner() { * request because the signature calculated by the service doesn't match * what we sent. * @param request the request to sign. - * @param credentials credentials used to sign the request. + * @param executionAttributes request executionAttributes which contain the credentials. */ @Override - public void sign(SignableRequest request, AWSCredentials credentials) { + public SdkHttpFullRequest sign(SdkHttpFullRequest request, + ExecutionAttributes executionAttributes) { int c = INVOCATION_COUNT.incrementAndGet(); LOG.info("Signing request #{}", c); - String host = request.getEndpoint().getHost(); + String host = request.host(); String bucketName = parseBucketFromHost(host); try { lastStoreValue = CustomSignerInitializer @@ -199,19 +200,11 @@ public void sign(SignableRequest request, AWSCredentials credentials) { throw new RuntimeException("Failed to get current Ugi", e); } if (bucketName.equals("kms")) { - AWS4Signer realKMSSigner = new AWS4Signer(); - realKMSSigner.setServiceName("kms"); - if (lastStoreValue != null) { - realKMSSigner.setRegionName(lastStoreValue.conf.get(TEST_REGION_KEY)); - } - realKMSSigner.sign(request, credentials); + Aws4Signer realKMSSigner = Aws4Signer.create(); + return realKMSSigner.sign(request, executionAttributes); } else { - AWSS3V4Signer realSigner = new AWSS3V4Signer(); - realSigner.setServiceName("s3"); - if (lastStoreValue != null) { - realSigner.setRegionName(lastStoreValue.conf.get(TEST_REGION_KEY)); - } - realSigner.sign(request, credentials); + AwsS3V4Signer realSigner = AwsS3V4Signer.create(); + return realSigner.sign(request, executionAttributes); } } @@ -235,11 +228,11 @@ private String parseBucketFromHost(String host) { String accessPointName = bucketName.substring(0, bucketName.length() - (accountId.length() + 1)); Arn arn = Arn.builder() - .withAccountId(accountId) - .withPartition("aws") - .withRegion(hostBits[2]) - .withResource("accesspoint" + "/" + accessPointName) - .withService("s3").build(); + .accountId(accountId) + .partition("aws") + .region(hostBits[2]) + .resource("accesspoint" + "/" + accessPointName) + .service("s3").build(); bucketName = arn.toString(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java index 37c2dce4e1d72..852f03ea618fd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java @@ -146,19 +146,22 @@ public static void assertTouchForbidden(final FileSystem fs, final Path path) * @param roleARN ARN of role * @return the new configuration */ - @SuppressWarnings("deprecation") public static Configuration newAssumedRoleConfig( final Configuration srcConf, final String roleARN) { Configuration conf = new Configuration(srcConf); removeBaseAndBucketOverrides(conf, + S3A_BUCKET_PROBE, DELEGATION_TOKEN_BINDING, ASSUMED_ROLE_ARN, - AWS_CREDENTIALS_PROVIDER); + AWS_CREDENTIALS_PROVIDER, + ASSUMED_ROLE_SESSION_DURATION); conf.set(AWS_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.NAME); conf.set(ASSUMED_ROLE_ARN, roleARN); conf.set(ASSUMED_ROLE_SESSION_NAME, "test"); conf.set(ASSUMED_ROLE_SESSION_DURATION, "15m"); + // force in bucket resolution during startup + conf.setInt(S3A_BUCKET_PROBE, 1); disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java index c5ed9dbaac429..b9d547635f7f3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java @@ -21,7 +21,7 @@ import java.net.URI; import java.net.URISyntaxException; -import com.amazonaws.auth.AWSCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; import org.junit.Before; import org.junit.Test; @@ -94,13 +94,13 @@ public void testMarshalledCredentialProviderSession() throws Throwable { new Configuration(false), credentials, MarshalledCredentials.CredentialTypeRequired.SessionOnly); - AWSCredentials aws = provider.getCredentials(); + AwsCredentials aws = provider.resolveCredentials(); assertEquals(credentials.toString(), credentials.getAccessKey(), - aws.getAWSAccessKeyId()); + aws.accessKeyId()); assertEquals(credentials.toString(), credentials.getSecretKey(), - aws.getAWSSecretKey()); + aws.secretAccessKey()); // because the credentials are set to full only, creation will fail } @@ -119,7 +119,7 @@ public void testCredentialTypeMismatch() throws Throwable { MarshalledCredentials.CredentialTypeRequired.FullOnly); // because the credentials are set to full only, creation will fail intercept(NoAuthWithAWSException.class, "test", - () -> provider.getCredentials()); + () -> provider.resolveCredentials()); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java index ca87b5c1b34a6..595e2687276b1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java @@ -19,8 +19,6 @@ import java.io.Closeable; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; import java.util.HashMap; @@ -28,12 +26,10 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.DefaultRequest; -import com.amazonaws.SignableRequest; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.Signer; -import com.amazonaws.auth.SignerFactory; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Rule; @@ -284,7 +280,7 @@ private void attemptSignAndVerify(String identifier, String bucket, throws IOException, InterruptedException { ugi.doAs((PrivilegedExceptionAction) () -> { Signer signer = new SignerForInitializerTest(); - SignableRequest signableRequest = constructSignableRequest(bucket); + SdkHttpFullRequest signableRequest = constructSignableRequest(bucket); signer.sign(signableRequest, null); verifyStoreValueInSigner(expectNullStoreInfo, bucket, identifier); return null; @@ -336,8 +332,10 @@ public static class SignerForTest1 implements Signer { private static boolean initialized = false; @Override - public void sign(SignableRequest request, AWSCredentials credentials) { + public SdkHttpFullRequest sign(SdkHttpFullRequest sdkHttpFullRequest, + ExecutionAttributes executionAttributes) { initialized = true; + return sdkHttpFullRequest; } public static void reset() { @@ -354,8 +352,10 @@ public static class SignerForTest2 implements Signer { private static boolean initialized = false; @Override - public void sign(SignableRequest request, AWSCredentials credentials) { + public SdkHttpFullRequest sign(SdkHttpFullRequest sdkHttpFullRequest, + ExecutionAttributes executionAttributes) { initialized = true; + return sdkHttpFullRequest; } public static void reset() { @@ -472,11 +472,15 @@ public static class SignerForInitializerTest implements Signer { private static StoreValue retrievedStoreValue; @Override - public void sign(SignableRequest request, AWSCredentials credentials) { - String bucketName = request.getEndpoint().getHost(); + public SdkHttpFullRequest sign(SdkHttpFullRequest sdkHttpFullRequest, + ExecutionAttributes executionAttributes) { + String bucket = sdkHttpFullRequest.host().split("//")[1]; + // remove trailing slash + String bucketName = bucket.substring(0, bucket.length() - 1); try { retrievedStoreValue = SignerInitializerForTest .getStoreInfo(bucketName, UserGroupInformation.getCurrentUser()); + return sdkHttpFullRequest; } catch (IOException e) { throw new RuntimeException("Failed to get current ugi", e); } @@ -579,12 +583,9 @@ private String createTokenIdentifierString(String identifier, return identifier + "_" + bucketName + "_" + user; } - private SignableRequest constructSignableRequest(String bucketName) - throws URISyntaxException { - DefaultRequest signableRequest = new DefaultRequest( - AmazonWebServiceRequest.NOOP, "fakeservice"); - URI uri = new URI("s3://" + bucketName + "/"); - signableRequest.setEndpoint(uri); - return signableRequest; + private SdkHttpFullRequest constructSignableRequest(String bucketName) { + String host = "s3://" + bucketName + "/"; + return SdkHttpFullRequest.builder().host(host).protocol("https").method(SdkHttpMethod.GET) + .build(); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java index 3a7d78d68f7d5..4c7cd5c667999 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/CountInvocationsProvider.java @@ -20,8 +20,10 @@ import java.util.concurrent.atomic.AtomicLong; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import org.apache.hadoop.fs.s3a.CredentialInitializationException; @@ -29,21 +31,37 @@ * Simple AWS credential provider which counts how often it is invoked. */ public class CountInvocationsProvider - implements AWSCredentialsProvider { + implements AwsCredentialsProvider { + + private static final Logger LOG = LoggerFactory.getLogger( + CountInvocationsProvider.class); public static final String NAME = CountInvocationsProvider.class.getName(); public static final AtomicLong COUNTER = new AtomicLong(0); + private final AtomicLong instanceCounter = new AtomicLong(0); + @Override - public AWSCredentials getCredentials() { - COUNTER.incrementAndGet(); - throw new CredentialInitializationException("no credentials"); + public AwsCredentials resolveCredentials() { + final long global = COUNTER.incrementAndGet(); + final long local = instanceCounter.incrementAndGet(); + final String msg = + String.format("counter with global count %d and local count %d", global, local); + LOG.debug("resolving credentials from {}", msg); + throw new CredentialInitializationException("no credentials from " + msg); } - @Override - public void refresh() { + public long getInstanceCounter() { + return instanceCounter.get(); + } + @Override + public String toString() { + return "CountInvocationsProvider{" + + "instanceCounter=" + instanceCounter.get() + + "; global counter=" + COUNTER.get() + + '}'; } public static long getInvocationCount() { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java index 295125169a00c..ebad90336f7d0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java @@ -26,14 +26,16 @@ import java.net.URI; import java.nio.file.AccessDeniedException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -254,7 +256,6 @@ public void testGetDTfromFileSystem() throws Throwable { } @Test - @SuppressWarnings("deprecation") public void testAddTokensFromFileSystem() throws Throwable { describe("verify FileSystem.addDelegationTokens() collects tokens"); S3AFileSystem fs = getFileSystem(); @@ -276,7 +277,7 @@ public void testAddTokensFromFileSystem() throws Throwable { AWSCredentialProviderList providerList = requireNonNull( delegationTokens.getCredentialProviders(), "providers"); - providerList.getCredentials(); + providerList.resolveCredentials(); } @Test @@ -323,14 +324,15 @@ protected Credentials createDelegationTokens() throws IOException { * Create a FS with a delegated token, verify it works as a filesystem, * and that you can pick up the same DT from that FS too. */ - @SuppressWarnings("deprecation") @Test public void testDelegatedFileSystem() throws Throwable { describe("Delegation tokens can be passed to a new filesystem;" + " if role restricted, permissions are tightened."); S3AFileSystem fs = getFileSystem(); // force a probe of the remote FS to make sure its endpoint is valid - fs.getObjectMetadata(new Path("/")); + // TODO: Check what should happen here. Calling headObject() on the root path fails in V2, + // with the error that key cannot be empty. + // fs.getObjectMetadata(new Path("/")); readLandsatMetadata(fs); URI uri = fs.getUri(); @@ -577,29 +579,29 @@ public void testDelegationBindingMismatch2() throws Throwable { * @return result of the HEAD * @throws Exception failure */ - @SuppressWarnings("deprecation") - protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) + protected HeadBucketResponse readLandsatMetadata(final S3AFileSystem delegatedFS) throws Exception { AWSCredentialProviderList testingCreds - = delegatedFS.shareCredentials("testing"); + = delegatedFS.getS3AInternals().shareCredentials("testing"); URI landsat = new URI(DEFAULT_CSVTEST_FILE); DefaultS3ClientFactory factory = new DefaultS3ClientFactory(); - factory.setConf(new Configuration(delegatedFS.getConf())); + Configuration conf = delegatedFS.getConf(); + factory.setConf(conf); String host = landsat.getHost(); S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) .withPathUri(new URI("s3a://localhost/")) - .withEndpoint(DEFAULT_ENDPOINT) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) - .withUserAgentSuffix("ITestSessionDelegationInFilesystem"); - AmazonS3 s3 = factory.createS3Client(landsat, parameters); + .withUserAgentSuffix("ITestSessionDelegationInFilesystem") + .withRegion(Region.US_WEST_2); + S3Client s3 = factory.createS3Client(landsat, parameters); return Invoker.once("HEAD", host, - () -> s3.getObjectMetadata(host, landsat.getPath().substring(1))); + () -> s3.headBucket(b -> b.bucket(host))); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index fab7ffdbb76f8..efc775966859d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.net.URI; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSSessionCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import org.hamcrest.Matchers; import org.junit.Test; import org.slf4j.Logger; @@ -186,11 +186,15 @@ public void testCreateAndUseDT() throws Throwable { final MarshalledCredentials creds; try(S3ADelegationTokens dt2 = instantiateDTSupport(getConfiguration())) { dt2.start(); + // first creds are good + dt2.getCredentialProviders().resolveCredentials(); + + // reset to the original dt dt2.resetTokenBindingToDT(originalDT); - final AWSSessionCredentials awsSessionCreds + final AwsSessionCredentials awsSessionCreds = verifySessionCredentials( - dt2.getCredentialProviders().getCredentials()); + dt2.getCredentialProviders().resolveCredentials()); final MarshalledCredentials origCreds = fromAWSCredentials( awsSessionCreds); @@ -249,7 +253,7 @@ public void testCreateWithRenewer() throws Throwable { * @return the retrieved DT. This is only for error reporting. * @throws IOException failure. */ - @SuppressWarnings({"OptionalGetWithoutIsPresent", "deprecation"}) + @SuppressWarnings({"OptionalGetWithoutIsPresent"}) protected AbstractS3ATokenIdentifier verifyCredentialPropagation( final S3AFileSystem fs, final MarshalledCredentials session, @@ -278,7 +282,7 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation( LOG.info("Regenerated DT is {}", newDT); final MarshalledCredentials creds2 = fromAWSCredentials( verifySessionCredentials( - delegationTokens2.getCredentialProviders().getCredentials())); + delegationTokens2.getCredentialProviders().resolveCredentials())); assertEquals("Credentials", session, creds2); assertTrue("Origin in " + boundId, boundId.getOrigin() @@ -287,12 +291,12 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation( } } - private AWSSessionCredentials verifySessionCredentials( - final AWSCredentials creds) { - AWSSessionCredentials session = (AWSSessionCredentials) creds; - assertNotNull("access key", session.getAWSAccessKeyId()); - assertNotNull("secret key", session.getAWSSecretKey()); - assertNotNull("session token", session.getSessionToken()); + private AwsSessionCredentials verifySessionCredentials( + final AwsCredentials creds) { + AwsSessionCredentials session = (AwsSessionCredentials) creds; + assertNotNull("access key", session.accessKeyId()); + assertNotNull("secret key", session.secretAccessKey()); + assertNotNull("session token", session.sessionToken()); return session; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java index 88d9ebfcdfdc3..992643ff8ce98 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java @@ -37,7 +37,6 @@ import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.SESSION_TOKEN_KIND; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java index e517a41629cb9..258c34b5cb84f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java @@ -741,8 +741,8 @@ private void validateContent(Path dir, */ private void validateStorageClass(Path dir, String expectedStorageClass) throws Exception { Path expectedFile = getPart0000(dir); - S3AFileSystem fs = getFileSystem(); - String actualStorageClass = fs.getObjectMetadata(expectedFile).getStorageClass(); + String actualStorageClass = getS3AInternals().getObjectMetadata(expectedFile) + .storageClassAsString(); Assertions.assertThat(actualStorageClass) .describedAs("Storage class of object %s", expectedFile) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java index 6f2953762439a..e64822d8c8802 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java @@ -29,21 +29,20 @@ import java.util.UUID; import java.util.stream.Collectors; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; -import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; -import com.amazonaws.services.s3.model.MultipartUpload; -import com.amazonaws.services.s3.model.MultipartUploadListing; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest; +import software.amazon.awssdk.services.s3.model.ListMultipartUploadsResponse; +import software.amazon.awssdk.services.s3.model.MultipartUpload; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; import org.apache.hadoop.util.Lists; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.junit.AfterClass; @@ -81,6 +80,7 @@ import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.test.HadoopTestBase; + import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -339,7 +339,7 @@ public abstract static class JobCommitterTest // created in Before private StagingTestBase.ClientResults results = null; private StagingTestBase.ClientErrors errors = null; - private AmazonS3 mockClient = null; + private S3Client mockClient = null; @Before public void setupJob() throws Exception { @@ -448,7 +448,7 @@ protected File getTempDir() { public static class ClientResults implements Serializable { private static final long serialVersionUID = -3137637327090709905L; // For inspection of what the committer did - private final Map requests = + private final Map requests = Maps.newHashMap(); private final List uploads = Lists.newArrayList(); private final List parts = Lists.newArrayList(); @@ -461,7 +461,7 @@ public static class ClientResults implements Serializable { Maps.newHashMap(); private final List deletes = Lists.newArrayList(); - public Map getRequests() { + public Map getRequests() { return requests; } @@ -490,7 +490,7 @@ public List getDeletes() { } public List getDeletePaths() { - return deletes.stream().map(DeleteObjectRequest::getKey).collect( + return deletes.stream().map(DeleteObjectRequest::key).collect( Collectors.toList()); } @@ -619,197 +619,163 @@ private static T getArgumentAt(InvocationOnMock invocation, int index, * @param errors when (if any) to fail * @return the mock client to patch in to a committer/FS instance */ - public static AmazonS3 newMockS3Client(final ClientResults results, + public static S3Client newMockS3Client(final ClientResults results, final ClientErrors errors) { - AmazonS3Client mockClient = mock(AmazonS3Client.class); + S3Client mockClientV2 = mock(S3Client.class); final Object lock = new Object(); // initiateMultipartUpload - when(mockClient - .initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))) + when(mockClientV2 + .createMultipartUpload(any(CreateMultipartUploadRequest.class))) .thenAnswer(invocation -> { - LOG.debug("initiateMultipartUpload for {}", mockClient); + LOG.debug("initiateMultipartUpload for {}", mockClientV2); synchronized (lock) { if (results.requests.size() == errors.failOnInit) { if (errors.recover) { errors.failOnInit(-1); } - throw new AmazonClientException( - "Mock Fail on init " + results.requests.size()); + throw AwsServiceException.builder() + .message("Mock Fail on init " + results.requests.size()) + .build(); } String uploadId = UUID.randomUUID().toString(); - InitiateMultipartUploadRequest req = getArgumentAt(invocation, - 0, InitiateMultipartUploadRequest.class); + CreateMultipartUploadRequest req = getArgumentAt(invocation, + 0, CreateMultipartUploadRequest.class); results.requests.put(uploadId, req); - results.activeUploads.put(uploadId, req.getKey()); + results.activeUploads.put(uploadId, req.key()); results.uploads.add(uploadId); - return newResult(results.requests.get(uploadId), uploadId); + return CreateMultipartUploadResponse.builder() + .uploadId(uploadId) + .build(); } }); // uploadPart - when(mockClient.uploadPart(any(UploadPartRequest.class))) + when(mockClientV2.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))) .thenAnswer(invocation -> { - LOG.debug("uploadPart for {}", mockClient); + LOG.debug("uploadPart for {}", mockClientV2); synchronized (lock) { if (results.parts.size() == errors.failOnUpload) { if (errors.recover) { errors.failOnUpload(-1); } LOG.info("Triggering upload failure"); - throw new AmazonClientException( - "Mock Fail on upload " + results.parts.size()); + throw AwsServiceException.builder() + .message("Mock Fail on upload " + results.parts.size()) + .build(); } UploadPartRequest req = getArgumentAt(invocation, 0, UploadPartRequest.class); results.parts.add(req); String etag = UUID.randomUUID().toString(); - List etags = results.tagsByUpload.get(req.getUploadId()); + List etags = results.tagsByUpload.get(req.uploadId()); if (etags == null) { etags = Lists.newArrayList(); - results.tagsByUpload.put(req.getUploadId(), etags); + results.tagsByUpload.put(req.uploadId(), etags); } etags.add(etag); - return newResult(req, etag); + return UploadPartResponse.builder().eTag(etag).build(); } }); // completeMultipartUpload - when(mockClient + when(mockClientV2 .completeMultipartUpload(any(CompleteMultipartUploadRequest.class))) .thenAnswer(invocation -> { - LOG.debug("completeMultipartUpload for {}", mockClient); + LOG.debug("completeMultipartUpload for {}", mockClientV2); synchronized (lock) { if (results.commits.size() == errors.failOnCommit) { if (errors.recover) { errors.failOnCommit(-1); } - throw new AmazonClientException( - "Mock Fail on commit " + results.commits.size()); + throw AwsServiceException.builder() + .message("Mock Fail on commit " + results.commits.size()) + .build(); } CompleteMultipartUploadRequest req = getArgumentAt(invocation, 0, CompleteMultipartUploadRequest.class); - String uploadId = req.getUploadId(); + String uploadId = req.uploadId(); removeUpload(results, uploadId); results.commits.add(req); - - return newResult(req); + return CompleteMultipartUploadResponse.builder().build(); } }); // abortMultipartUpload mocking doAnswer(invocation -> { - LOG.debug("abortMultipartUpload for {}", mockClient); + LOG.debug("abortMultipartUpload for {}", mockClientV2); synchronized (lock) { if (results.aborts.size() == errors.failOnAbort) { if (errors.recover) { errors.failOnAbort(-1); } - throw new AmazonClientException( - "Mock Fail on abort " + results.aborts.size()); + throw AwsServiceException.builder() + .message("Mock Fail on abort " + results.aborts.size()) + .build(); } AbortMultipartUploadRequest req = getArgumentAt(invocation, 0, AbortMultipartUploadRequest.class); - String id = req.getUploadId(); + String id = req.uploadId(); removeUpload(results, id); results.aborts.add(req); return null; } }) - .when(mockClient) + .when(mockClientV2) .abortMultipartUpload(any(AbortMultipartUploadRequest.class)); // deleteObject mocking doAnswer(invocation -> { - LOG.debug("deleteObject for {}", mockClient); + LOG.debug("deleteObject for {}", mockClientV2); synchronized (lock) { results.deletes.add(getArgumentAt(invocation, 0, DeleteObjectRequest.class)); return null; } }) - .when(mockClient) + .when(mockClientV2) .deleteObject(any(DeleteObjectRequest.class)); - // deleteObject mocking - doAnswer(invocation -> { - LOG.debug("deleteObject for {}", mockClient); - synchronized (lock) { - results.deletes.add(new DeleteObjectRequest( - getArgumentAt(invocation, 0, String.class), - getArgumentAt(invocation, 1, String.class) - )); - return null; - } - }).when(mockClient) - .deleteObject(any(String.class), any(String.class)); - // to String returns the debug information - when(mockClient.toString()).thenAnswer( + when(mockClientV2.toString()).thenAnswer( invocation -> "Mock3AClient " + results + " " + errors); - when(mockClient + when(mockClientV2 .listMultipartUploads(any(ListMultipartUploadsRequest.class))) .thenAnswer(invocation -> { synchronized (lock) { - MultipartUploadListing l = new MultipartUploadListing(); - l.setMultipartUploads( - results.activeUploads.entrySet().stream() - .map(e -> newMPU(e.getKey(), e.getValue())) - .collect(Collectors.toList())); - return l; + return ListMultipartUploadsResponse.builder() + .uploads(results.activeUploads.entrySet().stream() + .map(e -> MultipartUpload.builder() + .uploadId(e.getKey()) + .key(e.getValue()) + .build()) + .collect(Collectors.toList())) + .build(); } }); - return mockClient; + return mockClientV2; } /** * Remove an upload from the upload map. * @param results result set * @param uploadId The upload ID to remove - * @throws AmazonS3Exception with error code 404 if the id is unknown. + * @throws AwsServiceException with error code 404 if the id is unknown. */ protected static void removeUpload(final ClientResults results, final String uploadId) { String removed = results.activeUploads.remove(uploadId); if (removed == null) { // upload doesn't exist - AmazonS3Exception ex = new AmazonS3Exception( - "not found " + uploadId); - ex.setStatusCode(404); - throw ex; + throw AwsServiceException.builder() + .message("not found " + uploadId) + .statusCode(404) + .build(); } } - private static CompleteMultipartUploadResult newResult( - CompleteMultipartUploadRequest req) { - return new CompleteMultipartUploadResult(); - } - - - private static MultipartUpload newMPU(String id, String path) { - MultipartUpload up = new MultipartUpload(); - up.setUploadId(id); - up.setKey(path); - return up; - } - - private static UploadPartResult newResult(UploadPartRequest request, - String etag) { - UploadPartResult result = new UploadPartResult(); - result.setPartNumber(request.getPartNumber()); - result.setETag(etag); - return result; - } - - private static InitiateMultipartUploadResult newResult( - InitiateMultipartUploadRequest request, String uploadId) { - InitiateMultipartUploadResult result = new InitiateMultipartUploadResult(); - result.setUploadId(uploadId); - return result; - } - /** * create files in the attempt path that should be found by * {@code getTaskOutput}. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java index 4d24c07dacfe2..f96cf97ebd7f4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java @@ -27,7 +27,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import com.amazonaws.services.s3.model.PartETag; +import software.amazon.awssdk.services.s3.model.CompletedPart; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.assertj.core.api.Assertions; @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -159,8 +160,8 @@ private void createTasks() throws IOException { // step1: a list of tags. // this is the md5sum of hadoop 3.2.1.tar String tag = "9062dcf18ffaee254821303bbd11c72b"; - List etags = IntStream.rangeClosed(1, BLOCKS_PER_TASK + 1) - .mapToObj(i -> new PartETag(i, tag)) + List etags = IntStream.rangeClosed(1, BLOCKS_PER_TASK + 1) + .mapToObj(i -> CompletedPart.builder().partNumber(i).eTag(tag).build()) .collect(Collectors.toList()); SinglePendingCommit base = new SinglePendingCommit(); base.setBucket(BUCKET); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java index 11edf0d216376..71ed0b6891a58 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java @@ -31,9 +31,9 @@ import java.util.UUID; import java.util.stream.Collectors; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; @@ -51,7 +51,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.AWSClientIOException; import org.apache.hadoop.fs.s3a.MockS3AFileSystem; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter; @@ -70,6 +69,7 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; + import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.*; @@ -112,7 +112,7 @@ public class TestStagingCommitter extends StagingTestBase.MiniDFSTest { // created in Before private StagingTestBase.ClientResults results = null; private StagingTestBase.ClientErrors errors = null; - private AmazonS3 mockClient = null; + private S3Client mockClient = null; private File tmpDir; /** @@ -473,7 +473,7 @@ public void testTaskInitializeFailure() throws Exception { writeOutputFile(tac.getTaskAttemptID(), attemptPath, UUID.randomUUID().toString(), 10); - intercept(AWSClientIOException.class, + intercept(IOException.class, "Fail on init 1", "Should fail during init", () -> committer.commitTask(tac)); @@ -501,7 +501,7 @@ public void testTaskSingleFileUploadFailure() throws Exception { writeOutputFile(tac.getTaskAttemptID(), attemptPath, UUID.randomUUID().toString(), 10); - intercept((Class) AWSClientIOException.class, + intercept(IOException.class, "Fail on upload 2", "Should fail during upload", () -> { @@ -513,7 +513,7 @@ public void testTaskSingleFileUploadFailure() throws Exception { 1, results.getUploads().size()); assertEquals("Should abort the upload", results.getUploads().get(0), - results.getAborts().get(0).getUploadId()); + results.getAborts().get(0).uploadId()); assertPathDoesNotExist(fs, "Should remove the attempt path", attemptPath); } @@ -532,7 +532,7 @@ public void testTaskMultiFileUploadFailure() throws Exception { writeOutputFile(tac.getTaskAttemptID(), attemptPath, UUID.randomUUID().toString(), 10); - intercept((Class) AWSClientIOException.class, + intercept(IOException.class, "Fail on upload 5", "Should fail during upload", () -> { @@ -564,7 +564,7 @@ public void testTaskUploadAndAbortFailure() throws Exception { writeOutputFile(tac.getTaskAttemptID(), attemptPath, UUID.randomUUID().toString(), 10); - intercept((Class) AWSClientIOException.class, + intercept(IOException.class, "Fail on upload 5", "Should suppress abort failure, propagate upload failure", ()-> { @@ -637,7 +637,7 @@ public void testJobCommitFailure() throws Exception { errors.failOnCommit(5); setMockLogLevel(MockS3AFileSystem.LOG_NAME); - intercept(AWSClientIOException.class, + intercept(IOException.class, "Fail on commit 5", "Should propagate the commit failure", () -> { @@ -645,17 +645,16 @@ public void testJobCommitFailure() throws Exception { return jobCommitter.toString(); }); - Set commits = results.getCommits() .stream() .map(commit -> - "s3a://" + commit.getBucketName() + "/" + commit.getKey()) + "s3a://" + commit.bucket() + "/" + commit.key()) .collect(Collectors.toSet()); Set deletes = results.getDeletes() .stream() .map(delete -> - "s3a://" + delete.getBucketName() + "/" + delete.getKey()) + "s3a://" + delete.bucket() + "/" + delete.key()) .collect(Collectors.toSet()); Assertions.assertThat(commits) @@ -729,14 +728,14 @@ private Set runTasks(JobContext jobContext, private static Set getAbortedIds( List aborts) { return aborts.stream() - .map(AbortMultipartUploadRequest::getUploadId) + .map(AbortMultipartUploadRequest::uploadId) .collect(Collectors.toSet()); } private static Set getCommittedIds( List commits) { return commits.stream() - .map(CompleteMultipartUploadRequest::getUploadId) + .map(CompleteMultipartUploadRequest::uploadId) .collect(Collectors.toSet()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java index 4e82b94314d34..6ace7462e78a6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java @@ -24,7 +24,7 @@ import java.util.Set; import java.util.UUID; -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.PathExistsException; import org.apache.hadoop.mapreduce.JobContext; + import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.mockito.Mockito.*; @@ -146,10 +147,10 @@ public void testAppend() throws Exception { protected void verifyFilesCreated( final PartitionedStagingCommitter committer) { Set files = Sets.newHashSet(); - for (InitiateMultipartUploadRequest request : + for (CreateMultipartUploadRequest request : getMockResults().getRequests().values()) { - assertEquals(BUCKET, request.getBucketName()); - files.add(request.getKey()); + assertEquals(BUCKET, request.bucket()); + files.add(request.key()); } Assertions.assertThat(files) .describedAs("Should have the right number of uploads") diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java index d28f4279f1ba0..fbad671e1fa66 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java @@ -16,7 +16,6 @@ import java.io.IOException; import java.net.URI; -import com.amazonaws.services.s3.model.CryptoStorageMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +76,7 @@ protected void verifyReadBytes(FileSystem.Statistics stats) { *
    * NOTE: if Client side encryption is enabled, expected bytes written * should increase by 16(padding of data) + bytes for the key ID set + 94(KMS - * key generation) in case of storage type{@link CryptoStorageMode} as + * key generation) in case of storage type CryptoStorageMode as * ObjectMetadata(Default). If Crypto Storage mode is instruction file then * add additional bytes as that file is stored separately and would account * for bytes written. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java index 24f5ddf6d89b4..967fb2df73be3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java @@ -28,7 +28,6 @@ import java.util.TreeSet; import java.util.stream.Collectors; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; import org.assertj.core.api.Assertions; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java index 2610f54b44e9e..73db942973211 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java @@ -23,13 +23,14 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import com.amazonaws.AmazonClientException; +import software.amazon.awssdk.core.exception.SdkException; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; @@ -207,7 +208,7 @@ private BlockingFakeDirMarkerFS() { @Override protected void maybeCreateFakeParentDirectory(final Path path) - throws IOException, AmazonClientException { + throws IOException, SdkException { LOG.info("waking anything blocked on the signal semaphore"); // notify anything waiting signalCreatingFakeParentDirectory.release(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java index 82592b1d01950..81bd8a5efe2e4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java @@ -26,7 +26,9 @@ import java.util.List; import java.util.Map; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.assertj.core.api.Assertions; import org.assertj.core.util.Lists; import org.junit.Before; @@ -206,20 +208,20 @@ public void testMetadataCopySkipsMagicAttribute() throws Throwable { final String owner = "x-header-owner"; final String root = "root"; CONTEXT_ACCESSORS.userHeaders.put(owner, root); - final ObjectMetadata source = CONTEXT_ACCESSORS + final HeadObjectResponse source = CONTEXT_ACCESSORS .getObjectMetadata(MAGIC_KEY); - final Map sourceUserMD = source.getUserMetadata(); + final Map sourceUserMD = source.metadata(); Assertions.assertThat(sourceUserMD.get(owner)) .describedAs("owner header in copied MD") .isEqualTo(root); - ObjectMetadata dest = new ObjectMetadata(); - headerProcessing.cloneObjectMetadata(source, dest); + Map destUserMetadata = new HashMap<>(); + headerProcessing.cloneObjectMetadata(source, destUserMetadata, CopyObjectRequest.builder()); - Assertions.assertThat(dest.getUserMetadata().get(X_HEADER_MAGIC_MARKER)) + Assertions.assertThat(destUserMetadata.get(X_HEADER_MAGIC_MARKER)) .describedAs("Magic marker header in copied MD") .isNull(); - Assertions.assertThat(dest.getUserMetadata().get(owner)) + Assertions.assertThat(destUserMetadata.get(owner)) .describedAs("owner header in copied MD") .isEqualTo(root); } @@ -307,20 +309,24 @@ public RequestFactory getRequestFactory() { } @Override - public ObjectMetadata getObjectMetadata(final String key) + public HeadObjectResponse getObjectMetadata(final String key) throws IOException { if (MAGIC_KEY.equals(key)) { - ObjectMetadata omd = new ObjectMetadata(); - omd.setUserMetadata(userHeaders); - omd.setContentLength(len); - omd.setLastModified(date); - return omd; + return HeadObjectResponse.builder() + .metadata(userHeaders) + .contentLength(len) + .lastModified(date.toInstant()).build(); } else { throw new FileNotFoundException(key); } } + @Override + public HeadBucketResponse getBucketMetadata() throws IOException { + return HeadBucketResponse.builder().build(); + } + public void setHeader(String key, String val) { userHeaders.put(key, val); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestNetworkBinding.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestNetworkBinding.java index 7f51d2b45362c..919a89b8c1dd0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestNetworkBinding.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestNetworkBinding.java @@ -18,14 +18,10 @@ package org.apache.hadoop.fs.s3a.impl; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.client.builder.AwsClientBuilder; -import org.junit.Ignore; import org.junit.Test; import org.apache.hadoop.test.AbstractHadoopTestBase; -import static org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createEndpointConfiguration; import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion; import static org.assertj.core.api.Assertions.assertThat; @@ -63,43 +59,4 @@ private static void assertRegionFixup(String region, String expected) { .describedAs("Fixup of %s", region) .isEqualTo(expected); } - - @Test - public void testNull() throws Throwable { - expectEndpoint("", true, "unused"); - } - - @Test - @Ignore("disabled until endpoint logic works for S3 client builder API") - public void testUSEastEndpoint() throws Throwable { - expectEndpoint(US_EAST_1, false, US_EAST_1); - } - - @Test - @Ignore("disabled until endpoint logic works for S3 client builder API") - public void testUSWestEndpoint() throws Throwable { - expectEndpoint(US_WEST_2, false, US_WEST_2); - } - - public void expectEndpoint(final String src, - final boolean expectNull, - final String expectRegion) { - AwsClientBuilder.EndpointConfiguration epr = - createEndpointConfiguration(src, new ClientConfiguration(), src); - String eprStr = epr == null - ? "(empty)" - : ("(" + epr.getServiceEndpoint() + " " + epr.getSigningRegion()); - if (expectNull) { - assertThat(epr) - .describedAs("Endpoint configuration of %s =", - src, eprStr) - .isNull(); - } else { - assertThat(epr) - .describedAs("Endpoint configuration of %s =", - src, eprStr) - .hasFieldOrPropertyWithValue("serviceEndpoint", src) - .hasFieldOrPropertyWithValue("signingRegion", expectRegion); - } - } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java index 7c85142d4376d..1fb576a55514c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java @@ -18,21 +18,20 @@ package org.apache.hadoop.fs.s3a.impl; -import java.io.ByteArrayInputStream; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.concurrent.atomic.AtomicLong; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.services.s3.model.CannedAccessControlList; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.awscore.AwsRequest; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.api.RequestFactory; import org.apache.hadoop.fs.s3a.audit.AWSRequestAnalyzer; @@ -57,7 +56,7 @@ public class TestRequestFactory extends AbstractHadoopTestBase { private final AWSRequestAnalyzer analyzer = new AWSRequestAnalyzer(); /** - * Count of requests analyzed via the {@link #a(AmazonWebServiceRequest)} + * Count of requests analyzed via the {@link #a(AwsRequest.Builder)} * call. */ private int requestsAnalyzed; @@ -81,27 +80,31 @@ public void testRequestFactoryWithEncryption() throws Throwable { */ @Test public void testRequestFactoryWithCannedACL() throws Throwable { - CannedAccessControlList acl = CannedAccessControlList.BucketOwnerFullControl; + String acl = "bucket-owner-full-control"; RequestFactory factory = RequestFactoryImpl.builder() .withBucket("bucket") .withCannedACL(acl) .build(); String path = "path"; String path2 = "path2"; - ObjectMetadata md = factory.newObjectMetadata(128); - Assertions.assertThat( - factory.newPutObjectRequest(path, md, - null, new ByteArrayInputStream(new byte[0])) - .getCannedAcl()) + HeadObjectResponse md = HeadObjectResponse.builder().contentLength(128L).build(); + + Assertions.assertThat(factory.newPutObjectRequestBuilder(path, null, 128, false) + .build() + .acl() + .toString()) .describedAs("ACL of PUT") .isEqualTo(acl); - Assertions.assertThat(factory.newCopyObjectRequest(path, path2, md) - .getCannedAccessControlList()) + Assertions.assertThat(factory.newCopyObjectRequestBuilder(path, path2, md) + .build() + .acl() + .toString()) .describedAs("ACL of COPY") .isEqualTo(acl); - Assertions.assertThat(factory.newMultipartUploadRequest(path, - null) - .getCannedACL()) + Assertions.assertThat(factory.newMultipartUploadRequestBuilder(path, null) + .build() + .acl() + .toString()) .describedAs("ACL of MPU") .isEqualTo(acl); } @@ -132,21 +135,18 @@ private final class CountRequests private final AtomicLong counter = new AtomicLong(); @Override - public T prepareRequest(final T t) { + public void prepareRequest(final SdkRequest.Builder t) { counter.addAndGet(1); - return t; } } /** * Analyze the request, log the output, return the info. - * @param request request. - * @param type of request. + * @param builder request builder. * @return value */ - private AWSRequestAnalyzer.RequestInfo - a(T request) { - AWSRequestAnalyzer.RequestInfo info = analyzer.analyze(request); + private AWSRequestAnalyzer.RequestInfo a(AwsRequest.Builder builder) { + AWSRequestAnalyzer.RequestInfo info = analyzer.analyze(builder.build()); LOG.info("{}", info); requestsAnalyzed++; return info; @@ -160,27 +160,25 @@ private void createFactoryObjects(RequestFactory factory) throws IOException { String path = "path"; String path2 = "path2"; String id = "1"; - ObjectMetadata md = factory.newObjectMetadata(128); - a(factory.newAbortMultipartUploadRequest(path, id)); - a(factory.newCompleteMultipartUploadRequest(path, id, + a(factory.newAbortMultipartUploadRequestBuilder(path, id)); + a(factory.newCompleteMultipartUploadRequestBuilder(path, id, new ArrayList<>())); - a(factory.newCopyObjectRequest(path, path2, md)); - a(factory.newDeleteObjectRequest(path)); - a(factory.newBulkDeleteRequest(new ArrayList<>())); + a(factory.newCopyObjectRequestBuilder(path, path2, + HeadObjectResponse.builder().build())); + a(factory.newDeleteObjectRequestBuilder(path)); + a(factory.newBulkDeleteRequestBuilder(new ArrayList<>())); a(factory.newDirectoryMarkerRequest(path)); - a(factory.newGetObjectRequest(path)); - a(factory.newGetObjectMetadataRequest(path)); - a(factory.newListMultipartUploadsRequest(path)); - a(factory.newListObjectsV1Request(path, "/", 1)); - a(factory.newListNextBatchOfObjectsRequest(new ObjectListing())); - a(factory.newListObjectsV2Request(path, "/", 1)); - a(factory.newMultipartUploadRequest(path, null)); - File srcfile = new File("/tmp/a"); - a(factory.newPutObjectRequest(path, - factory.newObjectMetadata(-1), null, srcfile)); - ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]); - a(factory.newPutObjectRequest(path, md, null, stream)); - a(factory.newSelectRequest(path)); + a(factory.newGetObjectRequestBuilder(path)); + a(factory.newHeadObjectRequestBuilder(path)); + a(factory.newListMultipartUploadsRequestBuilder(path)); + a(factory.newListObjectsV1RequestBuilder(path, "/", 1)); + a(factory.newListObjectsV2RequestBuilder(path, "/", 1)); + a(factory.newMultipartUploadRequestBuilder(path, null)); + a(factory.newPutObjectRequestBuilder(path, + PutObjectOptions.keepingDirs(), -1, true)); + a(factory.newPutObjectRequestBuilder(path, + PutObjectOptions.deletingDirs(), 1024, false)); + a(factory.newSelectRequestBuilder(path)); } /** @@ -193,25 +191,18 @@ public void testMultipartUploadRequest() throws Throwable { RequestFactory factory = RequestFactoryImpl.builder() .withBucket("bucket") .withRequestPreparer(countRequests) + .withMultipartPartCountLimit(2) .build(); String path = "path"; - String path2 = "path2"; String id = "1"; - File srcfile = File.createTempFile("file", ""); - try { - ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]); - - a(factory.newUploadPartRequest(path, id, 1, 0, stream, null, 0)); - a(factory.newUploadPartRequest(path, id, 2, 128_000_000, - null, srcfile, 0)); - // offset is past the EOF - intercept(IllegalArgumentException.class, () -> - factory.newUploadPartRequest(path, id, 3, 128_000_000, - null, srcfile, 128)); - } finally { - srcfile.delete(); - } + + a(factory.newUploadPartRequestBuilder(path, id, 1, 0)); + a(factory.newUploadPartRequestBuilder(path, id, 2, 128_000_000)); + // partNumber is past the limit + intercept(PathIOException.class, () -> + factory.newUploadPartRequestBuilder(path, id, 3, 128_000_000)); + assertThat(countRequests.counter.get()) .describedAs("request preparation count") .isEqualTo(requestsAnalyzed); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java index 33a44a9ad78f7..7042737b31085 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java @@ -19,13 +19,15 @@ package org.apache.hadoop.fs.s3a.impl; import java.io.IOException; +import java.io.InputStream; -import com.amazonaws.internal.SdkFilterInputStream; +import software.amazon.awssdk.http.Abortable; import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.hadoop.test.HadoopTestBase; + import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DRAIN_BUFFER_SIZE; import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_INPUT_STREAM_STATISTICS; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -127,7 +129,6 @@ public void testStreamUnderflow() throws Throwable { public void testReadFailure() throws Throwable { int threshold = 50; SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/", - null, new FakeSDKInputStream(BYTES, threshold), false, BYTES, @@ -145,7 +146,6 @@ public void testReadFailure() throws Throwable { public void testReadFailureDoesNotSurfaceInAbort() throws Throwable { int threshold = 50; SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/", - null, new FakeSDKInputStream(BYTES, threshold), true, BYTES, @@ -183,7 +183,6 @@ private SDKStreamDrainer drainer(int remaining, boolean shouldAbort, FakeSDKInputStream in) throws Throwable { SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/", - null, in, shouldAbort, remaining, @@ -246,7 +245,8 @@ private static SDKStreamDrainer assertBytesRead(final SDKStreamDrainer drainer, * Fake stream; generates data dynamically. * Only overrides the methods used in stream draining. */ - private static final class FakeSDKInputStream extends SdkFilterInputStream { + private static final class FakeSDKInputStream extends InputStream + implements Abortable { private final int capacity; @@ -264,7 +264,6 @@ private static final class FakeSDKInputStream extends SdkFilterInputStream { * @param readToRaiseIOE position to raise an IOE, or -1 */ private FakeSDKInputStream(final int capacity, final int readToRaiseIOE) { - super(null); this.capacity = capacity; this.readToRaiseIOE = readToRaiseIOE; } @@ -282,11 +281,6 @@ public void abort() { aborted = true; } - @Override - protected boolean isAborted() { - return aborted; - } - @Override public int read() throws IOException { if (bytesRead >= capacity) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java index ccb0c0e79ec7e..48791f557c002 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java @@ -27,9 +27,10 @@ import java.util.concurrent.Callable; import java.util.stream.Collectors; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ObjectMetadata; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -48,6 +49,7 @@ import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.store.audit.AuditSpan; + import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; @@ -156,7 +158,7 @@ public static Collection params() { /** * S3 Client of the FS. */ - private AmazonS3 s3client; + private S3Client s3client; /** * Path to a file under the marker. @@ -212,7 +214,7 @@ protected Configuration createConfiguration() { public void setup() throws Exception { super.setup(); S3AFileSystem fs = getFileSystem(); - s3client = fs.getAmazonS3ClientForTesting("markers"); + s3client = getS3AInternals().getAmazonS3Client("markers"); bucket = fs.getBucket(); Path base = new Path(methodPath(), "base"); @@ -604,7 +606,8 @@ private void assertTestObjectsExist() throws Exception { */ private void put(final String key, final String content) throws Exception { exec("PUT " + key, () -> - s3client.putObject(bucket, key, content)); + s3client.putObject(b -> b.bucket(bucket).key(key), + RequestBody.fromString(content))); } /** * Delete an object. @@ -613,7 +616,7 @@ private void put(final String key, final String content) throws Exception { */ private void deleteObject(final String key) throws Exception { exec("DELETE " + key, () -> { - s3client.deleteObject(bucket, key); + s3client.deleteObject(b -> b.bucket(bucket).key(key)); return "deleted " + key; }); } @@ -624,10 +627,10 @@ private void deleteObject(final String key) throws Exception { * @return a description of the object. */ private String head(final String key) throws Exception { - ObjectMetadata md = exec("HEAD " + key, () -> - s3client.getObjectMetadata(bucket, key)); + HeadObjectResponse response = exec("HEAD " + key, () -> + s3client.headObject(b -> b.bucket(bucket).key(key))); return String.format("Object %s of length %d", - key, md.getInstanceLength()); + key, response.contentLength()); } /** @@ -655,7 +658,7 @@ private T exec(String op, Callable call) throws Exception { ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); try (AuditSpan span = getSpanSource().createSpan(op, null, null)) { return call.call(); - } catch (AmazonClientException ex) { + } catch (SdkException ex) { throw S3AUtils.translateException(op, "", ex); } finally { timer.end(op); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java index 6e2f547a22ec1..5fbbc3a127997 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java @@ -21,11 +21,12 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.concurrent.CompletableFuture; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.S3Object; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import org.apache.hadoop.fs.impl.prefetch.Validate; import org.apache.hadoop.fs.s3a.S3AInputStream; @@ -55,7 +56,7 @@ class MockS3ARemoteObject extends S3ARemoteObject { super( S3APrefetchFakes.createReadContext(null, KEY, size, 1, 1), S3APrefetchFakes.createObjectAttributes(BUCKET, KEY, size), - S3APrefetchFakes.createInputStreamCallbacks(BUCKET, KEY), + S3APrefetchFakes.createInputStreamCallbacks(BUCKET), EmptyS3AStatisticsContext.EMPTY_INPUT_STREAM_STATISTICS, S3APrefetchFakes.createChangeTracker(BUCKET, KEY, size) ); @@ -68,7 +69,8 @@ class MockS3ARemoteObject extends S3ARemoteObject { } @Override - public InputStream openForRead(long offset, int size) throws IOException { + public ResponseInputStream openForRead(long offset, int size) + throws IOException { Validate.checkLessOrEqual(offset, "offset", size(), "size()"); Validate.checkLessOrEqual(size, "size", size() - offset, "size() - offset"); @@ -77,11 +79,15 @@ public InputStream openForRead(long offset, int size) throws IOException { throw new IOException("Throwing because throwExceptionOnOpen is true "); } int bufSize = (int) Math.min(size, size() - offset); - return new ByteArrayInputStream(contents, (int) offset, bufSize); + GetObjectResponse objectResponse = GetObjectResponse.builder().build(); + return new ResponseInputStream(objectResponse, + AbortableInputStream.create(new ByteArrayInputStream(contents, + (int) offset, bufSize), () -> {})); } @Override - public void close(InputStream inputStream, int numRemainingBytes) { + public void close(ResponseInputStream inputStream, + int numRemainingBytes) { // do nothing since we do not use a real S3 stream. } @@ -92,7 +98,8 @@ public static byte byteAtOffset(int offset) { public static S3AInputStream.InputStreamCallbacks createClient(String bucketName) { return new S3AInputStream.InputStreamCallbacks() { @Override - public S3Object getObject(GetObjectRequest request) { + public ResponseInputStream getObject( + GetObjectRequest request) { return null; } @@ -102,8 +109,8 @@ public CompletableFuture submit(CallableRaisingIOE operation) { } @Override - public GetObjectRequest newGetRequest(String key) { - return new GetObjectRequest(bucketName, key); + public GetObjectRequest.Builder newGetRequestBuilder(String key) { + return GetObjectRequest.builder().bucket(bucketName).key(key); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java index 6cf2ab241e239..3bf9965861f8a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java @@ -31,10 +31,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -63,6 +63,7 @@ import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.util.functional.CallableRaisingIOE; + import static org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR; import static org.apache.hadoop.fs.s3a.Constants.HADOOP_TMP_DIR; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatisticsStore; @@ -176,32 +177,26 @@ public static ChangeTracker createChangeTracker( createObjectAttributes(bucket, key, fileSize)); } - public static S3ObjectInputStream createS3ObjectInputStream(byte[] buffer) { - return new S3ObjectInputStream(new ByteArrayInputStream(buffer), null); + public static ResponseInputStream createS3ObjectInputStream( + GetObjectResponse objectResponse, byte[] buffer) { + return new ResponseInputStream(objectResponse, + AbortableInputStream.create(new ByteArrayInputStream(buffer), () -> {})); } public static S3AInputStream.InputStreamCallbacks createInputStreamCallbacks( - String bucket, - String key) { + String bucket) { - S3Object object = new S3Object() { - @Override - public S3ObjectInputStream getObjectContent() { - return createS3ObjectInputStream(new byte[8]); - } + GetObjectResponse objectResponse = GetObjectResponse.builder() + .eTag(E_TAG) + .build(); - @Override - public ObjectMetadata getObjectMetadata() { - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setHeader("ETag", E_TAG); - return metadata; - } - }; + ResponseInputStream responseInputStream = + createS3ObjectInputStream(objectResponse, new byte[8]); return new S3AInputStream.InputStreamCallbacks() { @Override - public S3Object getObject(GetObjectRequest request) { - return object; + public ResponseInputStream getObject(GetObjectRequest request) { + return responseInputStream; } @Override @@ -210,8 +205,8 @@ public CompletableFuture submit(CallableRaisingIOE operation) { } @Override - public GetObjectRequest newGetRequest(String key) { - return new GetObjectRequest(bucket, key); + public GetObjectRequest.Builder newGetRequestBuilder(String key) { + return GetObjectRequest.builder().bucket(bucket).key(key); } @Override @@ -230,9 +225,6 @@ public static S3ARemoteInputStream createInputStream( int prefetchBlockSize, int prefetchBlockCount) { - org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key); - - S3AFileStatus fileStatus = createFileStatus(key, fileSize); S3ObjectAttributes s3ObjectAttributes = createObjectAttributes(bucket, key, fileSize); S3AReadOpContext s3AReadOpContext = createReadContext( @@ -243,7 +235,7 @@ public static S3ARemoteInputStream createInputStream( prefetchBlockCount); S3AInputStream.InputStreamCallbacks callbacks = - createInputStreamCallbacks(bucket, key); + createInputStreamCallbacks(bucket); S3AInputStreamStatistics stats = s3AReadOpContext.getS3AStatisticsContext().newInputStreamStatistics(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java index 200b1fc282bac..1a30c04358bac 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java @@ -26,9 +26,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.IntFunction; -import com.amazonaws.event.ProgressEvent; -import com.amazonaws.event.ProgressEventType; -import com.amazonaws.event.ProgressListener; import org.assertj.core.api.Assertions; import org.junit.FixMethodOrder; import org.junit.Test; @@ -48,6 +45,8 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.Statistic; +import org.apache.hadoop.fs.s3a.impl.ProgressListener; +import org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent; import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.util.DurationInfo; @@ -377,10 +376,9 @@ protected int getPartitionSize() { } /** - * Progress callback from AWS. Likely to come in on a different thread. + * Progress callback. */ - private final class ProgressCallback implements Progressable, - ProgressListener { + private final class ProgressCallback implements Progressable, ProgressListener { private AtomicLong bytesTransferred = new AtomicLong(0); private AtomicLong uploadEvents = new AtomicLong(0); private AtomicInteger failures = new AtomicInteger(0); @@ -395,11 +393,8 @@ public void progress() { } @Override - public void progressChanged(ProgressEvent progressEvent) { - ProgressEventType eventType = progressEvent.getEventType(); - if (eventType.isByteCountEvent()) { - bytesTransferred.addAndGet(progressEvent.getBytesTransferred()); - } + public void progressChanged(ProgressListenerEvent eventType, long transferredBytes) { + switch (eventType) { case TRANSFER_PART_FAILED_EVENT: // failure @@ -408,6 +403,7 @@ public void progressChanged(ProgressEvent progressEvent) { break; case TRANSFER_PART_COMPLETED_EVENT: // completion + bytesTransferred.addAndGet(transferredBytes); long elapsedTime = timer.elapsedTime(); double elapsedTimeS = elapsedTime / 1.0e9; long written = bytesTransferred.get(); @@ -415,21 +411,18 @@ public void progressChanged(ProgressEvent progressEvent) { LOG.info(String.format( "Event %s; total uploaded=%d MB in %.1fs;" + " effective upload bandwidth = %.2f MB/s", - progressEvent, + eventType, writtenMB, elapsedTimeS, writtenMB / elapsedTimeS)); break; case REQUEST_BYTE_TRANSFER_EVENT: uploadEvents.incrementAndGet(); break; default: - if (!eventType.isByteCountEvent()) { - LOG.info("Event {}", progressEvent); - } + // nothing break; } } - @Override public String toString() { String sb = "ProgressCallback{" + "bytesTransferred=" + bytesTransferred.get() + diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java index 813eea8389f64..b586fb7dbabc6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java @@ -29,7 +29,7 @@ import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.assertj.core.api.Assertions; @@ -52,6 +52,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.concurrent.HadoopExecutors; + import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING; import static org.apache.hadoop.fs.s3a.Constants.BULK_DELETE_PAGE_SIZE; import static org.apache.hadoop.fs.s3a.Constants.BULK_DELETE_PAGE_SIZE_DEFAULT; @@ -228,7 +229,7 @@ private File deleteFiles(final int requestCount, Path basePath = path("testDeleteObjectThrottling"); final S3AFileSystem fs = getFileSystem(); final String base = fs.pathToKey(basePath); - final List fileList + final List fileList = buildDeleteRequest(base, entries); final FileWriter out = new FileWriter(csvFile); Csvout csvout = new Csvout(out, "\t", "\n"); @@ -304,12 +305,12 @@ private File deleteFiles(final int requestCount, } - private List buildDeleteRequest( + private List buildDeleteRequest( String base, int count) { - List request = new ArrayList<>(count); + List request = new ArrayList<>(count); for (int i = 0; i < count; i++) { - request.add(new DeleteObjectsRequest.KeyVersion( - String.format("%s/file-%04d", base, i))); + request.add(ObjectIdentifier.builder().key( + String.format("%s/file-%04d", base, i)).build()); } return request; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java index de903b3d75a57..173099bb2ca71 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.fs.s3a.S3ADataBlocks; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.Statistic; @@ -41,6 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -49,9 +51,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; @@ -249,18 +250,19 @@ public void testMultiPagesListingPerformanceAndCorrectness() = fs.getWriteOperationHelper(); final RequestFactory requestFactory = writeOperationHelper.getRequestFactory(); - List> futures = + List> futures = new ArrayList<>(numOfPutRequests); for (int i=0; i - writeOperationHelper.putObject(put, PutObjectOptions.keepingDirs(), null))); + PutObjectRequest.Builder putObjectRequestBuilder = requestFactory + .newPutObjectRequestBuilder(fs.pathToKey(file), + null, 0, false); + futures.add(submit(executorService, + () -> writeOperationHelper.putObject(putObjectRequestBuilder.build(), + PutObjectOptions.keepingDirs(), + new S3ADataBlocks.BlockUploadData(new FailingInputStream()), false, null))); } LOG.info("Waiting for PUTs to complete"); waitForCompletion(futures); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java index 99407467df56d..ccc71c58644f2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java @@ -125,8 +125,8 @@ private void skipQuietly(String text) { } protected void assertStorageClass(Path hugeFile) throws IOException { - S3AFileSystem fs = getFileSystem(); - String actual = fs.getObjectMetadata(hugeFile).getStorageClass(); + + String actual = getS3AInternals().getObjectMetadata(hugeFile).storageClassAsString(); assertTrue( "Storage class of object is " + actual + ", expected " + STORAGE_CLASS_REDUCED_REDUNDANCY, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java index 2c1a10a21d0ab..a3d41116182e5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java @@ -263,9 +263,9 @@ private static CsvFile writeStandardHeader(final CsvFile csv, protected static AWSServiceIOException verifyErrorCode(final String code, final AWSServiceIOException ex) { logIntercepted(ex); - if (!code.equals(ex.getErrorCode())) { + if (!code.equals(ex.awsErrorDetails().errorCode())) { throw new AssertionError("Expected Error code" + code - + " actual " + ex.getErrorCode(), + + " actual " + ex.awsErrorDetails().errorCode(), ex); } return ex; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java index 51da971fb7063..4d4af822ee50b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java @@ -415,7 +415,7 @@ public void testSelectSeekFullLandsat() throws Throwable { long increment = 64 * _1KB; // seek forward, comparing bytes - for(offset = 32 * _1KB; offset < _1MB; offset += increment) { + for(offset = 32 * _1KB; offset < 256 * _1KB; offset += increment) { seek(seekStream, offset); assertEquals("Seek position in " + seekStream, offset, seekStream.getPos()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java new file mode 100644 index 0000000000000..461aef726876c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/StreamPublisher.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.select; + +import java.util.Iterator; +import java.util.concurrent.Executor; +import java.util.stream.Stream; + +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.async.SdkPublisher; + +/** + * Publisher used to test the handling of asynchronous responses. + * @param The type of published elements. + */ +final class StreamPublisher implements SdkPublisher { + private final Executor executor; + private final Iterator iterator; + private Boolean done = false; + + StreamPublisher(Stream data, Executor executor) { + this.iterator = data.iterator(); + this.executor = executor; + } + + StreamPublisher(Stream data) { + this(data, Runnable::run); + } + + @Override + public void subscribe(Subscriber subscriber) { + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + if (done) { + return; + } + + if (n < 1) { + done = true; + executor.execute(() -> subscriber.onError(new IllegalArgumentException())); + return; + } + + for (long i = 0; i < n; i++) { + final T value; + try { + synchronized (iterator) { + value = iterator.hasNext() ? iterator.next() : null; + } + } catch (Throwable e) { + executor.execute(() -> subscriber.onError(e)); + break; + } + + if (value == null) { + done = true; + executor.execute(subscriber::onComplete); + break; + } else { + executor.execute(() -> subscriber.onNext(value)); + } + } + } + + @Override + public void cancel() { + done = true; + } + }); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestBlockingEnumeration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestBlockingEnumeration.java new file mode 100644 index 0000000000000..43bdcb062f0a3 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestBlockingEnumeration.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.select; + +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.junit.Assert; +import org.junit.Test; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.exception.SdkException; + +/** + * Unit tests for {@link BlockingEnumeration}. + */ +public final class TestBlockingEnumeration extends Assert { + + @Test + public void containsElement() { + SdkPublisher publisher = new StreamPublisher<>(Stream.of("foo")); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1); + + assertTrue(enumeration.hasMoreElements()); + assertEquals("foo", enumeration.nextElement()); + assertFalse(enumeration.hasMoreElements()); + } + + @Test + public void containsInjectedElement() { + SdkPublisher publisher = new StreamPublisher<>(Stream.of("foo")); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1, "bar"); + + assertTrue(enumeration.hasMoreElements()); + assertEquals("bar", enumeration.nextElement()); + assertTrue(enumeration.hasMoreElements()); + assertEquals("foo", enumeration.nextElement()); + assertFalse(enumeration.hasMoreElements()); + } + + @Test + public void throwsExceptionOnFirstElement() { + SdkPublisher publisher = new StreamPublisher<>( + Stream.of(0, 1) + .map(i -> { + throw SdkException.create("error!", null); + }), + Executors.newSingleThreadExecutor()); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1); + assertThrows(SdkException.class, enumeration::hasMoreElements); + } + + @Test + public void throwsExceptionAfterInjectedElement() { + SdkPublisher publisher = new StreamPublisher<>( + Stream.of(0, 1) + .peek(i -> { + throw SdkException.create("error!", null); + }), + Executors.newSingleThreadExecutor()); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1, 99); + assertTrue(enumeration.hasMoreElements()); + assertEquals(99, enumeration.nextElement().intValue()); + assertThrows(SdkException.class, enumeration::hasMoreElements); + } + + @Test + public void throwsNonSdkException() { + SdkPublisher publisher = new StreamPublisher<>( + Stream.of(0, 1) + .peek(i -> { + throw new RuntimeException("error!", null); + }), + Executors.newSingleThreadExecutor()); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1); + SdkException exception = Assert.assertThrows(SdkException.class, enumeration::hasMoreElements); + assertEquals(RuntimeException.class, exception.getCause().getClass()); + } + + @Test + public void throwsError() { + SdkPublisher publisher = new StreamPublisher<>( + Stream.of(0, 1) + .peek(i -> { + throw new Error("error!", null); + }), + Executors.newSingleThreadExecutor()); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1); + assertThrows(Error.class, enumeration::hasMoreElements); + } + + @Test + public void throwsExceptionOnSecondElement() { + SdkPublisher publisher = new StreamPublisher<>( + Stream.of(0, 1) + .peek(i -> { + if (i == 1) { + throw SdkException.create("error!", null); + } + }), + Executors.newSingleThreadExecutor()); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1); + assertTrue(enumeration.hasMoreElements()); + assertEquals(0, enumeration.nextElement().intValue()); + assertThrows(SdkException.class, enumeration::hasMoreElements); + } + + @Test + public void noMoreElementsAfterThrow() { + SdkPublisher publisher = new StreamPublisher<>( + Stream.of(0, 1) + .map(i -> { + throw SdkException.create("error!", null); + }), + Executors.newSingleThreadExecutor()); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, 1); + assertThrows(SdkException.class, enumeration::hasMoreElements); + assertFalse(enumeration.hasMoreElements()); + } + + @Test + public void buffersOnSameThread() { + verifyBuffering(10, 3, Runnable::run); + } + + @Test + public void publisherOnDifferentThread() { + verifyBuffering(5, 1, Executors.newSingleThreadExecutor()); + } + + @Test + public void publisherOnDifferentThreadWithBuffer() { + verifyBuffering(30, 10, Executors.newSingleThreadExecutor()); + } + + private static void verifyBuffering(int length, int bufferSize, Executor executor) { + AtomicInteger emitted = new AtomicInteger(); + SdkPublisher publisher = new StreamPublisher<>( + IntStream.range(0, length).boxed().peek(i -> emitted.incrementAndGet()), + executor); + + BlockingEnumeration enumeration = + new BlockingEnumeration<>(publisher, bufferSize); + + int pulled = 0; + while (true) { + try { + int expected = Math.min(length, pulled + bufferSize); + if (expected != emitted.get()) { + Thread.sleep(10); + } + assertEquals(expected, emitted.get()); + } catch (InterruptedException e) { + fail("Interrupted: " + e); + } + + if (!enumeration.hasMoreElements()) { + break; + } + + int i = enumeration.nextElement(); + assertEquals(pulled, i); + pulled++; + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java new file mode 100644 index 0000000000000..fdf3b5b725376 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/TestSelectEventStreamPublisher.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.select; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.services.s3.model.SelectObjectContentEventStream; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponse; + +/** + * Unit tests for {@link SelectEventStreamPublisher}. + */ +@RunWith(Parameterized.class) +public final class TestSelectEventStreamPublisher extends Assert { + + @Parameterized.Parameters(name = "threading-{0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {"main"}, + {"background"} + }); + } + + private final String threading; + + public TestSelectEventStreamPublisher(String threading) { + this.threading = threading; + } + + private Executor createExecutor() { + if (threading.equals("main")) { + return Runnable::run; + } else if (threading.equals("background")) { + return Executors.newSingleThreadExecutor(); + } else { + throw new IllegalArgumentException("Unknown: " + threading); + } + } + + @Test + public void emptyRecordsInputStream() throws IOException { + SelectEventStreamPublisher selectEventStreamPublisher = + createSelectPublisher(Stream.of( + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromByteArray(new byte[0])) + .build())); + + try (AbortableInputStream inputStream = + selectEventStreamPublisher.toRecordsInputStream(e -> {})) { + assertEquals(-1, inputStream.read()); + } + } + + @Test + public void multipleRecords() throws IOException { + SelectEventStreamPublisher selectEventStreamPublisher = + createSelectPublisher(Stream.of( + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("foo")) + .build(), + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("bar")) + .build())); + + try (AbortableInputStream inputStream = + selectEventStreamPublisher.toRecordsInputStream(e -> {})) { + String result = readAll(inputStream); + assertEquals("foobar", result); + } + } + + @Test + public void skipsOtherEvents() throws IOException { + SelectEventStreamPublisher selectEventStreamPublisher = + createSelectPublisher(Stream.of( + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("foo")) + .build(), + SelectObjectContentEventStream.progressBuilder() + .build(), + SelectObjectContentEventStream.statsBuilder() + .build(), + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("bar")) + .build(), + SelectObjectContentEventStream.endBuilder() + .build())); + + try (AbortableInputStream inputStream = + selectEventStreamPublisher.toRecordsInputStream(e -> {})) { + String result = readAll(inputStream); + assertEquals("foobar", result); + } + } + + @Test + public void callsOnEndEvent() throws IOException { + SelectEventStreamPublisher selectEventStreamPublisher = + createSelectPublisher(Stream.of( + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("foo")) + .build(), + SelectObjectContentEventStream.endBuilder() + .build())); + + AtomicBoolean endEvent = new AtomicBoolean(false); + try (AbortableInputStream inputStream = + selectEventStreamPublisher.toRecordsInputStream(e -> endEvent.set(true))) { + String result = readAll(inputStream); + assertEquals("foo", result); + } + + assertTrue(endEvent.get()); + } + + @Test + public void handlesErrors() throws IOException { + SelectEventStreamPublisher selectEventStreamPublisher = + createSelectPublisher(Stream.of( + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("foo")) + .build(), + SelectObjectContentEventStream.recordsBuilder() + .payload(SdkBytes.fromUtf8String("bar")) + .build()) + .map(e -> { + throw SdkException.create("error!", null); + })); + + try (AbortableInputStream inputStream = + selectEventStreamPublisher.toRecordsInputStream(e -> {})) { + assertThrows(SdkException.class, () -> readAll(inputStream)); + } + } + + private SelectEventStreamPublisher createSelectPublisher( + Stream stream) { + SdkPublisher sdkPublisher = + new StreamPublisher<>(stream, createExecutor()); + CompletableFuture future = + CompletableFuture.completedFuture(null); + SelectObjectContentResponse response = + SelectObjectContentResponse.builder().build(); + return new SelectEventStreamPublisher(future, response, sdkPublisher); + } + + private static String readAll(InputStream inputStream) throws IOException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + byte[] buffer = new byte[8096]; + int read; + while ((read = inputStream.read(buffer, 0, buffer.length)) != -1) { + outputStream.write(buffer, 0, read); + } + return outputStream.toString(); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java index e7696996dbd1a..8c97d896edbde 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestAWSStatisticCollection.java @@ -53,7 +53,7 @@ public void testLandsatStatistics() throws Throwable { conf.unset("fs.s3a.bucket.landsat-pds.endpoint"); try (S3AFileSystem fs = (S3AFileSystem) path.getFileSystem(conf)) { - fs.getObjectMetadata(path); + fs.getS3AInternals().getObjectMetadata(path); IOStatistics iostats = fs.getIOStatistics(); assertThatStatisticCounter(iostats, STORE_IO_REQUEST.getSymbol()) @@ -71,7 +71,7 @@ public void testCommonCrawlStatistics() throws Throwable { conf.set(ENDPOINT, DEFAULT_ENDPOINT); try (S3AFileSystem fs = (S3AFileSystem) path.getFileSystem(conf)) { - fs.getObjectMetadata(path); + fs.getS3AInternals().getObjectMetadata(path); IOStatistics iostats = fs.getIOStatistics(); assertThatStatisticCounter(iostats, STORE_IO_REQUEST.getSymbol()) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/ExtraAssertions.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/ExtraAssertions.java index 77c7736575c39..fdf5eb53e187e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/ExtraAssertions.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/ExtraAssertions.java @@ -147,7 +147,7 @@ public static T extractCause(Class expected, */ protected void assertStatusCode(AWSServiceIOException e, int code) throws AWSServiceIOException { - if (e.getStatusCode() != code) { + if (e.statusCode() != code) { throw e; } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java index fa1ad2db62af7..004e15676a04a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java @@ -21,10 +21,9 @@ import java.io.IOException; import java.util.List; -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.MultiObjectDeleteException; -import com.amazonaws.services.s3.transfer.model.CopyResult; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -33,6 +32,7 @@ import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus; import org.apache.hadoop.fs.s3a.S3AReadOpContext; import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; import org.apache.hadoop.fs.s3a.impl.OperationCallbacks; /** @@ -88,7 +88,7 @@ public RemoteIterator listFilesAndDirectoryMarkers( } @Override - public CopyResult copyFile( + public CopyObjectResponse copyFile( String srcKey, String destKey, S3ObjectAttributes srcAttributes, @@ -99,9 +99,9 @@ public CopyResult copyFile( @Override public void removeKeys( - List keysToDelete, + List keysToDelete, boolean deleteFakeDir) - throws MultiObjectDeleteException, AmazonClientException, + throws MultiObjectDeleteException, AwsServiceException, IOException { } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java index ffba558d11fd0..eee0c71950566 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java @@ -18,10 +18,12 @@ package org.apache.hadoop.fs.s3a.test; -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.amazonaws.services.s3.model.SelectObjectContentRequest; -import com.amazonaws.services.s3.model.SelectObjectContentResult; +import java.util.concurrent.CompletableFuture; + +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; +import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler; import org.apache.hadoop.fs.s3a.WriteOperationHelper; @@ -32,12 +34,14 @@ public class MinimalWriteOperationHelperCallbacks implements WriteOperationHelper.WriteOperationHelperCallbacks { @Override - public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + public CompletableFuture selectObjectContent( + SelectObjectContentRequest request, + SelectObjectContentResponseHandler th) { return null; } @Override - public CompleteMultipartUploadResult completeMultipartUpload( + public CompleteMultipartUploadResponse completeMultipartUpload( CompleteMultipartUploadRequest request) { return null; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java index 127fcbab75023..ab22c51f28b7b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java @@ -224,13 +224,6 @@ public void testRunNoArgs() throws Throwable { runToFailure(EXIT_USAGE, MARKERS); } - @Test - public void testRunWrongBucket() throws Throwable { - runToFailure(EXIT_NOT_FOUND, MARKERS, - AUDIT, - "s3a://this-bucket-does-not-exist-hopefully"); - } - /** * Run with a path that doesn't exist. */ From e4dd7fe1b7a622cdce7a50109ce98079bff4852d Mon Sep 17 00:00:00 2001 From: Jian Zhang <38941777+KeeProMise@users.noreply.github.com> Date: Wed, 13 Sep 2023 05:44:55 +0800 Subject: [PATCH 021/155] HDFS-17139. RBF: For the doc of the class RouterAdminProtocolTranslatorPB, it describes the function of the class ClientNamenodeProtocolTranslatorPB (#5917) --- .../hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java index 6b0df23c40692..db71c1cb70cf9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java @@ -112,8 +112,8 @@ import org.apache.hadoop.thirdparty.protobuf.ServiceException; /** - * This class forwards NN's ClientProtocol calls as RPC calls to the NN server - * while translating from the parameter types used in ClientProtocol to the + * This class forwards RouterAdminProtocol calls as RPC calls to the RouterAdmin server + * while translating from the parameter types used in RouterAdminProtocol to the * new PB types. */ @InterfaceAudience.Private From 4652d22b9195d35eac4d7e02d1f99ebc6a5835c7 Mon Sep 17 00:00:00 2001 From: dannytbecker <43830149+dannytbecker@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:46:08 -0700 Subject: [PATCH 022/155] HDFS-17178: BootstrapStandby needs to handle RollingUpgrade (#6018) --- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 + .../hadoop/hdfs/protocolPB/PBHelper.java | 4 +- .../server/common/HdfsServerConstants.java | 6 + .../hadoop/hdfs/server/common/Storage.java | 6 +- .../hadoop/hdfs/server/namenode/FSImage.java | 14 +-- .../hdfs/server/namenode/FSNamesystem.java | 4 +- .../hdfs/server/namenode/NNStorage.java | 25 ++-- .../server/namenode/ha/BootstrapStandby.java | 50 ++++++-- .../hdfs/server/protocol/NamespaceInfo.java | 23 ++++ .../hdfs/server/namenode/FSImageTestUtil.java | 7 +- .../namenode/ha/TestBootstrapStandby.java | 113 +++++++++++++++++- 11 files changed, 211 insertions(+), 45 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index baca814eccbda..9efea6b6d093a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -219,6 +219,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> lz4-java test + + org.apache.hadoop + hadoop-annotations + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index b5f7b9c80f25f..b7cee876623bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -348,10 +348,12 @@ public static BlockKey[] convertBlockKeys(List list) { public static NamespaceInfo convert(NamespaceInfoProto info) { StorageInfoProto storage = info.getStorageInfo(); - return new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(), + NamespaceInfo nsInfo = new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(), info.getBlockPoolID(), storage.getCTime(), info.getBuildVersion(), info.getSoftwareVersion(), info.getCapabilities(), convert(info.getState())); + nsInfo.setStorageInfo(convert(storage, NodeType.NAME_NODE)); + return nsInfo; } public static NamenodeCommand convert(NamenodeCommandProto cmd) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 8e343f998a626..ce788e78703cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -61,6 +61,12 @@ public interface HdfsServerConstants { */ int NAMENODE_LAYOUT_VERSION = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION; + /** + * Current minimum compatible version for NameNode + * Please see {@link NameNodeLayoutVersion.Feature} on adding new layout version. + */ + int MINIMUM_COMPATIBLE_NAMENODE_LAYOUT_VERSION + = NameNodeLayoutVersion.MINIMUM_COMPATIBLE_LAYOUT_VERSION; /** * Path components that are reserved in HDFS. *

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index 7a4535f33898c..16afb8496f2a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -1114,11 +1114,7 @@ protected boolean containsStorageDir(StorageLocation location, String bpid) } public NamespaceInfo getNamespaceInfo() { - return new NamespaceInfo( - getNamespaceID(), - getClusterID(), - null, - getCTime()); + return new NamespaceInfo(this); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index c149d6021bbb8..12f79e8c52e1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -258,7 +258,7 @@ boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target, if (startOpt == StartupOption.METADATAVERSION) { System.out.println("HDFS Image Version: " + layoutVersion); System.out.println("Software format version: " + - HdfsServerConstants.NAMENODE_LAYOUT_VERSION); + storage.getServiceLayoutVersion()); return false; } @@ -269,11 +269,11 @@ boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target, && startOpt != StartupOption.UPGRADEONLY && !RollingUpgradeStartupOption.STARTED.matches(startOpt) && layoutVersion < Storage.LAST_PRE_UPGRADE_LAYOUT_VERSION - && layoutVersion != HdfsServerConstants.NAMENODE_LAYOUT_VERSION) { + && layoutVersion != storage.getServiceLayoutVersion()) { throw new IOException( "\nFile system image contains an old layout version " + storage.getLayoutVersion() + ".\nAn upgrade to version " - + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + " is required.\n" + + storage.getServiceLayoutVersion() + " is required.\n" + "Please restart NameNode with the \"" + RollingUpgradeStartupOption.STARTED.getOptionString() + "\" option if a rolling upgrade is already started;" @@ -462,7 +462,7 @@ void doUpgrade(FSNamesystem target) throws IOException { long oldCTime = storage.getCTime(); storage.cTime = now(); // generate new cTime for the state int oldLV = storage.getLayoutVersion(); - storage.layoutVersion = HdfsServerConstants.NAMENODE_LAYOUT_VERSION; + storage.layoutVersion = storage.getServiceLayoutVersion(); List errorSDs = Collections.synchronizedList(new ArrayList()); @@ -523,11 +523,11 @@ void doRollback(FSNamesystem fsns) throws IOException { boolean canRollback = false; FSImage prevState = new FSImage(conf); try { - prevState.getStorage().layoutVersion = HdfsServerConstants.NAMENODE_LAYOUT_VERSION; + prevState.getStorage().layoutVersion = storage.getServiceLayoutVersion(); for (Iterator it = storage.dirIterator(false); it.hasNext();) { StorageDirectory sd = it.next(); if (!NNUpgradeUtil.canRollBack(sd, storage, prevState.getStorage(), - HdfsServerConstants.NAMENODE_LAYOUT_VERSION)) { + storage.getServiceLayoutVersion())) { continue; } LOG.info("Can perform rollback for " + sd); @@ -538,7 +538,7 @@ void doRollback(FSNamesystem fsns) throws IOException { // If HA is enabled, check if the shared log can be rolled back as well. editLog.initJournalsForWrite(); boolean canRollBackSharedEditLog = editLog.canRollBackSharedLog( - prevState.getStorage(), HdfsServerConstants.NAMENODE_LAYOUT_VERSION); + prevState.getStorage(), storage.getServiceLayoutVersion()); if (canRollBackSharedEditLog) { LOG.info("Can perform rollback for shared edit log."); canRollback = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e08002a8dde9b..eb8022dc63451 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1898,9 +1898,7 @@ long getCTime() { * Version of @see #getNamespaceInfo() that is not protected by a lock. */ NamespaceInfo unprotectedGetNamespaceInfo() { - return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(), - getClusterId(), getBlockPoolId(), - getFSImage().getStorage().getCTime(), getState()); + return new NamespaceInfo(getFSImage().getStorage(), getState()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index ca8b6fc752065..b008d0cf0e123 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -600,10 +600,17 @@ private void format(StorageDirectory sd) throws IOException { * Format all available storage directories. */ public void format(NamespaceInfo nsInfo) throws IOException { + format(nsInfo, false); + } + + /** + * Format all available storage directories. + */ + public void format(NamespaceInfo nsInfo, boolean isRollingUpgrade) + throws IOException { Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 || - nsInfo.getLayoutVersion() == - HdfsServerConstants.NAMENODE_LAYOUT_VERSION, - "Bad layout version: %s", nsInfo.getLayoutVersion()); + nsInfo.getLayoutVersion() == getServiceLayoutVersion() || + isRollingUpgrade, "Bad layout version: %s", nsInfo.getLayoutVersion()); this.setStorageInfo(nsInfo); this.blockpoolID = nsInfo.getBlockPoolID(); @@ -621,7 +628,7 @@ public static NamespaceInfo newNamespaceInfo() } public void format() throws IOException { - this.layoutVersion = HdfsServerConstants.NAMENODE_LAYOUT_VERSION; + this.layoutVersion = getServiceLayoutVersion(); for (Iterator it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); @@ -683,7 +690,7 @@ void readProperties(StorageDirectory sd, StartupOption startupOption) "storage directory " + sd.getRoot().getAbsolutePath()); } props.setProperty("layoutVersion", - Integer.toString(HdfsServerConstants.NAMENODE_LAYOUT_VERSION)); + Integer.toString(getServiceLayoutVersion())); } setFieldsFromProperties(props, sd); } @@ -706,7 +713,7 @@ private void setDeprecatedPropertiesForUpgrade(Properties props) { * This should only be used during upgrades. */ String getDeprecatedProperty(String prop) { - assert getLayoutVersion() > HdfsServerConstants.NAMENODE_LAYOUT_VERSION : + assert getLayoutVersion() > getServiceLayoutVersion() : "getDeprecatedProperty should only be done when loading " + "storage from past versions during upgrade."; return deprecatedProperties.get(prop); @@ -1133,11 +1140,7 @@ LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) { @Override public NamespaceInfo getNamespaceInfo() { - return new NamespaceInfo( - getNamespaceID(), - getClusterID(), - getBlockPoolID(), - getCTime()); + return new NamespaceInfo(this); } public String getNNDirectorySize() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index 7a9ce46b1159f..97af6f6509a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -170,6 +171,7 @@ private int doRun() throws IOException { NamenodeProtocol proxy = null; NamespaceInfo nsInfo = null; boolean isUpgradeFinalized = false; + boolean isRollingUpgrade = false; RemoteNameNodeInfo proxyInfo = null; for (int i = 0; i < remoteNNs.size(); i++) { proxyInfo = remoteNNs.get(i); @@ -180,8 +182,9 @@ private int doRun() throws IOException { // bootstrapping the other NNs from that layout, it will only contact the single NN. // However, if there cluster is already running and you are adding a NN later (e.g. // replacing a failed NN), then this will bootstrap from any node in the cluster. - nsInfo = proxy.versionRequest(); + nsInfo = getProxyNamespaceInfo(proxy); isUpgradeFinalized = proxy.isUpgradeFinalized(); + isRollingUpgrade = proxy.isRollingUpgrade(); break; } catch (IOException ioe) { LOG.warn("Unable to fetch namespace information from remote NN at " + otherIpcAddress @@ -199,10 +202,17 @@ private int doRun() throws IOException { return ERR_CODE_FAILED_CONNECT; } - if (!checkLayoutVersion(nsInfo)) { - LOG.error("Layout version on remote node (" + nsInfo.getLayoutVersion() - + ") does not match " + "this node's layout version (" - + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + ")"); + if (!checkLayoutVersion(nsInfo, isRollingUpgrade)) { + if(isRollingUpgrade) { + LOG.error("Layout version on remote node in rolling upgrade ({}, {})" + + " is not compatible based on minimum compatible version ({})", + nsInfo.getLayoutVersion(), proxyInfo.getIpcAddress(), + HdfsServerConstants.MINIMUM_COMPATIBLE_NAMENODE_LAYOUT_VERSION); + } else { + LOG.error("Layout version on remote node ({}) does not match this " + + "node's service layout version ({})", nsInfo.getLayoutVersion(), + nsInfo.getServiceLayoutVersion()); + } return ERR_CODE_INVALID_VERSION; } @@ -217,9 +227,11 @@ private int doRun() throws IOException { " Block pool ID: " + nsInfo.getBlockPoolID() + "\n" + " Cluster ID: " + nsInfo.getClusterID() + "\n" + " Layout version: " + nsInfo.getLayoutVersion() + "\n" + + " Service Layout version: " + nsInfo.getServiceLayoutVersion() + "\n" + " isUpgradeFinalized: " + isUpgradeFinalized + "\n" + + " isRollingUpgrade: " + isRollingUpgrade + "\n" + "====================================================="); - + NNStorage storage = new NNStorage(conf, dirsToFormat, editUrisToFormat); if (!isUpgradeFinalized) { @@ -231,7 +243,7 @@ private int doRun() throws IOException { if (!doPreUpgrade(storage, nsInfo)) { return ERR_CODE_ALREADY_FORMATTED; } - } else if (!format(storage, nsInfo)) { // prompt the user to format storage + } else if (!format(storage, nsInfo, isRollingUpgrade)) { // prompt the user to format storage return ERR_CODE_ALREADY_FORMATTED; } @@ -254,20 +266,26 @@ private int doRun() throws IOException { return 0; } + @VisibleForTesting + public NamespaceInfo getProxyNamespaceInfo(NamenodeProtocol proxy) + throws IOException { + return proxy.versionRequest(); + } + /** * Iterate over all the storage directories, checking if it should be * formatted. Format the storage if necessary and allowed by the user. * @return True if formatting is processed */ - private boolean format(NNStorage storage, NamespaceInfo nsInfo) - throws IOException { + private boolean format(NNStorage storage, NamespaceInfo nsInfo, + boolean isRollingUpgrade) throws IOException { // Check with the user before blowing away data. if (!Storage.confirmFormat(storage.dirIterable(null), force, interactive)) { storage.close(); return false; } else { // Format the storage (writes VERSION file) - storage.format(nsInfo); + storage.format(nsInfo, isRollingUpgrade); return true; } } @@ -302,7 +320,7 @@ private boolean doPreUpgrade(NNStorage storage, NamespaceInfo nsInfo) // format the storage. Although this format is done through the new // software, since in HA setup the SBN is rolled back through // "-bootstrapStandby", we should still be fine. - if (!isFormatted && !format(storage, nsInfo)) { + if (!isFormatted && !format(storage, nsInfo, false)) { return false; } @@ -405,8 +423,14 @@ private boolean checkLogsAvailableForRead(FSImage image, long imageTxId, } } - private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException { - return (nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION); + private boolean checkLayoutVersion(NamespaceInfo nsInfo, boolean isRollingUpgrade) { + if (isRollingUpgrade) { + // During a rolling upgrade the service layout versions may be different, + // but we should check that the layout version being sent is compatible + return nsInfo.getLayoutVersion() <= + HdfsServerConstants.MINIMUM_COMPATIBLE_NAMENODE_LAYOUT_VERSION; + } + return nsInfo.getLayoutVersion() == nsInfo.getServiceLayoutVersion(); } private void parseConfAndFindOtherNN() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java index 93125a22dbd24..52331ddc60ddd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java @@ -110,6 +110,29 @@ public NamespaceInfo(int nsID, String clusterID, String bpID, this.capabilities = capabilities; } + public NamespaceInfo(StorageInfo storage) { + super(storage); + if (storage instanceof NamespaceInfo) { + this.capabilities = ((NamespaceInfo)storage).capabilities; + this.blockPoolID = ((NamespaceInfo)storage).blockPoolID; + } else { + this.capabilities = CAPABILITIES_SUPPORTED; + } + this.buildVersion = Storage.getBuildVersion(); + this.softwareVersion = VersionInfo.getVersion(); + if (storage instanceof NNStorage) { + this.blockPoolID = ((NNStorage)storage).getBlockPoolID(); + } else { + this.blockPoolID = null; + } + + } + + public NamespaceInfo(StorageInfo storage, HAServiceState st) { + this(storage); + this.state = st; + } + public NamespaceInfo(int nsID, String clusterID, String bpID, long cT) { this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index 0334cb8a45f37..0c660bb7f11f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -609,9 +609,10 @@ public static long getNSQuota(FSNamesystem ns) { public static void assertNNFilesMatch(MiniDFSCluster cluster) throws Exception { List curDirs = Lists.newArrayList(); - curDirs.addAll(FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0)); - curDirs.addAll(FSImageTestUtil.getNameNodeCurrentDirs(cluster, 1)); - + for (int i = 0; i < cluster.getNumNameNodes(); i++) { + curDirs.addAll(FSImageTestUtil.getNameNodeCurrentDirs(cluster, i)); + } + // Ignore seen_txid file, since the newly bootstrapped standby // will have a higher seen_txid than the one it bootstrapped from. Set ignoredFiles = ImmutableSet.of("seen_txid"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java index 0e83bec11f313..f470a90f374c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java @@ -17,9 +17,15 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby.ERR_CODE_INVALID_VERSION; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import java.io.File; import java.io.IOException; @@ -28,6 +34,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; +import org.apache.hadoop.hdfs.server.common.HttpGetFailedException; +import org.apache.hadoop.hdfs.server.namenode.FSImage; +import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -172,6 +184,103 @@ public void testDownloadingLaterCheckpoint() throws Exception { restartNameNodesFromIndex(1); } + /** + * Test for downloading a checkpoint while the cluster is in rolling upgrade. + */ + @Test + public void testRollingUpgradeBootstrapStandby() throws Exception { + removeStandbyNameDirs(); + + int futureVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1; + + DistributedFileSystem fs = cluster.getFileSystem(0); + NameNodeAdapter.enterSafeMode(nn0, false); + NameNodeAdapter.saveNamespace(nn0); + NameNodeAdapter.leaveSafeMode(nn0); + + // Setup BootstrapStandby to think it is a future NameNode version + BootstrapStandby bs = spy(new BootstrapStandby()); + doAnswer(nsInfo -> { + NamespaceInfo nsInfoSpy = (NamespaceInfo) spy(nsInfo.callRealMethod()); + doReturn(futureVersion).when(nsInfoSpy).getServiceLayoutVersion(); + return nsInfoSpy; + }).when(bs).getProxyNamespaceInfo(any()); + + // BootstrapStandby should fail if the node has a future version + // and the cluster isn't in rolling upgrade + bs.setConf(cluster.getConfiguration(1)); + assertEquals("BootstrapStandby should return ERR_CODE_INVALID_VERSION", + ERR_CODE_INVALID_VERSION, bs.run(new String[]{"-force"})); + + // Start rolling upgrade + fs.rollingUpgrade(RollingUpgradeAction.PREPARE); + nn0 = spy(nn0); + + // Make nn0 think it is a future version + doAnswer(fsImage -> { + FSImage fsImageSpy = (FSImage) spy(fsImage.callRealMethod()); + doAnswer(storage -> { + NNStorage storageSpy = (NNStorage) spy(storage.callRealMethod()); + doReturn(futureVersion).when(storageSpy).getServiceLayoutVersion(); + return storageSpy; + }).when(fsImageSpy).getStorage(); + return fsImageSpy; + }).when(nn0).getFSImage(); + + // Roll edit logs a few times to inflate txid + nn0.getRpcServer().rollEditLog(); + nn0.getRpcServer().rollEditLog(); + // Make checkpoint + NameNodeAdapter.enterSafeMode(nn0, false); + NameNodeAdapter.saveNamespace(nn0); + NameNodeAdapter.leaveSafeMode(nn0); + + long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0) + .getFSImage().getMostRecentCheckpointTxId(); + assertEquals(11, expectedCheckpointTxId); + + for (int i = 1; i < maxNNCount; i++) { + // BootstrapStandby on Standby NameNode + bs.setConf(cluster.getConfiguration(i)); + bs.run(new String[]{"-force"}); + FSImageTestUtil.assertNNHasCheckpoints(cluster, i, + ImmutableList.of((int) expectedCheckpointTxId)); + } + + // Make sure the bootstrap was successful + FSImageTestUtil.assertNNFilesMatch(cluster); + + // We should now be able to start the standby successfully + restartNameNodesFromIndex(1, "-rollingUpgrade", "started"); + + // Cleanup standby dirs + for (int i = 1; i < maxNNCount; i++) { + cluster.shutdownNameNode(i); + } + removeStandbyNameDirs(); + + // BootstrapStandby should fail if it thinks it's version is future version + // before rolling upgrade is finalized; + doAnswer(nsInfo -> { + NamespaceInfo nsInfoSpy = (NamespaceInfo) spy(nsInfo.callRealMethod()); + nsInfoSpy.layoutVersion = futureVersion; + doReturn(futureVersion).when(nsInfoSpy).getServiceLayoutVersion(); + return nsInfoSpy; + }).when(bs).getProxyNamespaceInfo(any()); + + for (int i = 1; i < maxNNCount; i++) { + bs.setConf(cluster.getConfiguration(i)); + assertThrows("BootstrapStandby should fail the image transfer request", + HttpGetFailedException.class, () -> { + try { + bs.run(new String[]{"-force"}); + } catch (RuntimeException e) { + throw e.getCause(); + } + }); + } + } + /** * Test for the case where the shared edits dir doesn't have * all of the recent edit logs. @@ -336,10 +445,10 @@ private void removeStandbyNameDirs() { } } - private void restartNameNodesFromIndex(int start) throws IOException { + private void restartNameNodesFromIndex(int start, String... args) throws IOException { for (int i = start; i < maxNNCount; i++) { // We should now be able to start the standby successfully. - cluster.restartNameNode(i, false); + cluster.restartNameNode(i, false, args); } cluster.waitClusterUp(); From 475932c524aaecbe3a14df47d11e971b3bdf4e15 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 13 Sep 2023 05:47:37 +0800 Subject: [PATCH 023/155] YARN-11562. [Federation] GPG Support Query Policies In Web. (#6023) --- .../pom.xml | 7 ++ .../webapp/GPGController.java | 7 +- .../webapp/GPGPoliciesBlock.java | 110 ++++++++++++++++++ .../webapp/GPGPoliciesPage.java | 55 +++++++++ .../webapp/GPGWebApp.java | 3 +- .../webapp/NavBlock.java | 1 + .../webapp/TestGPGWebApp.java | 40 +++++++ 7 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesBlock.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesPage.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/TestGPGWebApp.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml index d8f3b60f6ebd7..f7c7c5913990c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml @@ -91,6 +91,13 @@ test + + org.apache.hadoop + hadoop-yarn-common + test-jar + test + + org.hsqldb hsqldb diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java index c3955ea3fb379..4b81c8a128c85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java @@ -38,7 +38,12 @@ public void index() { } public void overview() { - setTitle("GPG Details"); + setTitle("GPG"); render(GPGOverviewPage.class); } + + public void policies() { + setTitle("Global Policy Generator Policies"); + render(GPGPoliciesPage.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesBlock.java new file mode 100644 index 0000000000000..641576a30ca4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesBlock.java @@ -0,0 +1,110 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import com.google.inject.Inject; +import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; +import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; +import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Map; + +/** + * Overview block for the GPG Policies Web UI. + */ +public class GPGPoliciesBlock extends HtmlBlock { + + private final GlobalPolicyGenerator gpg; + + private final FederationStateStoreFacade facade; + + @Inject + GPGPoliciesBlock(GlobalPolicyGenerator gpg, ViewContext ctx) { + super(ctx); + this.gpg = gpg; + this.facade = FederationStateStoreFacade.getInstance(gpg.getConfig()); + } + + @Override + protected void render(Block html) { + try { + Collection policies = + facade.getPoliciesConfigurations().values(); + initYarnFederationPolicies(policies, html); + } catch (Exception e) { + LOG.error("Get GPGPolicies Error.", e); + } + } + + private void initYarnFederationPolicies(Collection policies, + Block html) throws FederationPolicyInitializationException { + + Hamlet.TBODY> tbody = html.table("#policies"). + thead(). + tr(). + th(".queue", "Queue Name"). + th(".policyType", "Policy Type"). + th(".routerPolicyWeights", "Router PolicyWeights"). + th(".amrmPolicyWeights", "Router AMRMPolicyWeights"). + th(".headroomAlpha", "Router Headroom Alpha"). + __().__(). + tbody(); + + if (policies != null) { + for (SubClusterPolicyConfiguration policy : policies) { + Hamlet.TR>> row = tbody.tr().td(policy.getQueue()); + // Policy Type + String type = policy.getType(); + row = row.td(type); + + // WeightedPolicyInfo + ByteBuffer params = policy.getParams(); + WeightedPolicyInfo weightedPolicyInfo = WeightedPolicyInfo.fromByteBuffer(params); + row = row.td(policyWeight2String(weightedPolicyInfo.getRouterPolicyWeights())); + row = row.td(policyWeight2String(weightedPolicyInfo.getAMRMPolicyWeights())); + row.td(String.valueOf(weightedPolicyInfo.getHeadroomAlpha())).__(); + } + } + + tbody.__().__(); + } + + /** + * We will convert the PolicyWeight to string format. + * + * @param weights PolicyWeight. + * @return string format PolicyWeight. example: SC-1:0.91, SC-2:0.09 + */ + private String policyWeight2String(Map weights) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : weights.entrySet()) { + sb.append(entry.getKey().toId()).append(": ").append(entry.getValue()).append(", "); + } + if (sb.length() > 2) { + sb.setLength(sb.length() - 2); + } + return sb.toString(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesPage.java new file mode 100644 index 0000000000000..f9ff5f5c8bd07 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesPage.java @@ -0,0 +1,55 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; + +/** + * Overview page for the GPG Policies Web UI. + */ +public class GPGPoliciesPage extends TwoColumnLayout { + + @Override + protected void preHead(Page.HTML<__> html) { + commonPreHead(html); + } + + protected void commonPreHead(Page.HTML<__> html) { + setTitle("Global Policy Generator Policies"); + set(ACCORDION_ID, "nav"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); + set(DATATABLES_ID, "policies"); + } + + @Override + protected Class content() { + return GPGPoliciesBlock.class; + } + + @Override + protected Class nav() { + return NavBlock.class; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java index eedb1790019eb..b3d63634ca7b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java @@ -25,7 +25,7 @@ /** * The GPG webapp. */ -public class GPGWebApp extends WebApp{ +public class GPGWebApp extends WebApp { private GlobalPolicyGenerator gpg; public GPGWebApp(GlobalPolicyGenerator gpg) { @@ -41,5 +41,6 @@ public void setup() { bind(GlobalPolicyGenerator.class).toInstance(gpg); } route("/", GPGController.class, "overview"); + route("/policies", GPGController.class, "policies"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java index 1fee9d08e8c1e..3650287df388a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java @@ -31,6 +31,7 @@ public void render(Block html) { h3("GPG"). ul(). li().a(url(""), "Overview").__(). + li().a(url("policies"), "Policies").__(). __(). h3("Tools"). ul(). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/TestGPGWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/TestGPGWebApp.java new file mode 100644 index 0000000000000..94ac4cbbc8604 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/TestGPGWebApp.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class TestGPGWebApp { + + private static final Logger LOG = LoggerFactory.getLogger(TestGPGWebApp.class); + + @Test + public void testGPGPoliciesPageWebView() + throws InterruptedException, YarnException, IOException { + LOG.info("testGPGPoliciesPageWebView."); + WebAppTests.testPage(GPGPoliciesPage.class, GlobalPolicyGenerator.class, + new GlobalPolicyGenerator()); + } +} From 9c8cdbe28e317e703560c3eefcde2f6eebcabb17 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 13 Sep 2023 05:48:40 +0800 Subject: [PATCH 024/155] YARN-11433. Router's main() should support generic options. (#6012) --- .../GlobalPolicyGenerator.java | 20 +++++++++- .../TestGlobalPolicyGenerator.java | 15 +++++++ .../hadoop/yarn/server/router/Router.java | 40 ++++++++++++++----- .../hadoop/yarn/server/router/TestRouter.java | 14 +++++++ 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index dc21d6ef36cd6..81a999d76a28e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; import java.io.IOException; +import java.io.PrintStream; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -37,6 +38,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; @@ -292,7 +294,19 @@ private String getHostName(Configuration config) public static void main(String[] argv) { try { - startGPG(argv, new YarnConfiguration()); + YarnConfiguration conf = new YarnConfiguration(); + GenericOptionsParser hParser = new GenericOptionsParser(conf, argv); + argv = hParser.getRemainingArgs(); + if (argv.length > 1) { + if (argv[0].equals("-format-policy-store")) { + // TODO: YARN-11561. [Federation] GPG Supports Format PolicyStateStore. + System.err.println("format-policy-store is not yet supported."); + } else { + printUsage(System.err); + } + } else { + startGPG(argv, conf); + } } catch (Throwable t) { LOG.error("Error starting global policy generator", t); System.exit(-1); @@ -307,4 +321,8 @@ public static long getGPGStartupTime() { public WebApp getWebApp() { return webApp; } + + private static void printUsage(PrintStream out) { + out.println("Usage: yarn gpg [-format-policy-store]"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java index 4f8521dd8aeac..a32970488f9c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java @@ -26,9 +26,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.util.List; import java.util.concurrent.TimeoutException; +import static org.junit.Assert.assertTrue; + /** * Unit test for GlobalPolicyGenerator. */ @@ -58,6 +62,17 @@ public void testGpgWithFederation() throws InterruptedException, TimeoutExceptio }, 100, 5000); } + @Test + public void testGPGCLI() { + ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); + ByteArrayOutputStream dataErr = new ByteArrayOutputStream(); + System.setOut(new PrintStream(dataOut)); + System.setErr(new PrintStream(dataErr)); + GlobalPolicyGenerator.main(new String[]{"-help", "-format-policy-store"}); + assertTrue(dataErr.toString().contains( + "Usage: yarn gpg [-format-policy-store]")); + } + @Test public void testUserProvidedUGIConf() throws Exception { String errMsg = "Invalid attribute value for " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 56bb91db52667..4761866253ff7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router; import java.io.IOException; +import java.io.PrintStream; import java.net.InetAddress; import java.net.URL; import java.net.InetSocketAddress; @@ -36,6 +37,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; @@ -295,18 +297,29 @@ public static void main(String[] argv) { StringUtils.startupShutdownMessage(Router.class, argv, LOG); Router router = new Router(); try { - - // Remove the old hook if we are rebooting. - if (null != routerShutdownHook) { - ShutdownHookManager.get().removeShutdownHook(routerShutdownHook); + GenericOptionsParser hParser = new GenericOptionsParser(conf, argv); + argv = hParser.getRemainingArgs(); + if (argv.length > 1) { + if (argv[0].equals("-format-state-store")) { + // TODO: YARN-11548. [Federation] Router Supports Format FederationStateStore. + System.err.println("format-state-store is not yet supported."); + } else if (argv[0].equals("-remove-application-from-state-store") && argv.length == 2) { + // TODO: YARN-11547. [Federation] + // Router Supports Remove individual application records from FederationStateStore. + System.err.println("remove-application-from-state-store is not yet supported."); + } else { + printUsage(System.err); + } + } else { + // Remove the old hook if we are rebooting. + if (null != routerShutdownHook) { + ShutdownHookManager.get().removeShutdownHook(routerShutdownHook); + } + routerShutdownHook = new CompositeServiceShutdownHook(router); + ShutdownHookManager.get().addShutdownHook(routerShutdownHook, SHUTDOWN_HOOK_PRIORITY); + router.init(conf); + router.start(); } - - routerShutdownHook = new CompositeServiceShutdownHook(router); - ShutdownHookManager.get().addShutdownHook(routerShutdownHook, - SHUTDOWN_HOOK_PRIORITY); - - router.init(conf); - router.start(); } catch (Throwable t) { LOG.error("Error starting Router", t); System.exit(-1); @@ -348,4 +361,9 @@ public static long getClusterTimeStamp() { public FedAppReportFetcher getFetcher() { return fetcher; } + + private static void printUsage(PrintStream out) { + out.println("Usage: yarn router [-format-state-store] | " + + "[-remove-application-from-state-store ]"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java index 5fb5e045e6630..f51ac6d2b8394 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.router; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.apache.hadoop.conf.Configuration; @@ -43,7 +44,9 @@ import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; import java.io.PrintWriter; import java.util.Collection; import java.util.HashMap; @@ -390,4 +393,15 @@ public Locale getLocale() { } } + @Test + public void testRouterCLI() { + ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); + ByteArrayOutputStream dataErr = new ByteArrayOutputStream(); + System.setOut(new PrintStream(dataOut)); + System.setErr(new PrintStream(dataErr)); + Router.main(new String[]{"-help", "-format-state-store"}); + assertTrue(dataErr.toString().contains( + "Usage: yarn router [-format-state-store] | " + + "[-remove-application-from-state-store ]")); + } } From 56b928b86f735ad97195d6f575037af3cdfff8be Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 13 Sep 2023 18:10:24 +0100 Subject: [PATCH 025/155] YARN-11498. Add exclusion for jettison everywhere jersey-json is loaded (#5786) All uses of jersey-json in the yarn and other hadoop modules now exclude the obsolete org.codehaus.jettison/jettison and so avoid all security issues which can come from the library. Contributed by PJ Fanning --- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++++ hadoop-common-project/hadoop-common/pom.xml | 4 ++++ hadoop-project/pom.xml | 4 ++++ hadoop-tools/hadoop-resourceestimator/pom.xml | 4 ++++ .../hadoop-yarn-applications-catalog-webapp/pom.xml | 4 ++++ .../hadoop-yarn-server-applicationhistoryservice/pom.xml | 4 ++++ .../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 4 ++++ .../hadoop-yarn-server-resourcemanager/pom.xml | 4 ++++ 8 files changed, 32 insertions(+) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 208345d5f5a53..c772a6ed684dd 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -443,6 +443,10 @@ javax.xml.bind jaxb-api + + org.codehaus.jettison + jettison + diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 41707f5002414..207f1f5351ee9 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -173,6 +173,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index cba562dd14036..08fe5af0eee69 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -949,6 +949,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml index 89e248f9a7efe..a6f6c691b36f6 100644 --- a/hadoop-tools/hadoop-resourceestimator/pom.xml +++ b/hadoop-tools/hadoop-resourceestimator/pom.xml @@ -94,6 +94,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index 3acd9ce0ea888..1a2c37faad4cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -123,6 +123,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index d0fd79aaa78e1..b55dd30a17182 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -117,6 +117,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index a66b666402094..3b5c373f50c33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -166,6 +166,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index d36ca02d7d099..71981e02e7be0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -122,6 +122,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.codehaus.jettison + jettison + From 8538af4638bdd4120c2aa0c0e2803a085e5ced74 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 14 Sep 2023 21:28:49 +0800 Subject: [PATCH 026/155] YARN-7599. [BackPort][GPG] ApplicationCleaner in Global Policy Generator. (#5934) Contributed by Botong Huang, Shilun Fan. Co-authored-by: Botong Huang Co-authored-by: slfan1989 Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../hadoop/yarn/conf/YarnConfiguration.java | 25 +++ .../src/main/resources/yarn-default.xml | 28 ++++ .../utils/FederationStateStoreFacade.java | 30 ++++ .../globalpolicygenerator/GPGUtils.java | 24 ++- .../GlobalPolicyGenerator.java | 22 ++- .../ApplicationCleaner.java | 153 ++++++++++++++++++ .../DefaultApplicationCleaner.java | 77 +++++++++ .../applicationcleaner/package-info.java | 19 +++ .../policygenerator/PolicyGenerator.java | 4 +- .../TestDefaultApplicationCleaner.java | 131 +++++++++++++++ .../policygenerator/TestPolicyGenerator.java | 2 +- 11 files changed, 510 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/package-info.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 874ee9d08d9ad..ef06299fcfd8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4432,6 +4432,31 @@ public static boolean isAclEnabled(Configuration conf) { public static final String GPG_KERBEROS_PRINCIPAL_HOSTNAME_KEY = FEDERATION_GPG_PREFIX + "kerberos.principal.hostname"; + // The application cleaner class to use + public static final String GPG_APPCLEANER_CLASS = + FEDERATION_GPG_PREFIX + "application.cleaner.class"; + public static final String DEFAULT_GPG_APPCLEANER_CLASS = + "org.apache.hadoop.yarn.server.globalpolicygenerator" + + ".applicationcleaner.DefaultApplicationCleaner"; + + // The interval at which the application cleaner runs, -1 means disabled + public static final String GPG_APPCLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "application.cleaner.interval-ms"; + public static final long DEFAULT_GPG_APPCLEANER_INTERVAL_MS = TimeUnit.SECONDS.toMillis(-1); + + /** + * Specifications on how (many times) to contact Router for apps. We need to + * do this because Router might return partial application list because some + * sub-cluster RM is not responsive (e.g. failing over). + * + * Should have three values separated by comma: minimal success retries, + * maximum total retry, retry interval (ms). + */ + public static final String GPG_APPCLEANER_CONTACT_ROUTER_SPEC = + FEDERATION_GPG_PREFIX + "application.cleaner.contact.router.spec"; + public static final String DEFAULT_GPG_APPCLEANER_CONTACT_ROUTER_SPEC = + "3,10,600000"; + public static final String FEDERATION_GPG_POLICY_PREFIX = FEDERATION_GPG_PREFIX + "policy.generator."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f82540b8f46cb..9697f7aa88c8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5538,6 +5538,14 @@ LINEAR + + + The Application Cleaner implementation class for GPG to use. + + yarn.federation.gpg.application.cleaner.class + org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.DefaultApplicationCleaner + + Flag to enable cross-origin (CORS) support in the GPG. This flag requires the CORS filter initializer to be added to the filter initializers @@ -5546,6 +5554,14 @@ false + + + The interval at which the application cleaner runs, -1 means disabled. + + yarn.federation.gpg.application.cleaner.interval-ms + -1s + + The http address of the GPG web application. @@ -5556,6 +5572,18 @@ 0.0.0.0:8069 + + + Specifications on how (many times) to contact Router for apps. We need to + do this because Router might return partial application list because some + sub-cluster RM is not responsive (e.g. failing over). + Should have three values separated by comma: minimal success retries, + maximum total retry, retry interval (ms). + + yarn.federation.gpg.application.cleaner.contact.router.spec + 3,10,600000 + + The https address of the GPG web application. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 26136b11de6f4..d4c259b51605e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -83,6 +83,9 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.slf4j.Logger; @@ -884,6 +887,33 @@ public void addApplicationHomeSubCluster(ApplicationId applicationId, } } + /** + * Get the {@code ApplicationHomeSubCluster} list representing the mapping of + * all submitted applications to it's home sub-cluster. + * + * @return the mapping of all submitted application to it's home sub-cluster + * @throws YarnException if the request is invalid/fails + */ + public List getApplicationsHomeSubCluster() throws YarnException { + GetApplicationsHomeSubClusterResponse response = stateStore.getApplicationsHomeSubCluster( + GetApplicationsHomeSubClusterRequest.newInstance()); + return response.getAppsHomeSubClusters(); + } + + /** + * Delete the mapping of home {@code SubClusterId} of a previously submitted + * {@code ApplicationId}. Currently response is empty if the operation is + * successful, if not an exception reporting reason for a failure. + * + * @param applicationId the application to delete the home sub-cluster of + * @throws YarnException if the request is invalid/fails + */ + public void deleteApplicationHomeSubCluster(ApplicationId applicationId) + throws YarnException { + stateStore.deleteApplicationHomeSubCluster( + DeleteApplicationHomeSubClusterRequest.newInstance(applicationId)); + } + /** * Update ApplicationHomeSubCluster to FederationStateStore. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index a802e37979bb7..02344a51493c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -40,6 +40,7 @@ import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; /** * GPGUtils contains utility functions for the GPG. @@ -58,11 +59,12 @@ private GPGUtils() { * @param webAddr WebAddress. * @param path url path. * @param returnType return type. + * @param selectParam query parameters. * @param conf configuration. * @return response entity. */ public static T invokeRMWebService(String webAddr, String path, final Class returnType, - Configuration conf) { + Configuration conf, String selectParam) { Client client = Client.create(); T obj; @@ -72,6 +74,11 @@ public static T invokeRMWebService(String webAddr, String path, final Class< String scheme = YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX; String webAddress = scheme + socketAddress.getHostName() + ":" + socketAddress.getPort(); WebResource webResource = client.resource(webAddress); + + if (selectParam != null) { + webResource = webResource.queryParam(RMWSConsts.DESELECTS, selectParam); + } + ClientResponse response = null; try { response = webResource.path(RM_WEB_SERVICE_PATH).path(path) @@ -92,6 +99,21 @@ public static T invokeRMWebService(String webAddr, String path, final Class< } } + /** + * Performs an invocation of the remote RMWebService. + * + * @param Generic T. + * @param webAddr WebAddress. + * @param path url path. + * @param returnType return type. + * @param config configuration. + * @return response entity. + */ + public static T invokeRMWebService(String webAddr, + String path, final Class returnType, Configuration config) { + return invokeRMWebService(webAddr, path, returnType, config, null); + } + /** * Creates a uniform weighting of 1.0 for each sub cluster. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index 81a999d76a28e..ba8ce856cdaa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.ApplicationCleaner; import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator; import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner; import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp; @@ -84,6 +85,7 @@ public class GlobalPolicyGenerator extends CompositeService { // Scheduler service that runs tasks periodically private ScheduledThreadPoolExecutor scheduledExecutorService; private SubClusterCleaner subClusterCleaner; + private ApplicationCleaner applicationCleaner; private PolicyGenerator policyGenerator; private String webAppAddress; private JvmPauseMonitor pauseMonitor; @@ -125,6 +127,12 @@ protected void serviceInit(Configuration conf) throws Exception { conf.getInt(YarnConfiguration.GPG_SCHEDULED_EXECUTOR_THREADS, YarnConfiguration.DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS)); this.subClusterCleaner = new SubClusterCleaner(conf, this.gpgContext); + + this.applicationCleaner = FederationStateStoreFacade.createInstance(conf, + YarnConfiguration.GPG_APPCLEANER_CLASS, + YarnConfiguration.DEFAULT_GPG_APPCLEANER_CLASS, ApplicationCleaner.class); + this.applicationCleaner.init(conf, this.gpgContext); + this.policyGenerator = new PolicyGenerator(conf, this.gpgContext); this.webAppAddress = WebAppUtils.getGPGWebAppURLWithoutScheme(conf); @@ -149,7 +157,7 @@ protected void serviceStart() throws Exception { super.serviceStart(); - // Scheduler SubClusterCleaner service + // Schedule SubClusterCleaner service Configuration config = getConfig(); long scCleanerIntervalMs = config.getTimeDuration( YarnConfiguration.GPG_SUBCLUSTER_CLEANER_INTERVAL_MS, @@ -161,6 +169,18 @@ protected void serviceStart() throws Exception { DurationFormatUtils.formatDurationISO(scCleanerIntervalMs)); } + // Schedule ApplicationCleaner service + long appCleanerIntervalMs = config.getTimeDuration( + YarnConfiguration.GPG_APPCLEANER_INTERVAL_MS, + YarnConfiguration.DEFAULT_GPG_APPCLEANER_INTERVAL_MS, TimeUnit.MILLISECONDS); + + if (appCleanerIntervalMs > 0) { + this.scheduledExecutorService.scheduleAtFixedRate(this.applicationCleaner, + 0, appCleanerIntervalMs, TimeUnit.MILLISECONDS); + LOG.info("Scheduled application cleaner with interval: {}", + DurationFormatUtils.formatDurationISO(appCleanerIntervalMs)); + } + // Schedule PolicyGenerator // We recommend using yarn.federation.gpg.policy.generator.interval // instead of yarn.federation.gpg.policy.generator.interval-ms diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java new file mode 100644 index 0000000000000..cd3f7618558e9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.lang3.time.DurationFormatUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The ApplicationCleaner is a runnable that cleans up old applications from + * table applicationsHomeSubCluster in FederationStateStore. + */ +public abstract class ApplicationCleaner implements Runnable { + private static final Logger LOG = + LoggerFactory.getLogger(ApplicationCleaner.class); + + private Configuration conf; + private GPGContext gpgContext; + + private int minRouterSuccessCount; + private int maxRouterRetry; + private long routerQueryIntevalMillis; + + public void init(Configuration config, GPGContext context) + throws YarnException { + + this.gpgContext = context; + this.conf = config; + + String routerSpecString = + this.conf.get(YarnConfiguration.GPG_APPCLEANER_CONTACT_ROUTER_SPEC, + YarnConfiguration.DEFAULT_GPG_APPCLEANER_CONTACT_ROUTER_SPEC); + String[] specs = routerSpecString.split(","); + if (specs.length != 3) { + throw new YarnException("Expect three comma separated values in " + + YarnConfiguration.GPG_APPCLEANER_CONTACT_ROUTER_SPEC + " but get " + + routerSpecString); + } + this.minRouterSuccessCount = Integer.parseInt(specs[0]); + this.maxRouterRetry = Integer.parseInt(specs[1]); + this.routerQueryIntevalMillis = Long.parseLong(specs[2]); + + if (this.minRouterSuccessCount > this.maxRouterRetry) { + throw new YarnException("minRouterSuccessCount " + + this.minRouterSuccessCount + + " should not be larger than maxRouterRetry" + this.maxRouterRetry); + } + if (this.minRouterSuccessCount <= 0) { + throw new YarnException("minRouterSuccessCount " + + this.minRouterSuccessCount + " should be positive"); + } + + LOG.info( + "Initialized AppCleaner with Router query with min success {}, " + + "max retry {}, retry interval {}", + this.minRouterSuccessCount, this.maxRouterRetry, + DurationFormatUtils.formatDurationISO(this.routerQueryIntevalMillis)); + } + + public GPGContext getGPGContext() { + return this.gpgContext; + } + + /** + * Query router for applications. + * + * @return the set of applications + * @throws YarnRuntimeException when router call fails + */ + public Set getAppsFromRouter() throws YarnRuntimeException { + String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); + + LOG.info(String.format("Contacting router at: %s", webAppAddress)); + AppsInfo appsInfo = GPGUtils.invokeRMWebService(webAppAddress, "apps", AppsInfo.class, conf, + DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString()); + + Set appSet = new HashSet<>(); + for (AppInfo appInfo : appsInfo.getApps()) { + appSet.add(ApplicationId.fromString(appInfo.getAppId())); + } + return appSet; + } + + /** + * Get the list of known applications in the cluster from Router. + * + * @return the list of known applications + * @throws YarnException if get app fails + */ + public Set getRouterKnownApplications() throws YarnException { + int successCount = 0, totalAttemptCount = 0; + Set resultSet = new HashSet<>(); + while (totalAttemptCount < this.maxRouterRetry) { + try { + Set routerApps = getAppsFromRouter(); + resultSet.addAll(routerApps); + LOG.info("Attempt {}: {} known apps from Router, {} in total", + totalAttemptCount, routerApps.size(), resultSet.size()); + + successCount++; + if (successCount >= this.minRouterSuccessCount) { + return resultSet; + } + + // Wait for the next attempt + try { + Thread.sleep(this.routerQueryIntevalMillis); + } catch (InterruptedException e) { + LOG.warn("Sleep interrupted after attempt {}.", totalAttemptCount); + } + } catch (Exception e) { + LOG.warn("Router query attempt {} failed.", totalAttemptCount, e); + } finally { + totalAttemptCount++; + } + } + throw new YarnException("Only " + successCount + + " success Router queries after " + totalAttemptCount + " retries"); + } + + @Override + public abstract void run(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java new file mode 100644 index 0000000000000..857d2e645d4c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner; + +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * The default ApplicationCleaner that cleans up old applications from table + * applicationsHomeSubCluster in FederationStateStore. + */ +public class DefaultApplicationCleaner extends ApplicationCleaner { + private static final Logger LOG = + LoggerFactory.getLogger(DefaultApplicationCleaner.class); + + @Override + public void run() { + Date now = new Date(); + LOG.info("Application cleaner run at time {}", now); + + FederationStateStoreFacade facade = getGPGContext().getStateStoreFacade(); + Set candidates = new HashSet<>(); + try { + List response = + facade.getApplicationsHomeSubCluster(); + for (ApplicationHomeSubCluster app : response) { + candidates.add(app.getApplicationId()); + } + LOG.info("{} app entries in FederationStateStore", candidates.size()); + + Set routerApps = getRouterKnownApplications(); + LOG.info("{} known applications from Router", routerApps.size()); + + candidates.removeAll(routerApps); + LOG.info("Deleting {} applications from statestore", candidates.size()); + if (LOG.isDebugEnabled()) { + LOG.debug("Apps to delete: {}.", candidates.stream().map(Object::toString) + .collect(Collectors.joining(","))); + } + for (ApplicationId appId : candidates) { + try { + facade.deleteApplicationHomeSubCluster(appId); + } catch (Exception e) { + LOG.error("deleteApplicationHomeSubCluster failed at application {}.", appId, e); + } + } + } catch (Throwable e) { + LOG.error("Application cleaner started at time {} fails. ", now, e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/package-info.java new file mode 100644 index 0000000000000..dd302c81f45ea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java index df28192a0c668..1f0fbd11a741c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java @@ -159,7 +159,7 @@ protected Map> getInfos( clusterInfo.put(sci.getSubClusterId(), new HashMap<>()); } Object ret = GPGUtils.invokeRMWebService(sci.getRMWebServiceAddress(), - e.getValue(), e.getKey(), getConf()); + e.getValue(), e.getKey(), conf); clusterInfo.get(sci.getSubClusterId()).put(e.getKey(), ret); } } @@ -181,7 +181,7 @@ protected Map getSchedulerInfo( for (SubClusterInfo sci : activeSubClusters.values()) { SchedulerTypeInfo sti = GPGUtils .invokeRMWebService(sci.getRMWebServiceAddress(), - RMWSConsts.SCHEDULER, SchedulerTypeInfo.class, getConf()); + RMWSConsts.SCHEDULER, SchedulerTypeInfo.class, conf); if(sti != null){ schedInfo.put(sci.getSubClusterId(), sti.getSchedulerInfo()); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java new file mode 100644 index 0000000000000..2d63c48236fb5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext; +import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContextImpl; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit test for DefaultApplicationCleaner in GPG. + */ +public class TestDefaultApplicationCleaner { + private Configuration conf; + private MemoryFederationStateStore stateStore; + private FederationStateStoreFacade facade; + private ApplicationCleaner appCleaner; + private GPGContext gpgContext; + + private List appIds; + // The list of applications returned by mocked router + private Set routerAppIds; + + @Before + public void setup() throws Exception { + conf = new YarnConfiguration(); + + // No Router query retry + conf.set(YarnConfiguration.GPG_APPCLEANER_CONTACT_ROUTER_SPEC, "1,1,0"); + + stateStore = new MemoryFederationStateStore(); + stateStore.init(conf); + + facade = FederationStateStoreFacade.getInstance(); + facade.reinitialize(stateStore, conf); + + gpgContext = new GPGContextImpl(); + gpgContext.setStateStoreFacade(facade); + + appCleaner = new TestableDefaultApplicationCleaner(); + appCleaner.init(conf, gpgContext); + + routerAppIds = new HashSet<>(); + + appIds = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + ApplicationId appId = ApplicationId.newInstance(0, i); + appIds.add(appId); + + SubClusterId subClusterId = + SubClusterId.newInstance("SUBCLUSTER-" + i); + + stateStore.addApplicationHomeSubCluster( + AddApplicationHomeSubClusterRequest.newInstance( + ApplicationHomeSubCluster.newInstance(appId, subClusterId))); + } + } + + @After + public void breakDown() { + if (stateStore != null) { + stateStore.close(); + stateStore = null; + } + } + + @Test + public void testFederationStateStoreAppsCleanUp() throws YarnException { + // Set first app to be still known by Router + ApplicationId appId = appIds.get(0); + routerAppIds.add(appId); + + // Another random app not in stateStore known by Router + appId = ApplicationId.newInstance(100, 200); + routerAppIds.add(appId); + + appCleaner.run(); + + // Only one app should be left + Assert.assertEquals(1, + stateStore + .getApplicationsHomeSubCluster( + GetApplicationsHomeSubClusterRequest.newInstance()) + .getAppsHomeSubClusters().size()); + } + + /** + * Testable version of DefaultApplicationCleaner. + */ + public class TestableDefaultApplicationCleaner + extends DefaultApplicationCleaner { + @Override + public Set getAppsFromRouter() throws YarnRuntimeException { + return routerAppIds; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java index 72e97f8a75087..446eeee2cd922 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java @@ -299,7 +299,7 @@ public void testCallRM() { String webAppAddress = getServiceAddress(NetUtils.createSocketAddr(rmAddress)); SchedulerTypeInfo sti = GPGUtils.invokeRMWebService(webAppAddress, RMWSConsts.SCHEDULER, - SchedulerTypeInfo.class, this.conf); + SchedulerTypeInfo.class, conf); Assert.assertNotNull(sti); SchedulerInfo schedulerInfo = sti.getSchedulerInfo(); From dea446419f8f93f8f4870e2e1be5db148361ed9a Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 14 Sep 2023 17:49:12 +0100 Subject: [PATCH 027/155] HADOOP-18895. Upgrade to commons-compress 1.24.0 (#6062) Contributed by PJ Fanning --- LICENSE-binary | 2 +- hadoop-project/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index b20d382f7bd18..e0c5923d043c4 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -301,7 +301,7 @@ net.java.dev.jna:jna:5.2.0 net.minidev:accessors-smart:1.2 org.apache.avro:avro:1.9.2 org.apache.commons:commons-collections4:4.2 -org.apache.commons:commons-compress:1.21 +org.apache.commons:commons-compress:1.24.0 org.apache.commons:commons-configuration2:2.8.0 org.apache.commons:commons-csv:1.9.0 org.apache.commons:commons-digester:1.8.1 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 08fe5af0eee69..479a2183ade67 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -116,7 +116,7 @@ 1.5.0 1.15 3.2.2 - 1.21 + 1.24.0 1.9.0 2.11.0 3.12.0 From 23360b3f6bed774a0c0007069c21201ce2c66d6e Mon Sep 17 00:00:00 2001 From: ConfX <114765570+teamconfx@users.noreply.github.com> Date: Thu, 14 Sep 2023 18:53:31 -0400 Subject: [PATCH 028/155] HADOOP-18824. ZKDelegationTokenSecretManager causes ArithmeticException due to improper numRetries value checking (#6052) --- .../delegation/ZKDelegationTokenSecretManager.java | 2 +- .../TestZKDelegationTokenSecretManager.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index fb9a2951f598a..34642ccdadda8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -222,7 +222,7 @@ public ZKDelegationTokenSecretManager(Configuration conf) { ZK_DTSM_ZK_CONNECTION_TIMEOUT_DEFAULT) ) .retryPolicy( - new RetryNTimes(numRetries, sessionT / numRetries)); + new RetryNTimes(numRetries, numRetries == 0 ? 0 : sessionT / numRetries)); } catch (Exception ex) { throw new RuntimeException("Could not Load ZK acls or auth: " + ex, ex); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java index e92a25ea0ed8f..469d87ab30c6f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java @@ -106,10 +106,22 @@ protected Configuration getSecretConf(String connectString) { @SuppressWarnings("unchecked") @Test public void testMultiNodeOperations() throws Exception { + testMultiNodeOperationsImpl(false); + } + + @Test + public void testMultiNodeOperationsWithZeroRetry() throws Exception { + testMultiNodeOperationsImpl(true); + } + + public void testMultiNodeOperationsImpl(boolean setZeroRetry) throws Exception { for (int i = 0; i < TEST_RETRIES; i++) { DelegationTokenManager tm1, tm2 = null; String connectString = zkServer.getConnectString(); Configuration conf = getSecretConf(connectString); + if (setZeroRetry) { + conf.setInt(ZKDelegationTokenSecretManager.ZK_DTSM_ZK_NUM_RETRIES, 0); + } tm1 = new DelegationTokenManager(conf, new Text("bla")); tm1.init(); tm2 = new DelegationTokenManager(conf, new Text("bla")); From e283375cdfba409fe4ba948c0f24ed073dcbb383 Mon Sep 17 00:00:00 2001 From: Vikas Kumar Date: Fri, 15 Sep 2023 10:02:47 +0530 Subject: [PATCH 029/155] HADOOP-18851: Performance improvement for DelegationTokenSecretManager. (#6001). Contributed by Vikas Kumar. Signed-off-by: Wei-Chiu Chuang Signed-off-by: He Xiaoqiao --- .../AbstractDelegationTokenSecretManager.java | 30 +++++++------- .../ZKDelegationTokenSecretManager.java | 40 +++++++++++-------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index 283e773c81795..cafa5135e68e6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -115,12 +115,12 @@ private String formatTokenId(TokenIdent id) { /** * Access to currentKey is protected by this object lock */ - private DelegationKey currentKey; + private volatile DelegationKey currentKey; - private long keyUpdateInterval; - private long tokenMaxLifetime; - private long tokenRemoverScanInterval; - private long tokenRenewInterval; + private final long keyUpdateInterval; + private final long tokenMaxLifetime; + private final long tokenRemoverScanInterval; + private final long tokenRenewInterval; /** * Whether to store a token's tracking ID in its TokenInformation. * Can be overridden by a subclass. @@ -486,17 +486,18 @@ private synchronized void removeExpiredKeys() { } @Override - protected synchronized byte[] createPassword(TokenIdent identifier) { + protected byte[] createPassword(TokenIdent identifier) { int sequenceNum; long now = Time.now(); sequenceNum = incrementDelegationTokenSeqNum(); identifier.setIssueDate(now); identifier.setMaxDate(now + tokenMaxLifetime); - identifier.setMasterKeyId(currentKey.getKeyId()); + DelegationKey delegationCurrentKey = currentKey; + identifier.setMasterKeyId(delegationCurrentKey.getKeyId()); identifier.setSequenceNumber(sequenceNum); LOG.info("Creating password for identifier: " + formatTokenId(identifier) - + ", currentKey: " + currentKey.getKeyId()); - byte[] password = createPassword(identifier.getBytes(), currentKey.getKey()); + + ", currentKey: " + delegationCurrentKey.getKeyId()); + byte[] password = createPassword(identifier.getBytes(), delegationCurrentKey.getKey()); DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now + tokenRenewInterval, password, getTrackingIdIfEnabled(identifier)); try { @@ -521,7 +522,6 @@ protected synchronized byte[] createPassword(TokenIdent identifier) { */ protected DelegationTokenInformation checkToken(TokenIdent identifier) throws InvalidToken { - assert Thread.holdsLock(this); DelegationTokenInformation info = getTokenInfo(identifier); String err; if (info == null) { @@ -541,7 +541,7 @@ protected DelegationTokenInformation checkToken(TokenIdent identifier) } @Override - public synchronized byte[] retrievePassword(TokenIdent identifier) + public byte[] retrievePassword(TokenIdent identifier) throws InvalidToken { return checkToken(identifier).getPassword(); } @@ -553,7 +553,7 @@ protected String getTrackingIdIfEnabled(TokenIdent ident) { return null; } - public synchronized String getTokenTrackingId(TokenIdent identifier) { + public String getTokenTrackingId(TokenIdent identifier) { DelegationTokenInformation info = getTokenInfo(identifier); if (info == null) { return null; @@ -567,7 +567,7 @@ public synchronized String getTokenTrackingId(TokenIdent identifier) { * @param password Password in the token. * @throws InvalidToken InvalidToken. */ - public synchronized void verifyToken(TokenIdent identifier, byte[] password) + public void verifyToken(TokenIdent identifier, byte[] password) throws InvalidToken { byte[] storedPassword = retrievePassword(identifier); if (!MessageDigest.isEqual(password, storedPassword)) { @@ -584,7 +584,7 @@ public synchronized void verifyToken(TokenIdent identifier, byte[] password) * @throws InvalidToken if the token is invalid * @throws AccessControlException if the user can't renew token */ - public synchronized long renewToken(Token token, + public long renewToken(Token token, String renewer) throws InvalidToken, IOException { ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DataInputStream in = new DataInputStream(buf); @@ -646,7 +646,7 @@ public synchronized long renewToken(Token token, * @throws InvalidToken for invalid token * @throws AccessControlException if the user isn't allowed to cancel */ - public synchronized TokenIdent cancelToken(Token token, + public TokenIdent cancelToken(Token token, String canceller) throws IOException { ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DataInputStream in = new DataInputStream(buf); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index 34642ccdadda8..da233c33fa73d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Stream; import org.apache.curator.ensemble.fixed.FixedEnsembleProvider; @@ -148,7 +149,7 @@ protected static CuratorFramework getCurator() { private final int seqNumBatchSize; private int currentSeqNum; private int currentMaxSeqNum; - + private final ReentrantLock currentSeqNumLock; private final boolean isTokenWatcherEnabled; public ZKDelegationTokenSecretManager(Configuration conf) { @@ -164,6 +165,7 @@ public ZKDelegationTokenSecretManager(Configuration conf) { ZK_DTSM_TOKEN_SEQNUM_BATCH_SIZE_DEFAULT); isTokenWatcherEnabled = conf.getBoolean(ZK_DTSM_TOKEN_WATCHER_ENABLED, ZK_DTSM_TOKEN_WATCHER_ENABLED_DEFAULT); + this.currentSeqNumLock = new ReentrantLock(true); if (CURATOR_TL.get() != null) { zkClient = CURATOR_TL.get().usingNamespace( @@ -520,24 +522,28 @@ protected int incrementDelegationTokenSeqNum() { // The secret manager will keep a local range of seq num which won't be // seen by peers, so only when the range is exhausted it will ask zk for // another range again - if (currentSeqNum >= currentMaxSeqNum) { - try { - // after a successful batch request, we can get the range starting point - currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize); - currentMaxSeqNum = currentSeqNum + seqNumBatchSize; - LOG.info("Fetched new range of seq num, from {} to {} ", - currentSeqNum+1, currentMaxSeqNum); - } catch (InterruptedException e) { - // The ExpirationThread is just finishing.. so dont do anything.. - LOG.debug( - "Thread interrupted while performing token counter increment", e); - Thread.currentThread().interrupt(); - } catch (Exception e) { - throw new RuntimeException("Could not increment shared counter !!", e); + try { + this.currentSeqNumLock.lock(); + if (currentSeqNum >= currentMaxSeqNum) { + try { + // after a successful batch request, we can get the range starting point + currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize); + currentMaxSeqNum = currentSeqNum + seqNumBatchSize; + LOG.info("Fetched new range of seq num, from {} to {} ", + currentSeqNum+1, currentMaxSeqNum); + } catch (InterruptedException e) { + // The ExpirationThread is just finishing.. so dont do anything.. + LOG.debug( + "Thread interrupted while performing token counter increment", e); + Thread.currentThread().interrupt(); + } catch (Exception e) { + throw new RuntimeException("Could not increment shared counter !!", e); + } } + return ++currentSeqNum; + } finally { + this.currentSeqNumLock.unlock(); } - - return ++currentSeqNum; } @Override From ab2bc90e090564b7466ab1b759fbb849612ae82b Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Fri, 15 Sep 2023 20:50:11 +0800 Subject: [PATCH 030/155] MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0. (#6042) Contributed by Chenyu Zheng. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../src/main/java/org/apache/hadoop/mapred/TaskLog.java | 3 ++- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 6 ++++-- .../src/main/resources/mapred-default.xml | 7 +++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index a0223dedd64ce..fdfa3e7710f47 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -466,7 +466,8 @@ public static long getTaskLogLength(JobConf conf) { } public static long getTaskLogLimitBytes(Configuration conf) { - return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; + return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) * + 1024; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 8ec984e777bb6..24689e09daf13 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -410,6 +410,8 @@ public interface MRJobConfig { public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; + public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240; + public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; @@ -758,11 +760,11 @@ public interface MRJobConfig { public static final String MR_AM_LOG_KB = MR_AM_PREFIX + "container.log.limit.kb"; - public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll + public static final int DEFAULT_MR_AM_LOG_KB = 10240; public static final String MR_AM_LOG_BACKUPS = MR_AM_PREFIX + "container.log.backups"; - public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; + public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll /**The number of splits when reporting progress in MR*/ public static final String MR_AM_NUM_PROGRESS_SPLITS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a6d68acda34ab..493e4c3a37e94 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -823,16 +823,15 @@ mapreduce.task.userlog.limit.kb - 0 - The maximum size of user-logs of each task in KB. 0 disables the cap. + 10240 + The maximum size of user-logs of each task in KB. yarn.app.mapreduce.am.container.log.limit.kb - 0 + 10240 The maximum size of the MRAppMaster attempt container logs in KB. - 0 disables the cap. From 510a7dcae0d6e04aed8cc15d1783db113ea287eb Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Fri, 15 Sep 2023 21:01:24 +0800 Subject: [PATCH 031/155] HDFS-17180. HttpFS Add Support getTrashRoots API (#6028) Contributed by Hualong Zhang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../fs/http/client/HttpFSFileSystem.java | 18 ++++++++ .../hadoop/fs/http/server/FSOperations.java | 28 ++++++++++++ .../http/server/HttpFSParametersProvider.java | 19 ++++++++ .../hadoop/fs/http/server/HttpFSServer.java | 9 ++++ .../server/metrics/HttpFSServerMetrics.java | 5 +++ .../fs/http/client/BaseTestHttpFSWith.java | 44 ++++++++++++++++++- 6 files changed, 122 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 09fb493a067bd..65b49cc9cf1ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -148,6 +148,7 @@ public class HttpFSFileSystem extends FileSystem public static final String EC_POLICY_NAME_PARAM = "ecpolicy"; public static final String OFFSET_PARAM = "offset"; public static final String LENGTH_PARAM = "length"; + public static final String ALLUSERS_PARAM = "allusers"; public static final Short DEFAULT_PERMISSION = 0755; public static final String ACLSPEC_DEFAULT = ""; @@ -287,6 +288,7 @@ public enum Operation { GETSTATUS(HTTP_GET), GETECPOLICIES(HTTP_GET), GETECCODECS(HTTP_GET), + GETTRASHROOTS(HTTP_GET), GET_BLOCK_LOCATIONS(HTTP_GET); private String httpMethod; @@ -1798,6 +1800,22 @@ public Map getAllErasureCodingCodecs() throws IOException { return JsonUtilClient.getErasureCodeCodecs(json); } + public Collection getTrashRoots(boolean allUsers) { + Map params = new HashMap(); + params.put(OP_PARAM, Operation.GETTRASHROOTS.toString()); + params.put(ALLUSERS_PARAM, Boolean.toString(allUsers)); + Path path = new Path(getUri().toString(), "/"); + try { + HttpURLConnection conn = getConnection(Operation.GETTRASHROOTS.getMethod(), + params, path, true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + return JsonUtilClient.getTrashRoots(json); + } catch (IOException e) { + return super.getTrashRoots(allUsers); + } + } + @VisibleForTesting static BlockLocation[] toBlockLocations(JSONObject json) throws IOException { ObjectMapper mapper = new ObjectMapper(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 7261504820c00..348b0bd1ce1d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -2396,4 +2396,32 @@ public Map execute(FileSystem fs) throws IOException { return ecCodecs; } } + + /** + * Executor that performs a FSGetTrashRoots operation. + */ + @InterfaceAudience.Private + public static class FSGetTrashRoots + implements FileSystemAccess.FileSystemExecutor { + final private boolean allUsers; + + public FSGetTrashRoots(boolean allUsers) { + this.allUsers = allUsers; + } + + @Override + public Map execute(FileSystem fs) throws IOException { + Map> paths = new HashMap<>(); + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = (DistributedFileSystem) fs; + paths.put("Paths", dfs.getTrashRoots(allUsers)); + } else { + throw new UnsupportedOperationException("getTrashRoots is " + + "not supported for HttpFs on " + fs.getClass() + + ". Please check your fs.defaultFS configuration"); + } + HttpFSServerWebApp.get().getMetrics().incrOpsTrashRoots(); + return paths; + } + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index 87f2d4d3dab5f..3f1058da2c960 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -132,6 +132,7 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.GETSTATUS, new Class[]{}); PARAMS_DEF.put(Operation.GETECPOLICIES, new Class[]{}); PARAMS_DEF.put(Operation.GETECCODECS, new Class[]{}); + PARAMS_DEF.put(Operation.GETTRASHROOTS, new Class[]{AllUsersParam.class}); PARAMS_DEF.put(Operation.GET_BLOCK_LOCATIONS, new Class[] {OffsetParam.class, LenParam.class}); } @@ -765,4 +766,22 @@ public ECPolicyParam() { super(NAME, null); } } + + /** + * Class for allusers parameter. + */ + @InterfaceAudience.Private + public static class AllUsersParam extends BooleanParam { + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.ALLUSERS_PARAM; + + /** + * Constructor. + */ + public AllUsersParam() { + super(NAME, false); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index c9276aae952a8..57a79a184741c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -55,6 +55,7 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrNameParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrSetFlagParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.XAttrValueParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AllUsersParam; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.web.JsonUtil; import org.apache.hadoop.http.JettyUtils; @@ -576,6 +577,14 @@ public InputStream run() throws Exception { response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } + case GETTRASHROOTS: { + Boolean allUsers = params.get(AllUsersParam.NAME, AllUsersParam.class); + FSOperations.FSGetTrashRoots command = new FSOperations.FSGetTrashRoots(allUsers); + Map json = fsExecute(user, command); + AUDIT_LOG.info("allUsers [{}]", allUsers); + response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); + break; + } default: { throw new IOException( MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value())); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java index 6e01c5d27985c..8314cd7353f29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java @@ -67,6 +67,7 @@ public class HttpFSServerMetrics { private @Metric MutableCounterLong opsStatus; private @Metric MutableCounterLong opsAllECPolicies; private @Metric MutableCounterLong opsECCodecs; + private @Metric MutableCounterLong opsTrashRoots; private final MetricsRegistry registry = new MetricsRegistry("httpfsserver"); private final String name; @@ -175,4 +176,8 @@ public void incrOpsAllECPolicies() { public void incrOpsECCodecs() { opsECCodecs.incr(); } + + public void incrOpsTrashRoots() { + opsTrashRoots.incr(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 0283d1d4adcc1..4c1d2d176b56f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -1219,7 +1219,7 @@ protected enum Operation { FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST, GET_SNAPSHOT_LIST, GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY, SATISFYSTORAGEPOLICY, GET_SNAPSHOT_DIFF_LISTING, GETFILEBLOCKLOCATIONS, - GETFILELINKSTATUS, GETSTATUS, GETECPOLICIES, GETECCODECS + GETFILELINKSTATUS, GETSTATUS, GETECPOLICIES, GETECCODECS, GETTRASHROOTS } @SuppressWarnings("methodlength") private void operation(Operation op) throws Exception { @@ -1374,6 +1374,9 @@ private void operation(Operation op) throws Exception { case GETECCODECS: testGetECCodecs(); break; + case GETTRASHROOTS: + testGetTrashRoots(); + break; } } @@ -2201,6 +2204,45 @@ private void testGetECCodecs() throws Exception { } } + private void testGetTrashRoots() throws Exception { + if (isLocalFS()) { + // do not test the getAllEEPolicies for local FS. + return; + } + final Path path = new Path("/"); + FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf()); + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = + (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf()); + FileSystem httpFs = this.getHttpFSFileSystem(); + + // Create trash root for user0 + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user0"); + String user0HomeStr = DFSUtilClient.getHomeDirectory(this.getProxiedFSConf(), ugi); + Path user0Trash = new Path(user0HomeStr, FileSystem.TRASH_PREFIX); + dfs.mkdirs(user0Trash); + + Collection dfsTrashRoots = dfs.getTrashRoots(true); + Collection diffTrashRoots = null; + + if (httpFs instanceof HttpFSFileSystem) { + HttpFSFileSystem httpFS = (HttpFSFileSystem) httpFs; + diffTrashRoots = httpFS.getTrashRoots(true); + } else if (httpFs instanceof WebHdfsFileSystem) { + WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) httpFs; + diffTrashRoots = webHdfsFileSystem.getTrashRoots(true); + } else { + Assert.fail(fs.getClass().getSimpleName() + + " is not of type HttpFSFileSystem or WebHdfsFileSystem"); + } + + // Validate getTrashRoots are the same as DistributedFileSystem + assertEquals(dfsTrashRoots.size(), diffTrashRoots.size()); + } else { + Assert.fail(fs.getClass().getSimpleName() + " is not of type DistributedFileSystem."); + } + } + private void assertHttpFsReportListingWithDfsClient(SnapshotDiffReportListing diffReportListing, SnapshotDiffReportListing dfsDiffReportListing) { Assert.assertEquals(diffReportListing.getCreateList().size(), From 120620c1b7f0a11e140a7d80d2bf7be4c5c83c1b Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 15 Sep 2023 15:45:09 +0100 Subject: [PATCH 032/155] HADOOP-18888. S3A. createS3AsyncClient() always enables multipart uploads (#6056) * The multipart flag fs.s3a.multipart.uploads.enabled is passed to the async client created * s3A connector bypasses the transfer manager entirely if disabled or for small files. Contributed by Steve Loughran --- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 2 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 92 +++++++++++++------ .../apache/hadoop/fs/s3a/S3AInternals.java | 6 ++ .../apache/hadoop/fs/s3a/S3ClientFactory.java | 24 +++++ .../s3a/scale/AbstractSTestS3AHugeFiles.java | 4 +- .../scale/ITestS3AHugeFilesNoMultipart.java | 53 +++++------ 6 files changed, 117 insertions(+), 64 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index 98c72d276628e..c85263f1903ab 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -112,7 +112,7 @@ public S3AsyncClient createS3AsyncClient( return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket) .httpClientBuilder(httpClientBuilder) .multipartConfiguration(multipartConfiguration) - .multipartEnabled(true) + .multipartEnabled(parameters.isMultipartCopy()) .build(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index e192135b9f312..9307c4c2650b6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -440,6 +440,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private boolean isMultipartUploadEnabled = DEFAULT_MULTIPART_UPLOAD_ENABLED; + /** + * Should file copy operations use the S3 transfer manager? + * True unless multipart upload is disabled. + */ + private boolean isMultipartCopyEnabled; + /** * A cache of files that should be deleted when the FileSystem is closed * or the JVM is exited. @@ -576,6 +582,9 @@ public void initialize(URI name, Configuration originalConf) intOption(conf, PREFETCH_BLOCK_COUNT_KEY, PREFETCH_BLOCK_DEFAULT_COUNT, 1); this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED, DEFAULT_MULTIPART_UPLOAD_ENABLED); + // multipart copy and upload are the same; this just makes it explicit + this.isMultipartCopyEnabled = isMultipartUploadEnabled; + initThreadPools(conf); int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION); @@ -982,6 +991,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { .withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS)) .withExecutionInterceptors(auditManager.createExecutionInterceptors()) .withMinimumPartSize(partSize) + .withMultipartCopyEnabled(isMultipartCopyEnabled) .withMultipartThreshold(multiPartThreshold) .withTransferManagerExecutor(unboundedThreadPool) .withRegion(region); @@ -1468,6 +1478,11 @@ public AWSCredentialProviderList shareCredentials(final String purpose) { LOG.debug("Sharing credentials for: {}", purpose); return credentials.share(); } + + @Override + public boolean isMultipartCopyEnabled() { + return S3AFileSystem.this.isMultipartUploadEnabled; + } } /** @@ -4436,37 +4451,56 @@ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, e); } - return readInvoker.retry( - action, srcKey, - true, - () -> { - CopyObjectRequest.Builder copyObjectRequestBuilder = - getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom); - changeTracker.maybeApplyConstraint(copyObjectRequestBuilder); - incrementStatistic(OBJECT_COPY_REQUESTS); - - Copy copy = transferManager.copy( - CopyRequest.builder() - .copyObjectRequest(copyObjectRequestBuilder.build()) - .build()); + CopyObjectRequest.Builder copyObjectRequestBuilder = + getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom); + changeTracker.maybeApplyConstraint(copyObjectRequestBuilder); + CopyObjectResponse response; - try { - CompletedCopy completedCopy = copy.completionFuture().join(); - CopyObjectResponse result = completedCopy.response(); - changeTracker.processResponse(result); - incrementWriteOperations(); - instrumentation.filesCopied(1, size); - return result; - } catch (CompletionException e) { - Throwable cause = e.getCause(); - if (cause instanceof SdkException) { - SdkException awsException = (SdkException)cause; - changeTracker.processException(awsException, "copy"); - throw awsException; + // transfer manager is skipped if disabled or the file is too small to worry about + final boolean useTransferManager = isMultipartCopyEnabled && size >= multiPartThreshold; + if (useTransferManager) { + // use transfer manager + response = readInvoker.retry( + action, srcKey, + true, + () -> { + incrementStatistic(OBJECT_COPY_REQUESTS); + + Copy copy = transferManager.copy( + CopyRequest.builder() + .copyObjectRequest(copyObjectRequestBuilder.build()) + .build()); + + try { + CompletedCopy completedCopy = copy.completionFuture().join(); + return completedCopy.response(); + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SdkException) { + SdkException awsException = (SdkException)cause; + changeTracker.processException(awsException, "copy"); + throw awsException; + } + throw extractException(action, srcKey, e); } - throw extractException(action, srcKey, e); - } - }); + }); + } else { + // single part copy bypasses transfer manager + // note, this helps with some mock testing, e.g. HBoss. as there is less to mock. + response = readInvoker.retry( + action, srcKey, + true, + () -> { + LOG.debug("copyFile: single part copy {} -> {} of size {}", srcKey, dstKey, size); + incrementStatistic(OBJECT_COPY_REQUESTS); + return s3Client.copyObject(copyObjectRequestBuilder.build()); + }); + } + + changeTracker.processResponse(response); + incrementWriteOperations(); + instrumentation.filesCopied(1, size); + return response; } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java index 23c4d3501206a..18d6c1af586ff 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInternals.java @@ -115,4 +115,10 @@ public interface S3AInternals { @AuditEntryPoint @Retries.RetryTranslated HeadBucketResponse getBucketMetadata() throws IOException; + + /** + * Is multipart copy enabled? + * @return true if the transfer manager is used to copy files. + */ + boolean isMultipartCopyEnabled(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index d4504cd08d74c..e2e792ebfb668 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -156,6 +156,11 @@ final class S3ClientCreationParameters { */ private long multiPartThreshold; + /** + * Multipart upload enabled. + */ + private boolean multipartCopy = true; + /** * Executor that the transfer manager will use to execute background tasks. */ @@ -399,5 +404,24 @@ public S3ClientCreationParameters withRegion( public Region getRegion() { return region; } + + /** + * Set the multipart flag.. + * + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) { + this.multipartCopy = value; + return this; + } + + /** + * Get the multipart flag. + * @return multipart flag + */ + public boolean isMultipartCopy() { + return multipartCopy; + } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java index 1a30c04358bac..75c6efbe2ab19 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java @@ -355,10 +355,10 @@ public long getFilesize() { /** * Is this expected to be a multipart upload? * Assertions will change if not. - * @return true by default. + * @return what the filesystem expects. */ protected boolean expectMultipartUpload() { - return true; + return getFileSystem().getS3AInternals().isMultipartCopyEnabled(); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java index ed300dba01ea3..e154ab5676f81 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java @@ -18,13 +18,10 @@ package org.apache.hadoop.fs.s3a.scale; +import org.assertj.core.api.Assertions; + import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.Constants; -import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3a.S3ATestUtils; -import org.apache.hadoop.fs.s3a.api.UnsupportedRequestException; import static org.apache.hadoop.fs.contract.ContractTestUtils.IO_CHUNK_BUFFER_SIZE; import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD; @@ -33,7 +30,6 @@ import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_UPLOADS_ENABLED; import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; -import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Use a single PUT for the whole upload/rename/delete workflow; include verification @@ -41,11 +37,6 @@ */ public class ITestS3AHugeFilesNoMultipart extends AbstractSTestS3AHugeFiles { - /** - * Size to ensure MPUs don't happen in transfer manager. - */ - public static final String S_1T = "1T"; - public static final String SINGLE_PUT_REQUEST_TIMEOUT = "1h"; /** @@ -56,11 +47,23 @@ protected String getBlockOutputBufferName() { return Constants.FAST_UPLOAD_BUFFER_DISK; } + /** + * Multipart upload is always disabled. + * @return false + */ @Override protected boolean expectMultipartUpload() { return false; } + /** + * Is multipart copy enabled? + * @return true if the transfer manager is used to copy files. + */ + private boolean isMultipartCopyEnabled() { + return getFileSystem().getS3AInternals().isMultipartCopyEnabled(); + } + /** * Create a configuration without multipart upload, * and a long request timeout to allow for a very slow @@ -77,35 +80,21 @@ protected Configuration createScaleConfiguration() { MULTIPART_SIZE, REQUEST_TIMEOUT); conf.setInt(IO_CHUNK_BUFFER_SIZE, 655360); - conf.set(MIN_MULTIPART_THRESHOLD, S_1T); - conf.set(MULTIPART_SIZE, S_1T); + conf.setInt(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE); + conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE); conf.setBoolean(MULTIPART_UPLOADS_ENABLED, false); conf.set(REQUEST_TIMEOUT, SINGLE_PUT_REQUEST_TIMEOUT); return conf; } /** - * After the file is created, attempt a rename with an FS - * instance with a small multipart threshold; - * this MUST be rejected. + * Verify multipart copy is disabled. */ @Override public void test_030_postCreationAssertions() throws Throwable { - assumeHugeFileExists(); - final Path hugefile = getHugefile(); - final Path hugefileRenamed = getHugefileRenamed(); - describe("renaming %s to %s", hugefile, hugefileRenamed); - S3AFileSystem fs = getFileSystem(); - fs.delete(hugefileRenamed, false); - // create a new fs with a small multipart threshold; expect rename failure. - final Configuration conf = new Configuration(fs.getConf()); - conf.setInt(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE); - conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE); - S3ATestUtils.disableFilesystemCaching(conf); - - try (FileSystem fs2 = FileSystem.get(fs.getUri(), conf)) { - intercept(UnsupportedRequestException.class, () -> - fs2.rename(hugefile, hugefileRenamed)); - } + super.test_030_postCreationAssertions(); + Assertions.assertThat(isMultipartCopyEnabled()) + .describedAs("Multipart copy should be disabled in %s", getFileSystem()) + .isFalse(); } } From 9e489b9ab5367b431849f957d0fe1ffadc43cd47 Mon Sep 17 00:00:00 2001 From: zhangshuyan <81411509+zhangshuyan0@users.noreply.github.com> Date: Mon, 18 Sep 2023 10:35:13 +0800 Subject: [PATCH 033/155] HDFS-17190. EC: Fix bug of OIV processing XAttr. (#6067). Contributed by Shuyan Zhang. Signed-off-by: He Xiaoqiao --- .../hdfs/server/namenode/XAttrFormat.java | 2 +- .../offlineImageViewer/PBImageTextWriter.java | 5 +- .../TestOfflineImageViewer.java | 46 ++++++++++++++++++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java index 4d46e691df217..69f21c176de5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java @@ -72,7 +72,7 @@ public static String getName(int record) { return SerialNumberManager.XATTR.getString(nid); } - static int toInt(XAttr a) { + public static int toInt(XAttr a) { int nid = SerialNumberManager.XATTR.getSerialNumber(a.getName()); int nsOrd = a.getNameSpace().ordinal(); long value = NS.BITS.combine(nsOrd & NS_MASK, 0L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index bd6c860ccf071..f2b329fa2ffd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -52,6 +52,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; @@ -514,6 +515,8 @@ public long getParentId(long id) throws IOException { private File filename; private int numThreads; private String parallelOutputFile; + private final XAttr ecXAttr = + XAttrHelper.buildXAttr(XATTR_ERASURECODING_POLICY); /** * Construct a PB FsImage writer to generate text file. @@ -1040,7 +1043,7 @@ public static void mergeFiles(String[] srcPaths, String resultPath) List xattrs = FSImageFormatPBINode.Loader.loadXAttrs(xattrFeatureProto, stringTable); for (XAttr xattr : xattrs) { - if (XATTR_ERASURECODING_POLICY.contains(xattr.getName())){ + if (xattr.equalsIgnoreValue(ecXAttr)){ try{ ByteArrayInputStream bIn = new ByteArrayInputStream(xattr.getValue()); DataInputStream dIn = new DataInputStream(bIn); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index b4ba775732921..c24c9132cbcd5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -29,6 +29,7 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -66,6 +67,7 @@ import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.SafeModeAction; +import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; @@ -73,20 +75,26 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; +import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.FsImageProto; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrCompactProto; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrFeatureProto; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; +import org.apache.hadoop.hdfs.server.namenode.XAttrFormat; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.net.NetUtils; @@ -719,7 +727,13 @@ private FsImageProto.INodeSection.INode createSampleFileInode() { .build(); } - private FsImageProto.INodeSection.INode createSampleDirInode() { + private FsImageProto.INodeSection.INode createSampleDirInode() + throws IOException { + return createSampleDirInode(false); + } + + private FsImageProto.INodeSection.INode createSampleDirInode( + boolean builXAttr) throws IOException { FsImageProto.INodeSection.AclFeatureProto.Builder acl = FsImageProto.INodeSection.AclFeatureProto.newBuilder() .addEntries(2); @@ -729,6 +743,19 @@ private FsImageProto.INodeSection.INode createSampleDirInode() { .setNsQuota(700) .setModificationTime(SAMPLE_TIMESTAMP) .setAcl(acl); + if (builXAttr) { + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + DataOutputStream dOut = new DataOutputStream(bOut); + WritableUtils.writeString(dOut, "test-value"); + XAttr a = XAttrHelper.buildXAttr("system.hdfs", bOut.toByteArray()); + XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder(); + XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto.newBuilder(); + int v = XAttrFormat.toInt(a); + xAttrCompactBuilder.setName(v); + xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue())); + b.addXAttrs(xAttrCompactBuilder.build()); + directory.setXAttrs(b); + } return FsImageProto.INodeSection.INode.newBuilder() .setType(FsImageProto.INodeSection.INode.Type.DIRECTORY) @@ -754,6 +781,11 @@ private FsImageProto.INodeSection.INode createSampleSymlink() { private PBImageDelimitedTextWriter createDelimitedWriterSpy() throws IOException { + return createDelimitedWriterSpy(false); + } + + private PBImageDelimitedTextWriter createDelimitedWriterSpy(boolean printECPolicy) + throws IOException { FsPermission fsPermission = new FsPermission( FsAction.ALL, FsAction.WRITE_EXECUTE, @@ -764,7 +796,9 @@ private PBImageDelimitedTextWriter createDelimitedWriterSpy() fsPermission); PBImageDelimitedTextWriter writer = new - PBImageDelimitedTextWriter(null, ",", ""); + PBImageDelimitedTextWriter(null, ",", "", false, + printECPolicy, 1, "-", new Configuration()); + PBImageDelimitedTextWriter writerSpy = spy(writer); when(writerSpy.getPermission(anyLong())).thenReturn(permStatus); return writerSpy; @@ -786,6 +820,14 @@ public void testWriterOutputEntryBuilderForDirectory() throws IOException { createSampleDirInode())); } + @Test + public void testECXAttr() throws IOException { + assertEquals("/path/dir,0,2000-01-01 00:00,1970-01-01 00:00" + + ",0,0,0,700,1000,drwx-wx-w-+,user_1,group_1,-", + createDelimitedWriterSpy(true).getEntry("/path/", + createSampleDirInode(true))); + } + @Test public void testWriterOutputEntryBuilderForSymlink() throws IOException { assertEquals("/path/sym,0,2000-01-01 00:00,2000-01-01 00:00" + From 18f9989ff2d3910afad9a0c586e3c7770feefc66 Mon Sep 17 00:00:00 2001 From: hfutatzhanghb Date: Mon, 18 Sep 2023 20:56:45 +0800 Subject: [PATCH 034/155] HDFS-17192. Add bock info when constructing remote block reader meets IOException. (#6081). Contributed by farmmamba. Signed-off-by: He Xiaoqiao --- .../org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index 72584e5f5a2a3..d013b10ef4df6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -769,7 +769,7 @@ private BlockReader getRemoteBlockReaderFromTcp() throws IOException { LOG.debug("Closed potentially stale remote peer {}", peer, ioe); } else { // Handle an I/O error we got when using a newly created peer. - LOG.warn("I/O error constructing remote block reader.", ioe); + LOG.warn("I/O error constructing remote block reader for block {}", block, ioe); throw ioe; } } finally { From 60f3a2b1019373599c25bc09704bafa0884e4a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=AB=A0=E9=94=A1=E5=B9=B3?= <40832063+zhangxiping1@users.noreply.github.com> Date: Tue, 19 Sep 2023 00:40:22 +0800 Subject: [PATCH 035/155] =?UTF-8?q?HDFS-17138=20RBF:=20We=20changed=20the?= =?UTF-8?q?=20hadoop.security.auth=5Fto=5Flocal=20configur=E2=80=A6=20(#59?= =?UTF-8?q?21)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AbstractDelegationTokenSecretManager.java | 7 +++- .../hdfs/security/TestDelegationToken.java | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index cafa5135e68e6..8378a47ceddfe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -81,7 +81,12 @@ class AbstractDelegationTokenSecretManager expiredTokens = new HashSet(); + expiredTokens.add(dtId); + setRules("RULE:[2:$1@$0](OtherUser.*)s/.*/OtherUser/"); + //rules was modified, causing the existing tokens + //(May be loaded from other storage systems like zookeeper) to fail to match the kerberos rules, + //return an exception that cannot be handled + new AbstractDelegationTokenSecretManager(10 * 1000, 10 * 1000, + 10 * 1000, 10 * 1000) { + @Override + public DelegationTokenIdentifier createIdentifier() { + return null; + } + public void logExpireTokens(Collection expiredTokens) + throws IOException { + super.logExpireTokens(expiredTokens); + } + }.logExpireTokens(expiredTokens); + } } From 23c22b282319a407a2b37a6ebbd4f85873351d36 Mon Sep 17 00:00:00 2001 From: Hexiaoqiao Date: Tue, 19 Sep 2023 01:50:53 +0800 Subject: [PATCH 036/155] HADOOP-18906. Increase default batch size of ZKDTSM token seqnum to reduce overflow speed of zonde dataVersion. (#6097) --- .../token/delegation/ZKDelegationTokenSecretManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index da233c33fa73d..2524929853cc7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -110,8 +110,9 @@ public abstract class ZKDelegationTokenSecretManager Date: Tue, 19 Sep 2023 12:38:36 +0100 Subject: [PATCH 037/155] HADOOP-18890. Remove use of okhttp in runtime code (#6057) Contributed by PJ Fanning --- LICENSE-binary | 8 +- NOTICE-binary | 13 ---- hadoop-client-modules/hadoop-client/pom.xml | 12 --- .../hadoop-huaweicloud/pom.xml | 4 - hadoop-common-project/hadoop-common/pom.xml | 4 +- .../dev-support/findbugsExcludeFile.xml | 13 ---- .../hadoop-hdfs-client/pom.xml | 22 ------ ...fRefreshTokenBasedAccessTokenProvider.java | 76 ++++++++++-------- .../CredentialBasedAccessTokenProvider.java | 78 ++++++++++--------- .../hdfs/web/oauth2/OAuth2Constants.java | 5 +- .../hadoop-hdfs-httpfs/pom.xml | 4 +- hadoop-project/pom.xml | 54 ++----------- 12 files changed, 100 insertions(+), 193 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index e0c5923d043c4..5a653da811602 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -241,8 +241,6 @@ com.google.guava:guava:27.0-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.microsoft.azure:azure-storage:7.0.0 com.nimbusds:nimbus-jose-jwt:9.31 -com.squareup.okhttp3:okhttp:4.10.0 -com.squareup.okio:okio:3.4.0 com.zaxxer:HikariCP:4.0.3 commons-beanutils:commons-beanutils:1.9.4 commons-cli:commons-cli:1.5.0 @@ -319,8 +317,8 @@ org.apache.hbase:hbase-common:1.7.1 org.apache.hbase:hbase-protocol:1.7.1 org.apache.htrace:htrace-core:3.1.0-incubating org.apache.htrace:htrace-core4:4.1.0-incubating -org.apache.httpcomponents:httpclient:4.5.6 -org.apache.httpcomponents:httpcore:4.4.10 +org.apache.httpcomponents:httpclient:4.5.13 +org.apache.httpcomponents:httpcore:4.4.13 org.apache.kafka:kafka-clients:2.8.2 org.apache.kerby:kerb-admin:2.0.3 org.apache.kerby:kerb-client:2.0.3 @@ -357,8 +355,6 @@ org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217 org.ehcache:ehcache:3.3.1 org.ini4j:ini4j:0.5.4 -org.jetbrains.kotlin:kotlin-stdlib:1.4.10 -org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10 org.lz4:lz4-java:1.7.1 org.objenesis:objenesis:2.6 org.xerial.snappy:snappy-java:1.1.10.1 diff --git a/NOTICE-binary b/NOTICE-binary index b96e052658876..6db51d08b42f0 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------ - -This product contains a modified portion of 'OkHttp', an open source -HTTP & SPDY client for Android and Java applications, which can be obtained -at: - - * LICENSE: - * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) - * HOMEPAGE: - * https://github.com/square/okhttp - * LOCATION_IN_GRPC: - * okhttp/third_party/okhttp - This product contains a modified portion of 'Netty', an open source networking library, which can be obtained at: diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index 9170bf4b549c9..08452aa20ef02 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -118,18 +118,6 @@ org.eclipse.jetty jetty-server - - org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - - - com.squareup.okhttp3 - okhttp - com.sun.jersey jersey-core diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml index b96883b9ac80d..4892a7ac8629f 100755 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml @@ -161,10 +161,6 @@ esdk-obs-java ${esdk.version} - - okio - com.squareup.okio - log4j-core org.apache.logging.log4j diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 207f1f5351ee9..c453546695b4c 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -394,8 +394,8 @@ test - com.squareup.okio - okio-jvm + org.jetbrains.kotlin + kotlin-stdlib-jdk8 test diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index 508388aa4812e..c96b3a99bd1c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -94,17 +94,4 @@ - - - - - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index 01ab5f41373f4..9e370788a6b61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -34,28 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> - - com.squareup.okhttp3 - okhttp - - - com.squareup.okio - okio-jvm - - - - - com.squareup.okio - okio-jvm - - - org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - org.apache.hadoop hadoop-common diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index e944e8c1c8d77..7b82cad215dde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -19,13 +19,10 @@ package org.apache.hadoop.hdfs.web.oauth2; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -33,7 +30,17 @@ import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.util.JsonSerialization; import org.apache.hadoop.util.Timer; +import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; @@ -103,34 +110,37 @@ public synchronized String getAccessToken() throws IOException { } void refresh() throws IOException { - OkHttpClient client = - new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - String bodyString = - Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId); - - RequestBody body = RequestBody.create(bodyString, URLENCODED); - - Request request = new Request.Builder().url(refreshURL).post(body).build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code " + response); - } - if (response.code() != HttpStatus.SC_OK) { - throw new IllegalArgumentException( - "Received invalid http response: " + response.code() + ", text = " - + response.toString()); + final List pairs = new ArrayList<>(); + pairs.add(new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN)); + pairs.add(new BasicNameValuePair(REFRESH_TOKEN, refreshToken)); + pairs.add(new BasicNameValuePair(CLIENT_ID, clientId)); + final RequestConfig config = RequestConfig.custom() + .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .build(); + try (CloseableHttpClient client = + HttpClientBuilder.create().setDefaultRequestConfig(config).build()) { + final HttpPost httpPost = new HttpPost(refreshURL); + httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8)); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED); + try (CloseableHttpResponse response = client.execute(httpPost)) { + final int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + throw new IllegalArgumentException( + "Received invalid http response: " + statusCode + ", text = " + + EntityUtils.toString(response.getEntity())); + } + Map responseBody = JsonSerialization.mapReader().readValue( + EntityUtils.toString(response.getEntity())); + + String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); + accessTokenTimer.setExpiresIn(newExpiresIn); + + accessToken = responseBody.get(ACCESS_TOKEN).toString(); } - - Map responseBody = JsonSerialization.mapReader().readValue(response.body().string()); - - String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); - accessTokenTimer.setExpiresIn(newExpiresIn); - - accessToken = responseBody.get(ACCESS_TOKEN).toString(); + } catch (RuntimeException e) { + throw new IOException("Exception while refreshing access token", e); } catch (Exception e) { throw new IOException("Exception while refreshing access token", e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java index 25ceb8846092b..1803e997adc64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java @@ -19,13 +19,10 @@ package org.apache.hadoop.hdfs.web.oauth2; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -33,7 +30,17 @@ import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.util.JsonSerialization; import org.apache.hadoop.util.Timer; +import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; @@ -97,38 +104,37 @@ public synchronized String getAccessToken() throws IOException { } void refresh() throws IOException { - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(), - GRANT_TYPE, CLIENT_CREDENTIALS, - CLIENT_ID, clientId); - - RequestBody body = RequestBody.create(bodyString, URLENCODED); - - Request request = new Request.Builder() - .url(refreshURL) - .post(body) + final List pairs = new ArrayList<>(); + pairs.add(new BasicNameValuePair(CLIENT_SECRET, getCredential())); + pairs.add(new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS)); + pairs.add(new BasicNameValuePair(CLIENT_ID, clientId)); + final RequestConfig config = RequestConfig.custom() + .setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) + .setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT) .build(); - try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) { - throw new IOException("Unexpected code " + response); - } - - if (response.code() != HttpStatus.SC_OK) { - throw new IllegalArgumentException("Received invalid http response: " - + response.code() + ", text = " + response.toString()); + try (CloseableHttpClient client = + HttpClientBuilder.create().setDefaultRequestConfig(config).build()) { + final HttpPost httpPost = new HttpPost(refreshURL); + httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8)); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED); + try (CloseableHttpResponse response = client.execute(httpPost)) { + final int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + throw new IllegalArgumentException( + "Received invalid http response: " + statusCode + ", text = " + + EntityUtils.toString(response.getEntity())); + } + Map responseBody = JsonSerialization.mapReader().readValue( + EntityUtils.toString(response.getEntity())); + + String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); + timer.setExpiresIn(newExpiresIn); + + accessToken = responseBody.get(ACCESS_TOKEN).toString(); } - - Map responseBody = JsonSerialization.mapReader().readValue( - response.body().string()); - - String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); - timer.setExpiresIn(newExpiresIn); - - accessToken = responseBody.get(ACCESS_TOKEN).toString(); + } catch (RuntimeException e) { + throw new IOException("Unable to obtain access token from credential", e); } catch (Exception e) { throw new IOException("Unable to obtain access token from credential", e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java index 2f28b65e40e92..dbe95aca31a0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java @@ -18,7 +18,6 @@ */ package org.apache.hadoop.hdfs.web.oauth2; -import okhttp3.MediaType; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -30,8 +29,8 @@ public final class OAuth2Constants { private OAuth2Constants() { /** Private constructor. **/ } - public static final MediaType URLENCODED - = MediaType.parse("application/x-www-form-urlencoded; charset=utf-8"); + public static final String URLENCODED + = "application/x-www-form-urlencoded; charset=utf-8"; /* Constants for OAuth protocol */ public static final String ACCESS_TOKEN = "access_token"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index b471fd062ddee..b5b264ffa8b54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -205,8 +205,8 @@ test - com.squareup.okhttp3 - okhttp + org.jetbrains.kotlin + kotlin-stdlib-jdk8 test diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 479a2183ade67..e95e7471c5533 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -131,10 +131,8 @@ 4.0.3 10.14.2.0 6.2.1.jre7 - 4.10.0 - 3.4.0 - 1.6.20 - 1.6.20 + 4.11.0 + 1.6.20 2.0.6.1 5.2.0 2.9.0 @@ -222,59 +220,21 @@ com.squareup.okhttp3 - okhttp + mockwebserver ${okhttp3.version} + test org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - - - com.squareup.okio - okio-jvm - - - - - com.squareup.okio - okio-jvm - ${okio.version} - - - org.jetbrains.kotlin - kotlin-stdlib - ${kotlin-stdlib.verion} - - - org.jetbrains - annotations + kotlin-stdlib-jdk8 org.jetbrains.kotlin - kotlin-stdlib-common - ${kotlin-stdlib-common.version} - - - com.squareup.okhttp3 - mockwebserver - ${okhttp3.version} + kotlin-stdlib-jdk8 + ${kotlin-stdlib.version} test - - - com.squareup.okio - okio-jvm - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - - jdiff From 512e39e89df4653bc36301048c4db05e39551ee3 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 20 Sep 2023 08:04:05 +0800 Subject: [PATCH 038/155] Revert "MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0. (#6042) Contributed by Chenyu Zheng." (#6103) This reverts commit ab2bc90e090564b7466ab1b759fbb849612ae82b. --- .../src/main/java/org/apache/hadoop/mapred/TaskLog.java | 3 +-- .../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 6 ++---- .../src/main/resources/mapred-default.xml | 7 ++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index fdfa3e7710f47..a0223dedd64ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -466,8 +466,7 @@ public static long getTaskLogLength(JobConf conf) { } public static long getTaskLogLimitBytes(Configuration conf) { - return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) * - 1024; + return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 24689e09daf13..8ec984e777bb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -410,8 +410,6 @@ public interface MRJobConfig { public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; - public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240; - public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; @@ -760,11 +758,11 @@ public interface MRJobConfig { public static final String MR_AM_LOG_KB = MR_AM_PREFIX + "container.log.limit.kb"; - public static final int DEFAULT_MR_AM_LOG_KB = 10240; + public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll public static final String MR_AM_LOG_BACKUPS = MR_AM_PREFIX + "container.log.backups"; - public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll + public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; /**The number of splits when reporting progress in MR*/ public static final String MR_AM_NUM_PROGRESS_SPLITS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 493e4c3a37e94..a6d68acda34ab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -823,15 +823,16 @@ mapreduce.task.userlog.limit.kb - 10240 - The maximum size of user-logs of each task in KB. + 0 + The maximum size of user-logs of each task in KB. 0 disables the cap. yarn.app.mapreduce.am.container.log.limit.kb - 10240 + 0 The maximum size of the MRAppMaster attempt container logs in KB. + 0 disables the cap. From 39f36d90713d80b5dde4a4d0be92532cabb00ae9 Mon Sep 17 00:00:00 2001 From: ConfX <114765570+teamconfx@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:46:57 -0400 Subject: [PATCH 039/155] HDFS-17105. mistakenly purge editLogs even after it is empty in NNStorageRetentionManager. (#6036). Contributed by ConfX. Signed-off-by: He Xiaoqiao --- .../hadoop/hdfs/server/namenode/NNStorageRetentionManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index b81cfccd75734..0564a05c3201b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -80,6 +80,9 @@ public NNStorageRetentionManager( Preconditions.checkArgument(numExtraEditsToRetain >= 0, DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY + " must not be negative"); + Preconditions.checkArgument(maxExtraEditsSegmentsToRetain >= 0, + DFSConfigKeys.DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_KEY + + " must not be negative"); this.storage = storage; this.purgeableLogs = purgeableLogs; From f24b73e5f3ac640f491231f02d9d8afaf1855b5c Mon Sep 17 00:00:00 2001 From: Pranav Saxena <108325433+saxenapranav@users.noreply.github.com> Date: Wed, 20 Sep 2023 01:54:36 -0700 Subject: [PATCH 040/155] HADOOP-18873. ABFS: AbfsOutputStream doesnt close DataBlocks object. (#6010) AbfsOutputStream to close the dataBlock object created for the upload. Contributed By: Pranav Saxena --- .../apache/hadoop/fs/store/DataBlocks.java | 4 +- .../fs/azurebfs/AzureBlobFileSystem.java | 2 +- .../fs/azurebfs/AzureBlobFileSystemStore.java | 7 ++- .../azurebfs/services/AbfsOutputStream.java | 2 +- .../ITestAzureBlobFileSystemAppend.java | 59 +++++++++++++++++++ 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java index c70d0ee91e15e..0ae9ee6378b57 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java @@ -329,7 +329,7 @@ public String getKeyToBufferDir() { */ public static abstract class DataBlock implements Closeable { - enum DestState {Writing, Upload, Closed} + public enum DestState {Writing, Upload, Closed} private volatile DestState state = Writing; private final long index; @@ -375,7 +375,7 @@ protected final void verifyState(DestState expected) * * @return the current state. */ - final DestState getState() { + public final DestState getState() { return state; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 426ad8ca1e191..8f7fbc570221e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -330,7 +330,7 @@ public FSDataOutputStream create(final Path f, try { TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE, overwrite, tracingHeaderFormat, listener); - OutputStream outputStream = abfsStore.createFile(qualifiedPath, statistics, overwrite, + OutputStream outputStream = getAbfsStore().createFile(qualifiedPath, statistics, overwrite, permission == null ? FsPermission.getFileDefault() : permission, FsPermission.getUMask(getConf()), tracingContext); statIncrement(FILES_CREATED); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 49dff3360899e..ff37ee5ee901e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -707,7 +707,7 @@ private AbfsOutputStreamContext populateAbfsOutputStreamContext( .withWriteMaxConcurrentRequestCount(abfsConfiguration.getWriteMaxConcurrentRequestCount()) .withMaxWriteRequestsToQueue(abfsConfiguration.getMaxWriteRequestsToQueue()) .withLease(lease) - .withBlockFactory(blockFactory) + .withBlockFactory(getBlockFactory()) .withBlockOutputActiveBlocks(blockOutputActiveBlocks) .withClient(client) .withPosition(position) @@ -1940,6 +1940,11 @@ void setClient(AbfsClient client) { this.client = client; } + @VisibleForTesting + DataBlocks.BlockFactory getBlockFactory() { + return blockFactory; + } + @VisibleForTesting void setNamespaceEnabled(Trilean isNamespaceEnabled){ this.isNamespaceEnabled = isNamespaceEnabled; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java index 4268dc3f918a1..89b0fe2040f6e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java @@ -344,7 +344,7 @@ private void uploadBlockAsync(DataBlocks.DataBlock blockToUpload, outputStreamStatistics.uploadSuccessful(bytesLength); return null; } finally { - IOUtils.close(blockUploadData); + IOUtils.close(blockUploadData, blockToUpload); } }); writeOperations.add(new WriteOperation(job, offset, bytesLength)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java index dbe4b42a67df3..7d182f936b7bb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java @@ -20,15 +20,31 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.io.OutputStream; +import java.util.HashSet; import java.util.Random; +import java.util.Set; +import org.assertj.core.api.Assertions; import org.junit.Test; +import org.mockito.Mockito; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.store.BlockUploadStatistics; +import org.apache.hadoop.fs.store.DataBlocks; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER; +import static org.apache.hadoop.fs.store.DataBlocks.DATA_BLOCKS_BUFFER_ARRAY; +import static org.apache.hadoop.fs.store.DataBlocks.DATA_BLOCKS_BUFFER_DISK; +import static org.apache.hadoop.fs.store.DataBlocks.DATA_BLOCKS_BYTEBUFFER; +import static org.apache.hadoop.fs.store.DataBlocks.DataBlock.DestState.Closed; +import static org.apache.hadoop.fs.store.DataBlocks.DataBlock.DestState.Writing; /** * Test append operations. @@ -90,4 +106,47 @@ public void testTracingForAppend() throws IOException { fs.getFileSystemId(), FSOperationType.APPEND, false, 0)); fs.append(testPath, 10); } + + @Test + public void testCloseOfDataBlockOnAppendComplete() throws Exception { + Set blockBufferTypes = new HashSet<>(); + blockBufferTypes.add(DATA_BLOCKS_BUFFER_DISK); + blockBufferTypes.add(DATA_BLOCKS_BYTEBUFFER); + blockBufferTypes.add(DATA_BLOCKS_BUFFER_ARRAY); + for (String blockBufferType : blockBufferTypes) { + Configuration configuration = new Configuration(getRawConfiguration()); + configuration.set(DATA_BLOCKS_BUFFER, blockBufferType); + AzureBlobFileSystem fs = Mockito.spy( + (AzureBlobFileSystem) FileSystem.newInstance(configuration)); + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + Mockito.doReturn(store).when(fs).getAbfsStore(); + DataBlocks.DataBlock[] dataBlock = new DataBlocks.DataBlock[1]; + Mockito.doAnswer(getBlobFactoryInvocation -> { + DataBlocks.BlockFactory factory = Mockito.spy( + (DataBlocks.BlockFactory) getBlobFactoryInvocation.callRealMethod()); + Mockito.doAnswer(factoryCreateInvocation -> { + dataBlock[0] = Mockito.spy( + (DataBlocks.DataBlock) factoryCreateInvocation.callRealMethod()); + return dataBlock[0]; + }) + .when(factory) + .create(Mockito.anyLong(), Mockito.anyInt(), Mockito.any( + BlockUploadStatistics.class)); + return factory; + }).when(store).getBlockFactory(); + try (OutputStream os = fs.create( + new Path(getMethodName() + "_" + blockBufferType))) { + os.write(new byte[1]); + Assertions.assertThat(dataBlock[0].getState()) + .describedAs( + "On write of data in outputStream, state should become Writing") + .isEqualTo(Writing); + os.close(); + Mockito.verify(dataBlock[0], Mockito.times(1)).close(); + Assertions.assertThat(dataBlock[0].getState()) + .describedAs("On close of outputStream, state should become Closed") + .isEqualTo(Closed); + } + } + } } From 5512c9f92464d9c7d83a79d2460efc2537fc4c3f Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Wed, 20 Sep 2023 15:56:42 +0530 Subject: [PATCH 041/155] HADOOP-18797. Support Concurrent Writes With S3A Magic Committer (#6006) Jobs which commit their work to S3 thr magic committer now use a unique magic containing the job ID: __magic_job-${jobid} This allows for multiple jobs to write to the same destination simultaneously. Contributed by Syed Shameerur Rahman --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +- .../hadoop/fs/s3a/commit/CommitConstants.java | 6 ++- .../commit/InternalCommitterConstants.java | 4 +- .../fs/s3a/commit/MagicCommitPaths.java | 35 ++++++++-------- .../fs/s3a/commit/impl/CommitUtilsWithMR.java | 19 +++++---- .../commit/magic/MagicS3GuardCommitter.java | 3 +- .../hadoop-aws/committer_architecture.md | 42 +++++++++---------- .../markdown/tools/hadoop-aws/committers.md | 15 ++++--- .../s3a/commit/AbstractITCommitProtocol.java | 6 ++- .../s3a/commit/AbstractYarnClusterITest.java | 3 +- .../fs/s3a/commit/CommitterTestHelper.java | 6 ++- .../s3a/commit/ITestCommitOperationCost.java | 11 ++--- .../fs/s3a/commit/ITestCommitOperations.java | 6 +-- .../fs/s3a/commit/TestMagicCommitPaths.java | 26 ++++++------ .../integration/ITestS3ACommitterMRJob.java | 16 +++---- .../magic/ITestMagicCommitProtocol.java | 18 ++++---- .../magic/ITestS3AHugeMagicCommits.java | 9 ++-- .../ITestStagingCommitProtocol.java | 3 +- .../fs/s3a/impl/TestHeaderProcessing.java | 2 +- 19 files changed, 125 insertions(+), 109 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 9307c4c2650b6..fa7de69140fcf 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -3669,7 +3669,7 @@ public UserGroupInformation getOwner() { * directories. Has the semantics of Unix {@code 'mkdir -p'}. * Existence of the directory hierarchy is not an error. * Parent elements are scanned to see if any are a file, - * except under __magic paths. + * except under "MAGIC PATH" paths. * There the FS assumes that the destination directory creation * did that scan and that paths in job/task attempts are all * "well formed" @@ -4769,7 +4769,7 @@ public boolean isMagicCommitPath(Path path) { /** * Predicate: is a path under a magic commit path? - * True if magic commit is enabled and the path is under __magic, + * True if magic commit is enabled and the path is under "MAGIC PATH", * irrespective of file type. * @param path path to examine * @return true if the path is in a magic dir and the FS has magic writes enabled. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java index 6e2a5d8c9fbce..52df58d6a4b43 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java @@ -39,6 +39,8 @@ private CommitConstants() { * {@value}. */ public static final String MAGIC = "__magic"; + public static final String JOB_ID_PREFIX = "job-"; + public static final String MAGIC_PATH_PREFIX = MAGIC + "_" + JOB_ID_PREFIX; /** * Marker of the start of a directory tree for calculating @@ -280,10 +282,12 @@ private CommitConstants() { /** * Default configuration value for * {@link #FS_S3A_COMMITTER_ABORT_PENDING_UPLOADS}. + * It is disabled by default to support concurrent writes on the same + * parent directory but different partition/sub directory. * Value: {@value}. */ public static final boolean DEFAULT_FS_S3A_COMMITTER_ABORT_PENDING_UPLOADS = - true; + false; /** * The limit to the number of committed objects tracked during diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java index b2d4bfaeeabab..ee07e652bed65 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java @@ -25,8 +25,8 @@ import org.apache.hadoop.fs.s3a.commit.staging.PartitionedStagingCommitterFactory; import org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterFactory; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; /** * These are internal constants not intended for public use. @@ -108,7 +108,7 @@ private InternalCommitterConstants() { /** Error message for a path without a magic element in the list: {@value}. */ public static final String E_NO_MAGIC_PATH_ELEMENT - = "No " + MAGIC + " element in path"; + = "No " + MAGIC_PATH_PREFIX + " element in path"; /** * The UUID for jobs: {@value}. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java index 18bb90da8208b..011d811fc0231 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java @@ -21,13 +21,15 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.E_NO_MAGIC_PATH_ELEMENT; /** @@ -76,7 +78,8 @@ public static List splitPathToElements(Path path) { * @return true if a path is considered magic */ public static boolean isMagicPath(List elements) { - return elements.contains(MAGIC); + return elements.stream() + .anyMatch(element -> element.startsWith(MAGIC_PATH_PREFIX)); } /** @@ -96,9 +99,16 @@ public static boolean containsBasePath(List elements) { * @throws IllegalArgumentException if there is no magic element */ public static int magicElementIndex(List elements) { - int index = elements.indexOf(MAGIC); - checkArgument(index >= 0, E_NO_MAGIC_PATH_ELEMENT); - return index; + Optional index = IntStream.range(0, elements.size()) + .filter(i -> elements.get(i).startsWith(MAGIC_PATH_PREFIX)) + .boxed() + .findFirst(); + + if (index.isPresent()) { + return index.get(); + } else { + throw new IllegalArgumentException(E_NO_MAGIC_PATH_ELEMENT); + } } /** @@ -182,18 +192,9 @@ public static String lastElement(List strings) { return strings.get(strings.size() - 1); } - /** - * Get the magic subdirectory of a destination directory. - * @param destDir the destination directory - * @return a new path. - */ - public static Path magicSubdir(Path destDir) { - return new Path(destDir, MAGIC); - } - /** * Calculates the final destination of a file. - * This is the parent of any {@code __magic} element, and the filename + * This is the parent of any "MAGIC PATH" element, and the filename * of the path. That is: all intermediate child path elements are discarded. * Why so? paths under the magic path include job attempt and task attempt * subdirectories, which need to be skipped. @@ -208,8 +209,8 @@ public static List finalDestination(List elements) { if (isMagicPath(elements)) { List destDir = magicPathParents(elements); List children = magicPathChildren(elements); - checkArgument(!children.isEmpty(), "No path found under " + - MAGIC); + checkArgument(!children.isEmpty(), "No path found under the prefix " + + MAGIC_PATH_PREFIX); ArrayList dest = new ArrayList<>(destDir); if (containsBasePath(children)) { // there's a base marker in the path diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java index c38ab2e9ba1af..bccfa7523b9fa 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java @@ -26,9 +26,11 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.JOB_ID_PREFIX; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.TEMP_DATA; /** @@ -49,10 +51,13 @@ private CommitUtilsWithMR() { /** * Get the location of magic job attempts. * @param out the base output directory. + * @param jobUUID unique Job ID. * @return the location of magic job attempts. */ - public static Path getMagicJobAttemptsPath(Path out) { - return new Path(out, MAGIC); + public static Path getMagicJobAttemptsPath(Path out, String jobUUID) { + Preconditions.checkArgument(jobUUID != null && !(jobUUID.isEmpty()), + "Invalid job ID: %s", jobUUID); + return new Path(out, MAGIC_PATH_PREFIX + jobUUID); } /** @@ -76,7 +81,7 @@ public static Path getMagicJobAttemptPath(String jobUUID, int appAttemptId, Path dest) { return new Path( - getMagicJobAttemptsPath(dest), + getMagicJobAttemptsPath(dest, jobUUID), formatAppAttemptDir(jobUUID, appAttemptId)); } @@ -88,9 +93,7 @@ public static Path getMagicJobAttemptPath(String jobUUID, */ public static Path getMagicJobPath(String jobUUID, Path dest) { - return new Path( - getMagicJobAttemptsPath(dest), - formatJobDir(jobUUID)); + return getMagicJobAttemptsPath(dest, jobUUID); } /** @@ -102,7 +105,7 @@ public static Path getMagicJobPath(String jobUUID, */ public static String formatJobDir( String jobUUID) { - return String.format("job-%s", jobUUID); + return JOB_ID_PREFIX + jobUUID; } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java index 9ded64eedc056..518831b7d4330 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java @@ -105,7 +105,6 @@ public void setupJob(JobContext context) throws IOException { Path jobPath = getJobPath(); final FileSystem destFS = getDestinationFS(jobPath, context.getConfiguration()); - destFS.delete(jobPath, true); destFS.mkdirs(jobPath); } } @@ -132,7 +131,7 @@ protected ActiveCommit listPendingUploadsToCommit( */ public void cleanupStagingDirs() { final Path out = getOutputPath(); - Path path = magicSubdir(out); + Path path = getMagicJobPath(getUUID(), out); try(DurationInfo ignored = new DurationInfo(LOG, true, "Deleting magic directory %s", path)) { Invoker.ignoreIOExceptions(LOG, "cleanup magic directory", path.toString(), diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md index 30fcf157862f0..29ca6ff578b3e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md @@ -1308,12 +1308,12 @@ so returning the special new stream. -This is done with a "magic" temporary directory name, `__magic`, to indicate that all files +This is done with a "MAGIC PATH" (where the filesystem knows to remap paths with prefix `__magic_job-${jobId}`) temporary directory name to indicate that all files created under this path are not to be completed during the stream write process. Directories created under the path will still be created —this allows job- and task-specific directories to be created for individual job and task attempts. -For example, the pattern `__magic/${jobID}/${taskId}` could be used to +For example, the pattern `${MAGIC PATH}/${jobID}/${taskId}` could be used to store pending commits to the final directory for that specific task. If that task is committed, all pending commit files stored in that path will be loaded and used to commit the final uploads. @@ -1322,19 +1322,19 @@ Consider a job with the final directory `/results/latest` The intermediate directory for the task 01 attempt 01 of job `job_400_1` would be - /results/latest/__magic/job_400_1/_task_01_01 + /results/latest/__magic_job-400/job_400_1/_task_01_01 This would be returned as the temp directory. When a client attempted to create the file -`/results/latest/__magic/job_400_1/task_01_01/latest.orc.lzo` , the S3A FS would initiate +`/results/latest/__magic_job-400/job_400_1/task_01_01/latest.orc.lzo` , the S3A FS would initiate a multipart request with the final destination of `/results/latest/latest.orc.lzo`. As data was written to the output stream, it would be incrementally uploaded as individual multipart PUT operations On `close()`, summary data would be written to the file -`/results/latest/__magic/job400_1/task_01_01/latest.orc.lzo.pending`. +`/results/latest/__magic_job-400/job400_1/task_01_01/latest.orc.lzo.pending`. This would contain the upload ID and all the parts and etags of uploaded data. A marker file is also created, so that code which verifies that a newly created file @@ -1358,7 +1358,7 @@ to the job attempt. 1. These are merged into to a single `Pendingset` structure. 1. Which is saved to a `.pendingset` file in the job attempt directory. 1. Finally, the task attempt directory is deleted. In the example, this -would be to `/results/latest/__magic/job400_1/task_01_01.pendingset`; +would be to `/results/latest/__magic_job-400/job400_1/task_01_01.pendingset`; A failure to load any of the single pending upload files (i.e. the file @@ -1386,9 +1386,9 @@ file. To allow tasks to generate data in subdirectories, a special filename `__base` will be used to provide an extra cue as to the final path. When mapping an output -path `/results/latest/__magic/job_400/task_01_01/__base/2017/2017-01-01.orc.lzo.pending` +path `/results/latest/__magic_job-400/job_400/task_01_01/__base/2017/2017-01-01.orc.lzo.pending` to a final destination path, the path will become `/results/latest/2017/2017-01-01.orc.lzo`. -That is: all directories between `__magic` and `__base` inclusive will be ignored. +That is: all directories between `__magic_job-400` and `__base` inclusive will be ignored. **Issues** @@ -1479,16 +1479,16 @@ Job drivers themselves may be preempted. One failure case is that the entire execution framework failed; a new process must identify outstanding jobs with pending work, and abort them, then delete -the appropriate `__magic` directories. +the appropriate `"MAGIC PATH"` directories. -This can be done either by scanning the directory tree for `__magic` directories +This can be done either by scanning the directory tree for `"MAGIC PATH"` directories and scanning underneath them, or by using the `listMultipartUploads()` call to list multipart uploads under a path, then cancel them. The most efficient solution may be to use `listMultipartUploads` to identify all outstanding request, and use that -to identify which requests to cancel, and where to scan for `__magic` directories. +to identify which requests to cancel, and where to scan for `"MAGIC PATH"` directories. This strategy should address scalability problems when working with repositories with many millions of objects —rather than list all keys searching for those -with `/__magic/**/*.pending` in their name, work backwards from the active uploads to +with `/${MAGIC PATH}/**/*.pending` in their name, work backwards from the active uploads to the directories with the data. We may also want to consider having a cleanup operation in the S3 CLI to @@ -1569,11 +1569,11 @@ a directory, then it is not going to work: the existing data will not be cleaned up. A cleanup operation would need to be included in the job commit, deleting all files in the destination directory which where not being overwritten. -1. It requires a path element, such as `__magic` which cannot be used +1. It requires a path element, such as `"MAGIC PATH"` which cannot be used for any purpose other than for the storage of pending commit data. 1. Unless extra code is added to every FS operation, it will still be possible -to manipulate files under the `__magic` tree. That's not bad, just potentially +to manipulate files under the `"MAGIC PATH"` tree. That's not bad, just potentially confusing. 1. As written data is not materialized until the commit, it will not be possible @@ -1692,9 +1692,9 @@ must be used, which means: the V2 classes. #### Resolved issues -**Magic Committer: Name of directory** +**Magic Committer: Directory Naming** -The design proposes the name `__magic` for the directory. HDFS and +The design proposes `__magic_job-` as the prefix for the magic paths of different jobs for the directory. HDFS and the various scanning routines always treat files and directories starting with `_` as temporary/excluded data. @@ -1705,14 +1705,14 @@ It is legal to create subdirectories in a task work directory, which will then be moved into the destination directory, retaining that directory tree. -That is, a if the task working dir is `dest/__magic/app1/task1/`, all files -under `dest/__magic/app1/task1/part-0000/` must end up under the path +That is, a if the task working dir is `dest/${MAGIC PATH}/app1/task1/`, all files +under `dest/${MAGIC PATH}/app1/task1/part-0000/` must end up under the path `dest/part-0000/`. This behavior is relied upon for the writing of intermediate map data in an MR job. -This means it is not simply enough to strip off all elements of under `__magic`, +This means it is not simply enough to strip off all elements of under ``"MAGIC PATH"``, it is critical to determine the base path. Proposed: use the special name `__base` as a marker of the base element for @@ -1918,9 +1918,9 @@ bandwidth and the data upload bandwidth. No use is made of the cluster filesystem; there are no risks there. -A malicious user with write access to the `__magic` directory could manipulate +A malicious user with write access to the ``"MAGIC PATH"`` directory could manipulate or delete the metadata of pending uploads, or potentially inject new work int -the commit. Having access to the `__magic` directory implies write access +the commit. Having access to the ``"MAGIC PATH"`` directory implies write access to the parent destination directory: a malicious user could just as easily manipulate the final output, without needing to attack the committer's intermediate files. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md index 4dd69e8efe4a2..4c14921c4b4aa 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md @@ -507,7 +507,7 @@ performance. ### FileSystem client setup -The S3A connector can recognize files created under paths with `__magic/` as a parent directory. +The S3A connector can recognize files created under paths with `${MAGIC PATH}/` as a parent directory. This allows it to handle those files in a special way, such as uploading to a different location and storing the information needed to complete pending multipart uploads. @@ -686,7 +686,7 @@ The examples below shows how these options can be configured in XML. ### Disabling magic committer path rewriting -The magic committer recognizes when files are created under paths with `__magic/` as a parent directory +The magic committer recognizes when files are created under paths with `${MAGIC PATH}/` as a parent directory and redirects the upload to a different location, adding the information needed to complete the upload in the job commit operation. @@ -708,10 +708,12 @@ You will not be able to use the Magic Committer if this option is disabled. ## Concurrent Jobs writing to the same destination -It is sometimes possible for multiple jobs to simultaneously write to the same destination path. +It is sometimes possible for multiple jobs to simultaneously write to the same destination path. To support +such use case, The "MAGIC PATH" for each job is unique of the format `__magic_job-${jobId}` so that +multiple job running simultaneously do not step into each other. Before attempting this, the committers must be set to not delete all incomplete uploads on job commit, -by setting `fs.s3a.committer.abort.pending.uploads` to `false` +by setting `fs.s3a.committer.abort.pending.uploads` to `false`. This is set to `false`by default ```xml @@ -740,9 +742,6 @@ For example, for any job executed through Hadoop MapReduce, the Job ID can be us ``` -Even with these settings, the outcome of concurrent jobs to the same destination is -inherently nondeterministic -use with caution. - ## Troubleshooting ### `Filesystem does not have support for 'magic' committer` @@ -805,7 +804,7 @@ files which are actually written to a different destination than their stated pa This message should not appear through the committer itself —it will fail with the error message in the previous section, but may arise -if other applications are attempting to create files under the path `/__magic/`. +if other applications are attempting to create files under the path `/${MAGIC PATH}/`. ### `FileOutputCommitter` appears to be still used (from logs or delays in commits) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java index 258c34b5cb84f..67c88039aad1b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java @@ -1419,7 +1419,7 @@ public void testOutputFormatIntegration() throws Throwable { recordWriter.write(iw, iw); long expectedLength = 4; Path dest = recordWriter.getDest(); - validateTaskAttemptPathDuringWrite(dest, expectedLength); + validateTaskAttemptPathDuringWrite(dest, expectedLength, jobData.getCommitter().getUUID()); recordWriter.close(tContext); // at this point validateTaskAttemptPathAfterWrite(dest, expectedLength); @@ -1833,10 +1833,12 @@ public void testS3ACommitterFactoryBinding() throws Throwable { * itself. * @param p path * @param expectedLength + * @param jobId job id * @throws IOException IO failure */ protected void validateTaskAttemptPathDuringWrite(Path p, - final long expectedLength) throws IOException { + final long expectedLength, + String jobId) throws IOException { } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java index aa44c171d7715..39265f1d8eab2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java @@ -330,10 +330,11 @@ protected void applyCustomConfigOptions(JobConf jobConf) throws IOException { * called after the base assertions have all passed. * @param destPath destination of work * @param successData loaded success data + * @param jobId job id * @throws Exception failure */ protected void customPostExecutionValidation(Path destPath, - SuccessData successData) + SuccessData successData, String jobId) throws Exception { } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java index cd703f96da8dd..573dd23b18c80 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java @@ -34,7 +34,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.STREAM_CAPABILITY_MAGIC_OUTPUT; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.XA_MAGIC_MARKER; import static org.apache.hadoop.fs.s3a.commit.impl.CommitOperations.extractMagicFileLength; @@ -52,6 +52,8 @@ public class CommitterTestHelper { */ private final S3AFileSystem fileSystem; + public static final String JOB_ID = "123"; + /** * Constructor. * @param fileSystem filesystem to work with. @@ -108,7 +110,7 @@ public void assertFileLacksMarkerHeader(Path path) throws IOException { */ public static Path makeMagic(Path destFile) { return new Path(destFile.getParent(), - MAGIC + '/' + BASE + "/" + destFile.getName()); + MAGIC_PATH_PREFIX + JOB_ID + '/' + BASE + "/" + destFile.getName()); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java index d0c86b738ca10..fbe1a0a3120bc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java @@ -51,7 +51,8 @@ import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_METADATA_REQUESTS; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_INITIATED; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_PUT_REQUESTS; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; +import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.JOB_ID; import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.assertIsMagicStream; import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.makeMagic; import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_FILES_LIST_OP; @@ -123,12 +124,12 @@ protected String fileSystemIOStats() { @Test public void testMagicMkdir() throws Throwable { - describe("Mkdirs __magic always skips dir marker deletion"); + describe("Mkdirs 'MAGIC PATH' always skips dir marker deletion"); S3AFileSystem fs = getFileSystem(); Path baseDir = methodPath(); // create dest dir marker, always fs.mkdirs(baseDir); - Path magicDir = new Path(baseDir, MAGIC); + Path magicDir = new Path(baseDir, MAGIC_PATH_PREFIX + JOB_ID); verifyMetrics(() -> { fs.mkdirs(magicDir); return fileSystemIOStats(); @@ -151,10 +152,10 @@ public void testMagicMkdir() throws Throwable { */ @Test public void testMagicMkdirs() throws Throwable { - describe("Mkdirs __magic/subdir always skips dir marker deletion"); + describe("Mkdirs __magic_job-/subdir always skips dir marker deletion"); S3AFileSystem fs = getFileSystem(); Path baseDir = methodPath(); - Path magicDir = new Path(baseDir, MAGIC); + Path magicDir = new Path(baseDir, MAGIC_PATH_PREFIX + JOB_ID); fs.delete(baseDir, true); Path magicSubdir = new Path(magicDir, "subdir"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java index a76a65be8bb3e..64def00fd2871 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java @@ -207,7 +207,7 @@ private CommitOperations newCommitOperations() */ private static Path makeMagic(Path destFile) { return new Path(destFile.getParent(), - MAGIC + '/' + destFile.getName()); + MAGIC_PATH_PREFIX + JOB_ID + '/' + destFile.getName()); } @Test @@ -279,7 +279,7 @@ public void testBaseRelativePath() throws Throwable { S3AFileSystem fs = getFileSystem(); Path destDir = methodSubPath("testBaseRelativePath"); fs.delete(destDir, true); - Path pendingBaseDir = new Path(destDir, MAGIC + "/child/" + BASE); + Path pendingBaseDir = new Path(destDir, MAGIC_PATH_PREFIX + JOB_ID + "/child/" + BASE); String child = "subdir/child.txt"; Path pendingChildPath = new Path(pendingBaseDir, child); Path expectedDestPath = new Path(destDir, child); @@ -334,7 +334,7 @@ public void testMarkerFileRename() /** * Create a file through the magic commit mechanism. - * @param filename file to create (with __magic path.) + * @param filename file to create (with "MAGIC PATH".) * @param data data to write * @throws Exception failure */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java index fdc4ec8058a6c..610491867f8d9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java @@ -38,16 +38,16 @@ public class TestMagicCommitPaths extends Assert { private static final List MAGIC_AT_ROOT = - list(MAGIC); + list(MAGIC_PATH_PREFIX); private static final List MAGIC_AT_ROOT_WITH_CHILD = - list(MAGIC, "child"); + list(MAGIC_PATH_PREFIX, "child"); private static final List MAGIC_WITH_CHILD = - list("parent", MAGIC, "child"); + list("parent", MAGIC_PATH_PREFIX, "child"); private static final List MAGIC_AT_WITHOUT_CHILD = - list("parent", MAGIC); + list("parent", MAGIC_PATH_PREFIX); private static final List DEEP_MAGIC = - list("parent1", "parent2", MAGIC, "child1", "child2"); + list("parent1", "parent2", MAGIC_PATH_PREFIX, "child1", "child2"); public static final String[] EMPTY = {}; @@ -161,40 +161,40 @@ public void testFinalDestinationNoMagic() { @Test public void testFinalDestinationMagic1() { assertEquals(l("first", "2"), - finalDestination(l("first", MAGIC, "2"))); + finalDestination(l("first", MAGIC_PATH_PREFIX, "2"))); } @Test public void testFinalDestinationMagic2() { assertEquals(l("first", "3.txt"), - finalDestination(l("first", MAGIC, "2", "3.txt"))); + finalDestination(l("first", MAGIC_PATH_PREFIX, "2", "3.txt"))); } @Test public void testFinalDestinationRootMagic2() { assertEquals(l("3.txt"), - finalDestination(l(MAGIC, "2", "3.txt"))); + finalDestination(l(MAGIC_PATH_PREFIX, "2", "3.txt"))); } @Test(expected = IllegalArgumentException.class) public void testFinalDestinationMagicNoChild() { - finalDestination(l(MAGIC)); + finalDestination(l(MAGIC_PATH_PREFIX)); } @Test public void testFinalDestinationBaseDirectChild() { - finalDestination(l(MAGIC, BASE, "3.txt")); + finalDestination(l(MAGIC_PATH_PREFIX, BASE, "3.txt")); } @Test(expected = IllegalArgumentException.class) public void testFinalDestinationBaseNoChild() { - assertEquals(l(), finalDestination(l(MAGIC, BASE))); + assertEquals(l(), finalDestination(l(MAGIC_PATH_PREFIX, BASE))); } @Test public void testFinalDestinationBaseSubdirsChild() { assertEquals(l("2", "3.txt"), - finalDestination(l(MAGIC, "4", BASE, "2", "3.txt"))); + finalDestination(l(MAGIC_PATH_PREFIX, "4", BASE, "2", "3.txt"))); } /** @@ -203,7 +203,7 @@ public void testFinalDestinationBaseSubdirsChild() { @Test public void testFinalDestinationIgnoresBaseBeforeMagic() { assertEquals(l(BASE, "home", "3.txt"), - finalDestination(l(BASE, "home", MAGIC, "2", "3.txt"))); + finalDestination(l(BASE, "home", MAGIC_PATH_PREFIX, "2", "3.txt"))); } /** varargs to array. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java index 622ead2617fd6..71be373b407ed 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java @@ -75,7 +75,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.lsR; import static org.apache.hadoop.fs.s3a.S3AUtils.applyLocatedFiles; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_TMP_PATH; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; import static org.apache.hadoop.fs.s3a.commit.CommitConstants._SUCCESS; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.FS_S3A_COMMITTER_UUID; import static org.apache.hadoop.fs.s3a.commit.staging.Paths.getMultipartUploadCommitsDirectory; @@ -332,7 +332,7 @@ public void test_200_execute() throws Exception { assertPathDoesNotExist("temporary dir should only be from" + " classic file committers", new Path(outputPath, CommitConstants.TEMPORARY)); - customPostExecutionValidation(outputPath, successData); + customPostExecutionValidation(outputPath, successData, jobID); } @Override @@ -343,8 +343,8 @@ protected void applyCustomConfigOptions(final JobConf jobConf) @Override protected void customPostExecutionValidation(final Path destPath, - final SuccessData successData) throws Exception { - committerTestBinding.validateResult(destPath, successData); + final SuccessData successData, String jobId) throws Exception { + committerTestBinding.validateResult(destPath, successData, jobId); } /** @@ -482,7 +482,7 @@ protected void applyCustomConfigOptions(JobConf jobConf) * @throws Exception failure */ protected void validateResult(Path destPath, - SuccessData successData) + SuccessData successData, String jobId) throws Exception { } @@ -576,7 +576,7 @@ private MagicCommitterTestBinding() { } /** - * The result validation here is that there isn't a __magic directory + * The result validation here is that there isn't a "MAGIC PATH" directory * any more. * @param destPath destination of work * @param successData loaded success data @@ -584,9 +584,9 @@ private MagicCommitterTestBinding() { */ @Override protected void validateResult(final Path destPath, - final SuccessData successData) + final SuccessData successData, final String jobId) throws Exception { - Path magicDir = new Path(destPath, MAGIC); + Path magicDir = new Path(destPath, MAGIC_PATH_PREFIX + jobId); // if an FNFE isn't raised on getFileStatus, list out the directory // tree diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java index 32cab03770c2d..fa963a4b97064 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java @@ -42,6 +42,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.listAndFilter; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; +import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.getMagicJobPath; import static org.apache.hadoop.util.functional.RemoteIterators.toList; /** @@ -74,7 +75,7 @@ public void setup() throws Exception { public void assertJobAbortCleanedUp(JobData jobData) throws Exception { // special handling of magic directory; harmless in staging - Path magicDir = new Path(getOutDir(), MAGIC); + Path magicDir = getMagicJobPath(jobData.getCommitter().getUUID(), getOutDir()); ContractTestUtils.assertPathDoesNotExist(getFileSystem(), "magic dir ", magicDir); super.assertJobAbortCleanedUp(jobData); @@ -94,11 +95,12 @@ public MagicS3GuardCommitter createFailingCommitter( } protected void validateTaskAttemptPathDuringWrite(Path p, - final long expectedLength) throws IOException { + final long expectedLength, + String jobId) throws IOException { String pathStr = p.toString(); Assertions.assertThat(pathStr) .describedAs("Magic path") - .contains(MAGIC); + .contains("/" + MAGIC_PATH_PREFIX + jobId + "/"); assertPathDoesNotExist("task attempt visible", p); } @@ -129,7 +131,7 @@ protected void validateTaskAttemptPathAfterWrite(Path marker, /** * The magic committer paths are always on S3, and always have - * "__magic" in the path. + * "MAGIC PATH" in the path. * @param committer committer instance * @param context task attempt context * @throws IOException IO failure @@ -143,11 +145,11 @@ protected void validateTaskAttemptWorkingDirectory( + " with committer " + committer, "s3a", wd.getScheme()); Assertions.assertThat(wd.getPath()) - .contains('/' + CommitConstants.MAGIC + '/'); + .contains("/" + MAGIC_PATH_PREFIX + committer.getUUID() + "/"); } /** - * Verify that the __magic path for the application/tasks use the + * Verify that the "MAGIC PATH" for the application/tasks use the * committer UUID to ensure uniqueness in the case of more than * one job writing to the same destination path. */ @@ -164,7 +166,7 @@ public void testCommittersPathsHaveUUID() throws Throwable { Assertions.assertThat(taskAttemptPath.toString()) .describedAs("task path of %s", committer) .contains(committer.getUUID()) - .contains(MAGIC) + .contains("/" + MAGIC_PATH_PREFIX + committer.getUUID() + "/") .doesNotContain(TEMP_DATA) .endsWith(BASE) .contains(ta0); @@ -176,7 +178,7 @@ public void testCommittersPathsHaveUUID() throws Throwable { .describedAs("Temp task path of %s", committer) .contains(committer.getUUID()) .contains(TEMP_DATA) - .doesNotContain(MAGIC) + .doesNotContain("/" + MAGIC_PATH_PREFIX + committer.getUUID() + "/") .doesNotContain(BASE) .contains(ta0); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java index e1440a497b0d9..613fab192874a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.fs.s3a.commit.CommitUtils; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; @@ -59,6 +58,8 @@ public class ITestS3AHugeMagicCommits extends AbstractSTestS3AHugeFiles { ITestS3AHugeMagicCommits.class); private static final int COMMITTER_THREADS = 64; + private static final String JOB_ID = "123"; + private Path magicDir; private Path jobDir; @@ -100,8 +101,8 @@ public void setup() throws Exception { // set up the paths for the commit operation finalDirectory = new Path(getScaleTestDir(), "commit"); - magicDir = new Path(finalDirectory, MAGIC); - jobDir = new Path(magicDir, "job_001"); + magicDir = new Path(finalDirectory, MAGIC_PATH_PREFIX + JOB_ID); + jobDir = new Path(magicDir, "job_" + JOB_ID); String filename = "commit.bin"; setHugefile(new Path(finalDirectory, filename)); magicOutputFile = new Path(jobDir, filename); @@ -141,7 +142,7 @@ public void test_030_postCreationAssertions() throws Throwable { ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); CommitOperations operations = new CommitOperations(fs); Path destDir = getHugefile().getParent(); - assertPathExists("Magic dir", new Path(destDir, CommitConstants.MAGIC)); + assertPathExists("Magic dir", new Path(destDir, MAGIC_PATH_PREFIX + JOB_ID)); String destDirKey = fs.pathToKey(destDir); Assertions.assertThat(listMultipartUploads(fs, destDirKey)) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java index dd62064f86d5b..81c3af812ab95 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java @@ -111,7 +111,8 @@ protected void expectJobCommitToFail(JobContext jContext, } protected void validateTaskAttemptPathDuringWrite(Path p, - final long expectedLength) throws IOException { + final long expectedLength, + String jobId) throws IOException { // this is expected to be local FS ContractTestUtils.assertPathExists(getLocalFS(), "task attempt", p); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java index 81bd8a5efe2e4..f6be1f75cfa68 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java @@ -74,7 +74,7 @@ public class TestHeaderProcessing extends HadoopTestBase { private HeaderProcessing headerProcessing; private static final String MAGIC_KEY - = "dest/__magic/job1/ta1/__base/output.csv"; + = "dest/__magic_job-1/job1/ta1/__base/output.csv"; private static final String MAGIC_FILE = "s3a://bucket/" + MAGIC_KEY; From d273c13ab535e96d2c971744129dabde710dae04 Mon Sep 17 00:00:00 2001 From: Jian Zhang <38941777+KeeProMise@users.noreply.github.com> Date: Thu, 21 Sep 2023 01:04:29 +0800 Subject: [PATCH 042/155] HDFS-17198. RBF: fix bug of getRepresentativeQuorum when records have same dateModified (#6096) --- .../store/records/MembershipState.java | 19 +++++++ .../store/TestStateStoreMembershipState.java | 52 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java index 80889b3d4aa4a..b05afd07f6ce0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java @@ -358,4 +358,23 @@ public long getDeletionMs() { public static void setDeletionMs(long time) { MembershipState.deletionMs = time; } + + /** + * First use the comparator of the BaseRecord to compare the date modified. + * If they are equal, compare their primary keys to ensure that MembershipStates + * with the same date modified but reported by different routers will not be judged as equal. + * + * @param record the MembershipState object to be compared. + * @return a negative integer, zero, or a positive integer as this object + * is less than, equal to, or greater than the specified object. + */ + @Override + public int compareTo(BaseRecord record) { + int order = super.compareTo(record); + if (order == 0) { + MembershipState other = (MembershipState) record; + return this.getPrimaryKey().compareTo(other.getPrimaryKey()); + } + return order; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java index 533a1d97daf7f..f7f0970bd364e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java @@ -223,6 +223,58 @@ public void testRegistrationMajorityQuorum() assertEquals(quorumEntry.getRouterId(), ROUTERS[3]); } + /** + * Fix getRepresentativeQuorum when records have same date modified time. + */ + @Test + public void testRegistrationMajorityQuorumEqDateModified() + throws IOException { + + // Populate the state store with a set of non-matching elements + // 1) ns0:nn0 - Standby (newest) + // 2) ns0:nn0 - Active + // 3) ns0:nn0 - Active + // 4) ns0:nn0 - Active + // (2), (3), (4) have the same date modified time + // Verify the selected entry is the newest majority opinion (4) + String ns = "ns0"; + String nn = "nn0"; + + long dateModified = Time.now(); + // Active - oldest + MembershipState report = createRegistration( + ns, nn, ROUTERS[1], FederationNamenodeServiceState.ACTIVE); + report.setDateModified(dateModified); + assertTrue(namenodeHeartbeat(report)); + + // Active - 2nd oldest + report = createRegistration( + ns, nn, ROUTERS[2], FederationNamenodeServiceState.ACTIVE); + report.setDateModified(dateModified); + assertTrue(namenodeHeartbeat(report)); + + // Active - 3rd oldest + report = createRegistration( + ns, nn, ROUTERS[3], FederationNamenodeServiceState.ACTIVE); + report.setDateModified(dateModified); + assertTrue(namenodeHeartbeat(report)); + + // standby - newest overall + report = createRegistration( + ns, nn, ROUTERS[0], FederationNamenodeServiceState.STANDBY); + assertTrue(namenodeHeartbeat(report)); + + // Load and calculate quorum + assertTrue(getStateStore().loadCache(MembershipStore.class, true)); + + // Verify quorum entry + MembershipState quorumEntry = getNamenodeRegistration( + report.getNameserviceId(), report.getNamenodeId()); + assertNotNull(quorumEntry); + // The name node status should be active + assertEquals(FederationNamenodeServiceState.ACTIVE, quorumEntry.getState()); + } + @Test public void testRegistrationQuorumExcludesExpired() throws InterruptedException, IOException { From 42b8e6faa71c6247b7295dc82658e5db18feb8cd Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 21 Sep 2023 08:11:59 +0800 Subject: [PATCH 043/155] YARN-11570. Add YARN_GLOBALPOLICYGENERATOR_HEAPSIZE to yarn-env for GPG. (#6086) --- hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 ++++ hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 3 +++ hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 8e86b26cf2f0b..6ef4e2c8e283e 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -110,6 +110,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" globalpolicygenerator) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator' + # Backwards compatibility + if [[ -n "${YARN_GLOBALPOLICYGENERATOR_HEAPSIZE}" ]]; then + HADOOP_HEAPSIZE_MAX="${YARN_GLOBALPOLICYGENERATOR_HEAPSIZE}" + fi ;; jar) HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 5207de816bc26..89dc5a3f3da53 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -273,6 +273,9 @@ goto :eof set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\globalpolicygenerator-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator set YARN_OPTS=%YARN_OPTS% %YARN_GLOBALPOLICYGENERATOR_OPTS% + if defined YARN_GLOBALPOLICYGENERATOR_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%YARN_GLOBALPOLICYGENERATOR_HEAPSIZE%m + ) goto :eof :routeradmin diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index 8d2ab065788a3..55d03668ded96 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -164,6 +164,11 @@ # Global Policy Generator specific parameters ### +# Specify the max heapsize for the Global Policy Generator. If no units are +# given, it will be assumed to be in MB. +# Default is the same as HADOOP_HEAPSIZE_MAX +#export YARN_GLOBALPOLICYGENERATOR_HEAPSIZE= + # Specify the JVM options to be used when starting the GPG. # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS From 27cb55182168b00dcbc3613af3475a2327997d0a Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 21 Sep 2023 01:01:44 -0800 Subject: [PATCH 044/155] HADOOP-18829. S3A prefetch LRU cache eviction metrics (#5893) Contributed by: Viraj Jasani --- .../fs/impl/prefetch/CachingBlockManager.java | 12 +++-- .../prefetch/EmptyPrefetchingStatistics.java | 5 ++ .../impl/prefetch/PrefetchingStatistics.java | 5 ++ .../prefetch/SingleFilePerBlockCache.java | 51 ++++++++++++------- .../fs/statistics/StreamStatisticNames.java | 12 +++++ .../fs/impl/prefetch/TestBlockCache.java | 6 +-- .../fs/statistics/IOStatisticAssertions.java | 37 ++++++++++++++ .../hadoop/fs/s3a/S3AInstrumentation.java | 11 +++- .../org/apache/hadoop/fs/s3a/Statistic.java | 10 ++++ .../s3a/prefetch/S3ACachingBlockManager.java | 3 +- .../impl/EmptyS3AStatisticsContext.java | 5 ++ .../s3a/ITestS3APrefetchingLruEviction.java | 12 +++++ .../fs/s3a/prefetch/S3APrefetchFakes.java | 5 +- 13 files changed, 145 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java index 4461c118625a1..407cd63004846 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java @@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.statistics.DurationTracker; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import static java.util.Objects.requireNonNull; @@ -111,8 +112,10 @@ public abstract class CachingBlockManager extends BlockManager { * @param conf the configuration. * @param localDirAllocator the local dir allocator instance. * @param maxBlocksCount max blocks count to be kept in cache at any time. + * @param trackerFactory tracker with statistics to update. * @throws IllegalArgumentException if bufferPoolSize is zero or negative. */ + @SuppressWarnings("checkstyle:parameternumber") public CachingBlockManager( ExecutorServiceFuturePool futurePool, BlockData blockData, @@ -120,7 +123,8 @@ public CachingBlockManager( PrefetchingStatistics prefetchingStatistics, Configuration conf, LocalDirAllocator localDirAllocator, - int maxBlocksCount) { + int maxBlocksCount, + DurationTrackerFactory trackerFactory) { super(blockData); Validate.checkPositiveInteger(bufferPoolSize, "bufferPoolSize"); @@ -136,7 +140,7 @@ public CachingBlockManager( if (this.getBlockData().getFileSize() > 0) { this.bufferPool = new BufferPool(bufferPoolSize, this.getBlockData().getBlockSize(), this.prefetchingStatistics); - this.cache = this.createCache(maxBlocksCount); + this.cache = this.createCache(maxBlocksCount, trackerFactory); } this.ops = new BlockOperations(); @@ -559,8 +563,8 @@ private void addToCacheAndRelease(BufferData data, Future blockFuture, } } - protected BlockCache createCache(int maxBlocksCount) { - return new SingleFilePerBlockCache(prefetchingStatistics, maxBlocksCount); + protected BlockCache createCache(int maxBlocksCount, DurationTrackerFactory trackerFactory) { + return new SingleFilePerBlockCache(prefetchingStatistics, maxBlocksCount, trackerFactory); } protected void cachePut(int blockNumber, ByteBuffer buffer) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java index 177ff7abab8b7..d20a32728681f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java @@ -57,6 +57,11 @@ public void blockRemovedFromFileCache() { } + @Override + public void blockEvictedFromFileCache() { + + } + @Override public void prefetchOperationCompleted() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java index 9ce2dec5889f1..d1ea600be720f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java @@ -42,6 +42,11 @@ public interface PrefetchingStatistics extends IOStatisticsSource { */ void blockRemovedFromFileCache(); + /** + * A block has been evicted from the file cache. + */ + void blockEvictedFromFileCache(); + /** * A prefetch operation has completed. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java index 6aacb4d7c8417..ecf7e38f4f7cc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java @@ -47,10 +47,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.LocalDirAllocator; +import org.apache.hadoop.fs.statistics.DurationTracker; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.util.Preconditions; import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull; +import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTrackerFactory; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_FILE_CACHE_EVICTION; /** * Provides functionality necessary for caching blocks of data read from FileSystem. @@ -99,6 +103,11 @@ public class SingleFilePerBlockCache implements BlockCache { private final PrefetchingStatistics prefetchingStatistics; + /** + * Duration tracker factory required to track the duration of some operations. + */ + private final DurationTrackerFactory trackerFactory; + /** * File attributes attached to any intermediate temporary file created during index creation. */ @@ -209,14 +218,19 @@ private void setNext(Entry next) { * * @param prefetchingStatistics statistics for this stream. * @param maxBlocksCount max blocks count to be kept in cache at any time. + * @param trackerFactory tracker with statistics to update */ - public SingleFilePerBlockCache(PrefetchingStatistics prefetchingStatistics, int maxBlocksCount) { + public SingleFilePerBlockCache(PrefetchingStatistics prefetchingStatistics, + int maxBlocksCount, + DurationTrackerFactory trackerFactory) { this.prefetchingStatistics = requireNonNull(prefetchingStatistics); this.closed = new AtomicBoolean(false); this.maxBlocksCount = maxBlocksCount; Preconditions.checkArgument(maxBlocksCount > 0, "maxBlocksCount should be more than 0"); blocks = new ConcurrentHashMap<>(); blocksLock = new ReentrantReadWriteLock(); + this.trackerFactory = trackerFactory != null + ? trackerFactory : stubDurationTrackerFactory(); } /** @@ -430,25 +444,28 @@ private void addToLinkedListAndEvictIfRequired(Entry entry) { * @param elementToPurge Block entry to evict. */ private void deleteBlockFileAndEvictCache(Entry elementToPurge) { - boolean lockAcquired = elementToPurge.takeLock(Entry.LockType.WRITE, - PrefetchConstants.PREFETCH_WRITE_LOCK_TIMEOUT, - PrefetchConstants.PREFETCH_WRITE_LOCK_TIMEOUT_UNIT); - if (!lockAcquired) { - LOG.error("Cache file {} deletion would not be attempted as write lock could not" - + " be acquired within {} {}", elementToPurge.path, + try (DurationTracker ignored = trackerFactory.trackDuration(STREAM_FILE_CACHE_EVICTION)) { + boolean lockAcquired = elementToPurge.takeLock(Entry.LockType.WRITE, PrefetchConstants.PREFETCH_WRITE_LOCK_TIMEOUT, PrefetchConstants.PREFETCH_WRITE_LOCK_TIMEOUT_UNIT); - } else { - try { - if (Files.deleteIfExists(elementToPurge.path)) { - entryListSize--; - prefetchingStatistics.blockRemovedFromFileCache(); - blocks.remove(elementToPurge.blockNumber); + if (!lockAcquired) { + LOG.error("Cache file {} deletion would not be attempted as write lock could not" + + " be acquired within {} {}", elementToPurge.path, + PrefetchConstants.PREFETCH_WRITE_LOCK_TIMEOUT, + PrefetchConstants.PREFETCH_WRITE_LOCK_TIMEOUT_UNIT); + } else { + try { + if (Files.deleteIfExists(elementToPurge.path)) { + entryListSize--; + prefetchingStatistics.blockRemovedFromFileCache(); + blocks.remove(elementToPurge.blockNumber); + prefetchingStatistics.blockEvictedFromFileCache(); + } + } catch (IOException e) { + LOG.warn("Failed to delete cache file {}", elementToPurge.path, e); + } finally { + elementToPurge.releaseLock(Entry.LockType.WRITE); } - } catch (IOException e) { - LOG.warn("Failed to delete cache file {}", elementToPurge.path, e); - } finally { - elementToPurge.releaseLock(Entry.LockType.WRITE); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java index 50bbf45505cec..85b82287e3774 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java @@ -455,6 +455,18 @@ public final class StreamStatisticNames { public static final String STREAM_READ_BLOCK_ACQUIRE_AND_READ = "stream_read_block_acquire_read"; + /** + * Total number of blocks evicted from the disk cache. + */ + public static final String STREAM_EVICT_BLOCKS_FROM_FILE_CACHE + = "stream_evict_blocks_from_cache"; + + /** + * Track duration of LRU cache eviction for disk cache. + */ + public static final String STREAM_FILE_CACHE_EVICTION + = "stream_file_cache_eviction"; + private StreamStatisticNames() { } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java index b32ce20a37354..a0c83a63c2248 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java @@ -45,7 +45,7 @@ public class TestBlockCache extends AbstractHadoopTestBase { public void testArgChecks() throws Exception { // Should not throw. BlockCache cache = - new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance(), 2); + new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance(), 2, null); ByteBuffer buffer = ByteBuffer.allocate(16); @@ -55,7 +55,7 @@ public void testArgChecks() throws Exception { intercept(NullPointerException.class, null, - () -> new SingleFilePerBlockCache(null, 2)); + () -> new SingleFilePerBlockCache(null, 2, null)); } @@ -63,7 +63,7 @@ public void testArgChecks() throws Exception { @Test public void testPutAndGet() throws Exception { BlockCache cache = - new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance(), 2); + new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance(), 2, null); ByteBuffer buffer1 = ByteBuffer.allocate(BUFFER_SIZE); for (byte i = 0; i < BUFFER_SIZE; i++) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java index 755599f0c390c..8396cbd258d1c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java @@ -176,6 +176,23 @@ public static long verifyStatisticCounterValue( verifyStatisticsNotNull(stats).counters(), value); } + /** + * Assert that two counters have similar values. + * + * @param stats statistics source. + * @param key1 statistic first key. + * @param key2 statistic second key. + */ + public static void verifyStatisticCounterValues( + final IOStatistics stats, + final String key1, + final String key2) { + verifyStatisticValues(COUNTER, + key1, + key2, + verifyStatisticsNotNull(stats).counters()); + } + /** * Assert that a gauge has an expected value. * @param stats statistics source @@ -258,6 +275,26 @@ private static E verifyStatisticValue( return statistic; } + /** + * Assert that the given two statistics have same values. + * + * @param type type of the statistics. + * @param key1 statistic first key. + * @param key2 statistic second key. + * @param map map to look up. + * @param type of map element. + */ + private static void verifyStatisticValues( + final String type, + final String key1, + final String key2, + final Map map) { + final E statistic1 = lookupStatistic(type, key1, map); + final E statistic2 = lookupStatistic(type, key2, map); + assertThat(statistic1) + .describedAs("%s named %s and %s named %s", type, key1, type, key2) + .isEqualTo(statistic2); + } /** * Assert that a given statistic has an expected value. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java index 48977b510578b..9d34457ab9443 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java @@ -886,7 +886,8 @@ private InputStreamStatistics( StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES, StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED, - StreamStatisticNames.STREAM_READ_VERSION_MISMATCHES) + StreamStatisticNames.STREAM_READ_VERSION_MISMATCHES, + StreamStatisticNames.STREAM_EVICT_BLOCKS_FROM_FILE_CACHE) .withGauges(STREAM_READ_GAUGE_INPUT_POLICY, STREAM_READ_BLOCKS_IN_FILE_CACHE.getSymbol(), STREAM_READ_ACTIVE_PREFETCH_OPERATIONS.getSymbol(), @@ -899,7 +900,8 @@ private InputStreamStatistics( StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED, StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS, StreamStatisticNames.STREAM_READ_REMOTE_BLOCK_READ, - StreamStatisticNames.STREAM_READ_BLOCK_ACQUIRE_AND_READ) + StreamStatisticNames.STREAM_READ_BLOCK_ACQUIRE_AND_READ, + StreamStatisticNames.STREAM_FILE_CACHE_EVICTION) .build(); setIOStatistics(st); aborted = st.getCounterReference( @@ -1395,6 +1397,11 @@ public void blockRemovedFromFileCache() { incAllGauges(STREAM_READ_BLOCKS_IN_FILE_CACHE, -1); } + @Override + public void blockEvictedFromFileCache() { + increment(StreamStatisticNames.STREAM_EVICT_BLOCKS_FROM_FILE_CACHE); + } + @Override public void prefetchOperationCompleted() { incAllGauges(STREAM_READ_ACTIVE_PREFETCH_OPERATIONS, -1); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index ae761fe270f46..b5f7fcbcd016d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -447,6 +447,16 @@ public enum Statistic { "Total queue duration of all block uploads", TYPE_DURATION), + /* Stream prefetch file cache eviction */ + STREAM_EVICT_BLOCKS_FROM_FILE_CACHE( + StreamStatisticNames.STREAM_EVICT_BLOCKS_FROM_FILE_CACHE, + "Count of blocks evicted from the disk cache", + TYPE_COUNTER), + STREAM_FILE_CACHE_EVICTION( + StreamStatisticNames.STREAM_FILE_CACHE_EVICTION, + "Duration of the eviction of an element from LRU cache that holds disk cache blocks", + TYPE_DURATION), + /* committer stats */ COMMITTER_COMMITS_CREATED( "committer_commits_created", diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java index a02922053aa39..e008de3a79d25 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java @@ -76,7 +76,8 @@ public S3ACachingBlockManager( streamStatistics, conf, localDirAllocator, - conf.getInt(PREFETCH_MAX_BLOCKS_COUNT, DEFAULT_PREFETCH_MAX_BLOCKS_COUNT)); + conf.getInt(PREFETCH_MAX_BLOCKS_COUNT, DEFAULT_PREFETCH_MAX_BLOCKS_COUNT), + streamStatistics); Validate.checkNotNull(reader, "reader"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java index 6454065b240c6..e47656efd1800 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java @@ -241,6 +241,11 @@ public void blockRemovedFromFileCache() { } + @Override + public void blockEvictedFromFileCache() { + + } + @Override public void executorAcquired(Duration timeInQueue) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java index a8211e24ce777..0fa08f37cf909 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java @@ -45,7 +45,11 @@ import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY; import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY; import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_MAX_BLOCKS_COUNT; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValues; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticGaugeValue; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_EVICT_BLOCKS_FROM_FILE_CACHE; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_FILE_CACHE_EVICTION; import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE; /** @@ -172,6 +176,14 @@ public void testSeeksWithLruEviction() throws Throwable { LambdaTestUtils.eventually(TIMEOUT_MILLIS, INTERVAL_MILLIS, () -> { LOG.info("IO stats: {}", ioStats); verifyStatisticGaugeValue(ioStats, STREAM_READ_BLOCKS_IN_FILE_CACHE, 0); + // stream_evict_blocks_from_cache is expected to be higher than 4, however we might face + // transient failures due to async prefetch get cancel issues. While TIMEOUT_MILLIS is + // sufficient wait time, consider re-running the test if stream_evict_blocks_from_cache + // value stays lower than 4. + assertThatStatisticCounter(ioStats, + STREAM_EVICT_BLOCKS_FROM_FILE_CACHE).isGreaterThanOrEqualTo(4); + verifyStatisticCounterValues(ioStats, STREAM_EVICT_BLOCKS_FROM_FILE_CACHE, + STREAM_FILE_CACHE_EVICTION); }); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java index 3bf9965861f8a..89477a9071973 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java @@ -59,6 +59,7 @@ import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.s3a.statistics.impl.CountingChangeTracker; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.util.functional.CallableRaisingIOE; @@ -308,7 +309,7 @@ public static class FakeS3FilePerBlockCache extends SingleFilePerBlockCache { public FakeS3FilePerBlockCache(int readDelay, int writeDelay) { super(new EmptyS3AStatisticsContext().newInputStreamStatistics(), - Constants.DEFAULT_PREFETCH_MAX_BLOCKS_COUNT); + Constants.DEFAULT_PREFETCH_MAX_BLOCKS_COUNT, null); this.files = new ConcurrentHashMap<>(); this.readDelay = readDelay; this.writeDelay = writeDelay; @@ -381,7 +382,7 @@ public int read(ByteBuffer buffer, long offset, int size) } @Override - protected BlockCache createCache(int maxBlocksCount) { + protected BlockCache createCache(int maxBlocksCount, DurationTrackerFactory trackerFactory) { final int readDelayMs = 50; final int writeDelayMs = 200; return new FakeS3FilePerBlockCache(readDelayMs, writeDelayMs); From cc66683b1a96596c6e30d9b644471aee0537d683 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Thu, 21 Sep 2023 21:45:30 +0800 Subject: [PATCH 045/155] HDFS-17184. Improve BlockReceiver to throws DiskOutOfSpaceException when initialize. (#6044). Contributed by Haiyang Hu. Signed-off-by: He Xiaoqiao --- .../apache/hadoop/hdfs/server/datanode/BlockReceiver.java | 8 ++++---- .../fsdataset/RoundRobinVolumeChoosingPolicy.java | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 1c077098a9d3a..4829e8c578635 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -57,6 +57,7 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; import org.apache.hadoop.tracing.Span; import org.apache.hadoop.tracing.Tracer; @@ -274,10 +275,9 @@ class BlockReceiver implements Closeable { if (isCreate) { BlockMetadataHeader.writeHeader(checksumOut, diskChecksum); } - } catch (ReplicaAlreadyExistsException bae) { - throw bae; - } catch (ReplicaNotFoundException bne) { - throw bne; + } catch (ReplicaAlreadyExistsException | ReplicaNotFoundException + | DiskOutOfSpaceException e) { + throw e; } catch(IOException ioe) { if (replicaInfo != null) { replicaInfo.releaseAllBytesReserved(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java index fe010b35a4cd6..fcf07a820dace 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java @@ -117,14 +117,13 @@ private V chooseVolume(final int curVolumeIndex, final List volumes, maxAvailable = availableVolumeSize; } + LOG.warn("The volume[{}] with the available space (={} B) is " + + "less than the block size (={} B).", volume.getBaseURI(), + availableVolumeSize, blockSize); if (curVolume == startVolume) { throw new DiskOutOfSpaceException("Out of space: " + "The volume with the most available space (=" + maxAvailable + " B) is less than the block size (=" + blockSize + " B)."); - } else { - LOG.warn("The volume[{}] with the available space (={} B) is " - + "less than the block size (={} B).", volume.getBaseURI(), - availableVolumeSize, blockSize); } } } From 0780710f25a36f4471942edfe7a7f396cacb226d Mon Sep 17 00:00:00 2001 From: K0K0V0K <109747532+K0K0V0K@users.noreply.github.com> Date: Fri, 22 Sep 2023 15:09:17 +0200 Subject: [PATCH 046/155] YARN-11567 - Aggregate container launch debug artifacts on error (#6053) --- .../hadoop/yarn/conf/YarnConfiguration.java | 4 ++ .../src/main/resources/yarn-default.xml | 15 +++++ .../server/nodemanager/ContainerExecutor.java | 25 ++++++-- .../launcher/TestContainerLaunch.java | 57 +++++++++++++++++++ 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ef06299fcfd8c..bbb1ed6f8a7de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -150,7 +150,11 @@ private static void addDeprecatedKeys() { public static final String NM_LOG_CONTAINER_DEBUG_INFO = YarnConfiguration.NM_PREFIX + "log-container-debug-info.enabled"; + public static final String NM_LOG_CONTAINER_DEBUG_INFO_ON_ERROR = + YarnConfiguration.NM_PREFIX + "log-container-debug-info-on-error.enabled"; + public static final boolean DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO = true; + public static final boolean DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO_ON_ERROR = false; //////////////////////////////// // IPC Configs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9697f7aa88c8d..9fa600db4b039 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1656,6 +1656,21 @@ true + + Generate additional logs about container launches, + if container returned with non-zero exit code. + Currently, this creates a copy of the launch script and lists the + directory contents of the container work dir. When listing directory + contents, we follow symlinks to a max-depth of 5(including symlinks + which point to outside the container work dir) which may lead to a + slowness in launching containers. + If yarn.nodemanager.log-container-debug-info.enabled is true, + it does not have effect on the behavior. + + yarn.nodemanager.log-container-debug-info-on-error.enabled + false + + Amount of physical memory, in MB, that can be allocated for containers. If set to -1 and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 65e8183f69990..3d0dca622c123 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -102,6 +102,7 @@ public abstract class ContainerExecutor implements Configurable { private String[] whitelistVars; private int exitCodeFileTimeout = YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_EXIT_FILE_TIMEOUT; + private int containerExitCode; @Override public void setConf(Configuration conf) { @@ -303,7 +304,7 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) if (pidPath == null) { LOG.warn("{} is not active, returning terminated error", containerId); - + containerExitCode = ExitCode.TERMINATED.getExitCode(); return ExitCode.TERMINATED.getExitCode(); } @@ -335,7 +336,7 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) while (!file.exists() && msecLeft >= 0) { if (!isContainerActive(containerId)) { LOG.info("{} was deactivated", containerId); - + containerExitCode = ExitCode.TERMINATED.getExitCode(); return ExitCode.TERMINATED.getExitCode(); } @@ -350,7 +351,9 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) } try { - return Integer.parseInt(FileUtils.readFileToString(file, StandardCharsets.UTF_8).trim()); + containerExitCode = Integer.parseInt( + FileUtils.readFileToString(file, StandardCharsets.UTF_8).trim()); + return containerExitCode; } catch (NumberFormatException e) { throw new IOException("Error parsing exit code from pid " + pid, e); } @@ -453,9 +456,7 @@ public void writeLaunchEnv(OutputStream out, Map environment, } // dump debugging information if configured - if (getConf() != null && - getConf().getBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO, - YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO)) { + if (shouldWriteDebugInformation(getConf())) { sb.echo("Copying debugging information"); sb.copyDebugInformation(new Path(outFilename), new Path(logDir, outFilename)); @@ -488,6 +489,18 @@ protected File[] readDirAsUser(String user, Path dir) { return new File(dir.toString()).listFiles(); } + private boolean shouldWriteDebugInformation(Configuration config) { + return config != null && ( + config.getBoolean( + YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO, + YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO + ) || ( + config.getBoolean( + YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO_ON_ERROR, + YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO_ON_ERROR + ) && containerExitCode != 0)); + } + /** * The container exit code. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index bd135ff519382..6971d34b9d814 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -1844,6 +1844,63 @@ public void testDebuggingInformation() throws IOException { } } + @Test + public void testDebuggingInformationOnError() throws IOException { + File shellFile = null; + File tempFile = null; + Configuration conf = new YarnConfiguration(); + try { + shellFile = Shell.appendScriptExtension(tmpDir, "hello"); + tempFile = Shell.appendScriptExtension(tmpDir, "temp"); + String testCommand = Shell.WINDOWS ? "@echo \"hello\"" : "echo \"hello\""; + PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile)); + FileUtil.setExecutable(shellFile, true); + writer.println(testCommand); + writer.close(); + Map> resources = new HashMap<>(); + Map env = new HashMap<>(); + List commands = new ArrayList<>(); + if (Shell.WINDOWS) { + commands.add("cmd"); + commands.add("/c"); + commands.add("\"" + shellFile.getAbsolutePath() + "\""); + } else { + commands.add("/bin/sh \\\"" + shellFile.getAbsolutePath() + "\\\""); + } + conf.setBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO, false); + conf.setBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO_ON_ERROR, true); + FileOutputStream fos = new FileOutputStream(tempFile); + ContainerExecutor exec = new DefaultContainerExecutor(); + exec.setConf(conf); + LinkedHashSet nmVars = new LinkedHashSet<>(); + exec.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), "user", + tempFile.getName(), nmVars); + fos.flush(); + fos.close(); + FileUtil.setExecutable(tempFile, true); + Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor( + new String[]{tempFile.getAbsolutePath()}, tmpDir); + shexc.execute(); + assertThat(shexc.getExitCode()).isZero(); + File directorInfo = + new File(localLogDir, ContainerExecutor.DIRECTORY_CONTENTS); + File scriptCopy = new File(localLogDir, tempFile.getName()); + Assert.assertFalse("Directory info file missing", + directorInfo.exists()); + Assert.assertFalse("Copy of launch script missing", + scriptCopy.exists()); + } finally { + // cleanup + if (shellFile != null && shellFile.exists()) { + shellFile.delete(); + } + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } + } + } + /** * Test container launch fault. * @throws Exception From 13c5825c00499374f35dc3d97f16f89bb0b3e364 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth <954799+szilard-nemeth@users.noreply.github.com> Date: Fri, 22 Sep 2023 20:00:50 +0200 Subject: [PATCH 047/155] YARN-11573. Add config option to make container allocation prefer nodes without reserved containers (#6098) --- .../ActivityDiagnosticConstant.java | 1 + .../CapacitySchedulerConfiguration.java | 11 ++ .../allocator/RegularContainerAllocator.java | 31 ++++- .../TestCapacitySchedulerMultiNodes.java | 110 ++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java index ecaa88438a546..3eb6dc24e0901 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityDiagnosticConstant.java @@ -107,4 +107,5 @@ public class ActivityDiagnosticConstant { public final static String NODE_CAN_NOT_FIND_CONTAINER_TO_BE_UNRESERVED_WHEN_NEEDED = "Node can't find a container to be unreserved when needed"; + public static final String NODE_HAS_BEEN_RESERVED = "Node has been reserved"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 417638516251a..5ab237d282a6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -154,6 +154,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur @Private public static final boolean DEFAULT_RESERVE_CONT_LOOK_ALL_NODES = true; + public static final String SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS = PREFIX + + "skip-allocate-on-nodes-with-reserved-containers"; + + @Private + public static final boolean DEFAULT_SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS = false; + @Private public static final String MAXIMUM_ALLOCATION = "maximum-allocation"; @@ -938,6 +944,11 @@ public boolean getReservationContinueLook() { DEFAULT_RESERVE_CONT_LOOK_ALL_NODES); } + public boolean getSkipAllocateOnNodesWithReservedContainer() { + return getBoolean(SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS, + DEFAULT_SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS); + } + private static String getAclKey(QueueACL acl) { return "acl_" + StringUtils.toLowerCase(acl.toString()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index c46b0341f74d0..f211b65b34f47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -24,8 +24,11 @@ import java.util.Optional; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityLevel; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.DiagnosticsCollector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.api.records.Container; @@ -850,9 +853,23 @@ private ContainerAllocation allocate(Resource clusterResource, Iterator iter = schedulingPS.getPreferredNodeIterator( candidates); + while (iter.hasNext()) { FiCaSchedulerNode node = iter.next(); + // Do not schedule if there are any reservations to fulfill on the node + if (iter.hasNext() && + node.getReservedContainer() != null && + isSkipAllocateOnNodesWithReservedContainer()) { + LOG.debug("Skipping scheduling on node {} since it has already been" + + " reserved by {}", node.getNodeID(), + node.getReservedContainer().getContainerId()); + ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation( + activitiesManager, node, application, schedulerKey, + ActivityDiagnosticConstant.NODE_HAS_BEEN_RESERVED, ActivityLevel.NODE); + continue; + } + if (reservedContainer == null) { result = preCheckForNodeCandidateSet(node, schedulingMode, resourceLimits, schedulerKey); @@ -894,7 +911,19 @@ private ContainerAllocation allocate(Resource clusterResource, return result; } - + + private boolean isSkipAllocateOnNodesWithReservedContainer() { + ResourceScheduler scheduler = rmContext.getScheduler(); + boolean skipAllocateOnNodesWithReservedContainer = false; + if (scheduler instanceof CapacityScheduler) { + CapacityScheduler cs = (CapacityScheduler) scheduler; + CapacitySchedulerConfiguration csConf = cs.getConfiguration(); + skipAllocateOnNodesWithReservedContainer = + csConf.getSkipAllocateOnNodesWithReservedContainer(); + } + return skipAllocateOnNodesWithReservedContainer; + } + @Override public CSAssignment assignContainers(Resource clusterResource, CandidateNodeSet candidates, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java index 48e8569f0f7c5..b08903893ffeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java @@ -25,12 +25,16 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.slf4j.Logger; @@ -478,4 +482,110 @@ public void testMultiNodeSorterAfterHeartbeatInterval() throws Exception { rm.stop(); } + @Test(timeout=30000) + public void testSkipAllocationOnNodeReservedByAnotherApp() throws Exception { + CapacitySchedulerConfiguration newConf = + new CapacitySchedulerConfiguration(conf); + newConf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); + newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + + ".resource-based.sorting-interval.ms", 0); + newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", 1.0f); + newConf.set(CapacitySchedulerConfiguration.SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS, + "true"); + MockRM rm1 = new MockRM(newConf); + + rm1.start(); + MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8 * GB); + MockNM nm2 = rm1.registerNode("127.0.0.2:1235", 8 * GB); + + // launch an app to queue, AM container should be launched in nm1 + RMApp app1 = MockRMAppSubmitter.submit(rm1, MockRMAppSubmissionData.Builder + .createWithMemory(5 * GB, rm1) + .withAppName("app") + .withUser("user") + .withQueue("default") + .build()); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + + // launch another app to queue, AM container should be launched in nm2 + RMApp app2 = MockRMAppSubmitter.submit(rm1, MockRMAppSubmissionData.Builder + .createWithMemory(5 * GB, rm1) + .withAppName("app") + .withUser("user") + .withQueue("default") + .build()); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2); + + CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); + RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); + FiCaSchedulerApp schedulerApp1 = + cs.getApplicationAttempt(am1.getApplicationAttemptId()); + FiCaSchedulerApp schedulerApp2 = + cs.getApplicationAttempt(am2.getApplicationAttemptId()); + + // Ask a container with 4 GB memory size for app1, + am1.allocate("*", 4 * GB, 1, new ArrayList<>()); + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + + + // Check containers of app1 and app2. + Set reservedContainers = checkReservedContainers(cs, + rm1.getRMContext().getRMNodes(), 1); + Assert.assertEquals(1, reservedContainers.size()); + RMNode nodeWithReservedContainer = reservedContainers.iterator().next(); + LOG.debug("Reserved container on: {}", nodeWithReservedContainer); + + //Move reservation to nm1 for easier testing + if (nodeWithReservedContainer.getNodeID().getHost().startsWith("127.0.0.2")) { + moveReservation(cs, rm1, nm1, nm2, am1); + } + Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer()); + Assert.assertNull(cs.getNode(nm2.getNodeId()).getReservedContainer()); + + Assert.assertEquals(1, schedulerApp1.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp2.getLiveContainers().size()); + Assert.assertEquals(1, schedulerApp1.getReservedContainers().size()); + + //Make sure to have available headroom on the child queue, + // see: RegularContainerAllocator#checkHeadroom, + //that can make RegularContainerAllocator.preCheckForNodeCandidateSet to return + // ContainerAllocation.QUEUE_SKIPPED + MockNM nm3 = rm1.registerNode("127.0.0.3:1235", 3 * GB); + + //Allocate a container for app2, we expect this to be allocated on nm2 as + // nm1 has a reservation for another app + am2.allocate("*", 4 * GB, 1, new ArrayList<>()); + cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); + Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer()); + Assert.assertNotNull(cs.getNode(nm2.getNodeId()).getReservedContainer()); + + rm1.close(); + } + + private static void moveReservation(CapacityScheduler cs, + MockRM rm1, MockNM nm1, MockNM nm2, MockAM am1) { + RMNode sourceNode = rm1.getRMContext().getRMNodes().get(nm2.getNodeId()); + RMNode targetNode = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); + SchedulerApplicationAttempt firstSchedulerAppAttempt = + cs.getApplicationAttempt(am1.getApplicationAttemptId()); + FiCaSchedulerApp app = (FiCaSchedulerApp)firstSchedulerAppAttempt; + RMContainer reservedContainer = cs.getNode(sourceNode.getNodeID()).getReservedContainer(); + LOG.debug("Moving reservation"); + app.moveReservation(reservedContainer, + cs.getNode(sourceNode.getNodeID()), cs.getNode(targetNode.getNodeID())); + } + + private static Set checkReservedContainers(CapacityScheduler cs, + ConcurrentMap rmNodes, int expectedNumberOfContainers) { + Set result = new HashSet<>(); + for (Map.Entry entry : rmNodes.entrySet()) { + if (cs.getNode(entry.getKey()).getReservedContainer() != null) { + result.add(entry.getValue()); + } + } + + Assert.assertEquals(expectedNumberOfContainers, result.size()); + return result; + } } From ecee022e49269bd6612d32695c6b46a95c6bf11e Mon Sep 17 00:00:00 2001 From: zhangshuyan <81411509+zhangshuyan0@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:01:25 +0800 Subject: [PATCH 048/155] HDFS-17197. Show file replication when listing corrupt files. (#6095). Contributed by Shuyan Zhang. Signed-off-by: He Xiaoqiao --- .../hdfs/server/namenode/FSNamesystem.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index eb8022dc63451..d9b165f96ee0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6131,15 +6131,20 @@ void releaseBackupNode(NamenodeRegistration registration) static class CorruptFileBlockInfo { final String path; final Block block; + private final int replication; + private final String ecPolicy; - public CorruptFileBlockInfo(String p, Block b) { + CorruptFileBlockInfo(String p, Block b, int r, String ec) { path = p; block = b; + replication = r; + ecPolicy = ec; } @Override public String toString() { - return block.getBlockName() + "\t" + path; + return block.getBlockName() + "\t" + + (replication == -1 ? ecPolicy : replication) + "\t" + path; } } /** @@ -6195,7 +6200,21 @@ Collection listCorruptFileBlocks(String path, if (inode != null) { String src = inode.getFullPathName(); if (isParentEntry(src, path)) { - corruptFiles.add(new CorruptFileBlockInfo(src, blk)); + int repl = -1; + String ecPolicyName = null; + if (inode.isFile()) { + if (inode.asFile().isStriped()) { + ErasureCodingPolicy ecPolicy = + ErasureCodingPolicyManager.getInstance() + .getByID(inode.asFile().getErasureCodingPolicyID()); + if (ecPolicy != null) { + ecPolicyName = ecPolicy.getName(); + } + } else { + repl = inode.asFile().getFileReplication(); + } + } + corruptFiles.add(new CorruptFileBlockInfo(src, blk, repl, ecPolicyName)); count++; if (count >= maxCorruptFileBlocksReturn) break; From bf9975a1b315942ad34928a980b0fc5544361e36 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:23:02 +0800 Subject: [PATCH 049/155] YARN-9586. Need more doc for yarn.federation.policy-manager-params when LoadBasedRouterPolicy is used. (#6085) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../server/router/TestRouterServerUtil.java | 39 +++++++++++++++++++ .../src/test/resources/yarn-site.xml | 4 ++ .../src/site/markdown/Federation.md | 20 ++++++++++ 3 files changed, 63 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java index e82f67d12d560..dcf3bd5bc0eeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java @@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo; @@ -33,7 +36,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import static org.apache.hadoop.yarn.server.router.webapp.TestFederationInterceptorREST.getReservationSubmissionRequestInfo; import static org.junit.Assert.assertEquals; @@ -122,4 +128,37 @@ public void testConvertReservationDefinitionEmpty() throws Exception { "definitionInfo Or ReservationRequests is Null.", () -> RouterServerUtil.convertReservationDefinition(definitionInfo3)); } + + @Test + public void testLoadFederationPolicyManager() throws Exception { + + // In this unit test, we have configured the yarn-site.xml file with + // the yarn.federation.policy-manager-params parameter, + // and subsequently, we parse this parameter. + // We have configured two subclusters, SC-1 and SC-2, + // with routerPolicyWeights set to SC-1:0.7 and SC-2:0.3, + // and amrmPolicyWeights set to SC-1:0.6 and SC-2:0.4. + // Additionally, headroomAlpha is set to 1.0. + + YarnConfiguration conf = new YarnConfiguration(); + String defaultPolicyParamString = conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS, + YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS); + assertNotNull(defaultPolicyParamString); + ByteBuffer defaultPolicyParam = ByteBuffer.wrap( + defaultPolicyParamString.getBytes(StandardCharsets.UTF_8)); + WeightedPolicyInfo policyInfo = WeightedPolicyInfo.fromByteBuffer(defaultPolicyParam); + float headroomAlpha = policyInfo.getHeadroomAlpha(); + Map routerPolicyWeights = policyInfo.getRouterPolicyWeights(); + Map amrmPolicyWeights = policyInfo.getAMRMPolicyWeights(); + + SubClusterIdInfo sc1 = new SubClusterIdInfo("SC-1"); + SubClusterIdInfo sc2 = new SubClusterIdInfo("SC-2"); + + assertEquals(1.0, headroomAlpha, 0.001); + assertEquals(0.7, routerPolicyWeights.get(sc1), 0.001); + assertEquals(0.3, routerPolicyWeights.get(sc2), 0.001); + + assertEquals(0.6, amrmPolicyWeights.get(sc1), 0.001); + assertEquals(0.4, amrmPolicyWeights.get(sc2), 0.001); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml index 84d4171c79cd4..55461680b5b84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml @@ -39,4 +39,8 @@ yarn.resourcemanager.resource-profiles.source-file profiles/sample-profiles-1.json + + yarn.federation.policy-manager-params + {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index abe2384a39283..a61325238447f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -235,6 +235,26 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. |`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. | |`yarn.federation.machine-list` | `` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example:

    node1, subcluster1, rack1
    node2, subcluster2, rack1
    node3, subcluster3, rack2
    node4, subcluster3, rack2 | +- yarn.federation.policy-manager-params + + To configure the `yarn.federation.policy-manager-params` parameter, which represents the weight policy for the default queue, + and where the relevant information will be parsed as `WeightedPolicyInfo`. + + We can use the following JSON format for configuration: + + ```xml + + yarn.federation.policy-manager-params + {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} + + ``` + + This JSON configuration allows you to define the weight policy for default queue, where: + + - The `routerPolicyWeights` section specifies the weightings for router policies. For instance, with a weight of `0.3` assigned to `SC-2` and `0.7` assigned to `SC-1`, this configuration will allocate `30%` of submitted application requests to `SC-2` and `70%` to `SC-1`. + - The `amrmPolicyWeights` represents the allocation ratios for Application Master when request containers from different subclusters' RM. For instance, when an AM requests containers, it will request `40%` of the containers from `SC-2` and `60%` of the containers from `SC-1`. + - The `headroomAlpha` used by policies that balance weight-based and load-based considerations in their decisions. For policies that use this parameter, values close to 1 indicate that most of the decision should be based on currently observed headroom from various sub-clusters, values close to zero, indicate that the decision should be mostly based on weights and practically ignore current load. + How to configure the policy-manager -------------------- From f51162d70b44dee701ad3d2df7b5c4b1d24bc7f9 Mon Sep 17 00:00:00 2001 From: Benjamin Teke Date: Mon, 25 Sep 2023 16:24:03 +0200 Subject: [PATCH 050/155] YARN-11514. Extend SchedulerResponse with capacityVector (#5989) Co-authored-by: Benjamin Teke --- .../capacity/QueueCapacityVector.java | 4 + .../webapp/dao/CapacitySchedulerInfo.java | 7 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 8 +- .../dao/CapacitySchedulerQueueInfo.java | 12 +- .../dao/PartitionQueueCapacitiesInfo.java | 15 +- .../webapp/dao/QueueCapacitiesInfo.java | 17 +- .../dao/QueueCapacityVectorEntryInfo.java | 53 +++++ .../webapp/dao/QueueCapacityVectorInfo.java | 64 ++++++ .../TestRMWebServicesForCSWithPartitions.java | 2 +- .../webapp/dynamic-testAbsoluteMode-0.json | 80 +++++++ .../webapp/dynamic-testAbsoluteMode-16.json | 80 +++++++ .../webapp/dynamic-testAbsoluteMode-32.json | 80 +++++++ .../dynamic-testAbsoluteMode-legacy-0.json | 80 +++++++ .../webapp/dynamic-testPercentageMode-0.json | 80 +++++++ .../webapp/dynamic-testPercentageMode-16.json | 80 +++++++ .../webapp/dynamic-testPercentageMode-32.json | 80 +++++++ .../dynamic-testPercentageMode-legacy-0.json | 80 +++++++ .../webapp/dynamic-testWeightMode-0.json | 80 +++++++ .../webapp/dynamic-testWeightMode-16.json | 80 +++++++ .../webapp/dynamic-testWeightMode-32.json | 80 +++++++ .../dynamic-testWeightMode-after-aqc.json | 200 ++++++++++++++++++ .../dynamic-testWeightMode-before-aqc.json | 80 +++++++ .../dynamic-testWeightMode-legacy-0.json | 80 +++++++ .../dynamic-testWeightMode-legacy-16.json | 80 +++++++ .../dynamic-testWeightMode-legacy-32.json | 80 +++++++ ...namic-testWeightMode-legacy-after-aqc.json | 200 ++++++++++++++++++ ...amic-testWeightMode-legacy-before-aqc.json | 80 +++++++ ...-testSchedulerAbsoluteAndPercentage-0.json | 80 +++++++ ...testSchedulerAbsoluteAndPercentage-16.json | 80 +++++++ ...testSchedulerAbsoluteAndPercentage-32.json | 80 +++++++ ...dulerAbsoluteAndPercentageAndWeight-0.json | 80 +++++++ ...ulerAbsoluteAndPercentageAndWeight-16.json | 80 +++++++ ...ulerAbsoluteAndPercentageAndWeight-32.json | 80 +++++++ ...AbsoluteAndPercentageAndWeightMixed-0.json | 80 +++++++ ...bsoluteAndPercentageAndWeightMixed-16.json | 80 +++++++ ...bsoluteAndPercentageAndWeightMixed-32.json | 80 +++++++ ...ixed-testSchedulerAbsoluteAndWeight-0.json | 80 +++++++ ...xed-testSchedulerAbsoluteAndWeight-16.json | 80 +++++++ ...xed-testSchedulerAbsoluteAndWeight-32.json | 80 +++++++ ...ed-testSchedulerPercentageAndWeight-0.json | 80 +++++++ ...d-testSchedulerPercentageAndWeight-16.json | 80 +++++++ ...d-testSchedulerPercentageAndWeight-32.json | 80 +++++++ ...sponse-AbsoluteModeLegacyAutoCreation.json | 50 +++++ ...scheduler-response-NodeLabelDefaultAPI.xml | 55 +++++ .../scheduler-response-PerUserResources.json | 50 +++++ .../scheduler-response-PerUserResources.xml | 55 +++++ ...onse-PercentageModeLegacyAutoCreation.json | 40 ++++ .../resources/webapp/scheduler-response.json | 50 +++++ .../resources/webapp/scheduler-response.xml | 55 +++++ 49 files changed, 3389 insertions(+), 28 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorEntryInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java index 1cbc774395250..18252a6a9f05c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java @@ -281,5 +281,9 @@ public double getResourceValue() { public String getResourceName() { return resourceName; } + + public String getResourceWithPostfix() { + return resourceValue + vectorResourceType.getPostfix(); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 9e3c15b0273bc..5d5b3f26cc1f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo.getSortedQueueAclInfoList; @XmlRootElement(name = "capacityScheduler") @@ -47,6 +48,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float maxCapacity; protected float weight; protected float normalizedWeight; + protected QueueCapacityVectorInfo queueCapacityVectorInfo; protected String queueName; private String queuePath; protected int maxParallelApps; @@ -78,6 +80,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { this.queuePath = parent.getQueuePath(); this.usedCapacity = parent.getUsedCapacity() * 100; this.capacity = parent.getCapacity() * 100; + this.queueCapacityVectorInfo = new QueueCapacityVectorInfo( + parent.getConfiguredCapacityVector(NO_LABEL)); float max = parent.getMaximumCapacity(); if (max < EPSILON || max > 1f) max = 1f; @@ -86,8 +90,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight(); this.maxParallelApps = parent.getMaxParallelApps(); - capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), - parent.getQueueResourceQuotas(), false); + capacities = new QueueCapacitiesInfo(parent, false); queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); maximumAllocation = new ResourceInfo(parent.getMaximumAllocation()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index c6418ed063715..4a929940ea023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -25,13 +25,12 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @XmlRootElement @@ -105,9 +104,8 @@ protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { } @Override - protected void populateQueueCapacities(QueueCapacities qCapacities, - QueueResourceQuotas qResQuotas) { - capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas); + protected void populateQueueCapacities(CSQueue queue) { + capacities = new QueueCapacitiesInfo(queue, true); } public int getNumActiveApplications() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index b79a71712e53d..1b5a3714d68c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -33,7 +33,6 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.security.AccessType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue; @@ -41,7 +40,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; @XmlRootElement @@ -136,9 +134,7 @@ public class CapacitySchedulerQueueInfo { nodeLabels.addAll(labelSet); Collections.sort(nodeLabels); } - QueueCapacities qCapacities = q.getQueueCapacities(); - QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas(); - populateQueueCapacities(qCapacities, qResQuotas); + populateQueueCapacities(q); mode = CapacitySchedulerInfoHelper.getMode(q); queueType = CapacitySchedulerInfoHelper.getQueueType(q); @@ -210,10 +206,8 @@ protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { resources = new ResourcesInfo(queueResourceUsage, false); } - protected void populateQueueCapacities(QueueCapacities qCapacities, - QueueResourceQuotas qResQuotas) { - capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas, - false); + protected void populateQueueCapacities(CSQueue queue) { + capacities = new QueueCapacitiesInfo(queue, false); } public float getCapacity() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java index 1b66808356d87..b9ccf0ecb7791 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java @@ -32,6 +32,7 @@ public class PartitionQueueCapacitiesInfo { private String partitionName; + private QueueCapacityVectorInfo queueCapacityVectorInfo; private float capacity; private float usedCapacity; private float maxCapacity = 100; @@ -49,13 +50,15 @@ public class PartitionQueueCapacitiesInfo { public PartitionQueueCapacitiesInfo() { } - public PartitionQueueCapacitiesInfo(String partitionName, float capacity, - float usedCapacity, float maxCapacity, float absCapacity, + public PartitionQueueCapacitiesInfo(String partitionName, + QueueCapacityVectorInfo queueCapacityVectorInfo, + float capacity, float usedCapacity, float maxCapacity, float absCapacity, float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage, float weight, float normalizedWeight, Resource confMinRes, Resource confMaxRes, Resource effMinRes, Resource effMaxRes) { super(); + this.queueCapacityVectorInfo = queueCapacityVectorInfo; this.partitionName = partitionName; this.capacity = capacity; this.usedCapacity = usedCapacity; @@ -72,6 +75,14 @@ public PartitionQueueCapacitiesInfo(String partitionName, float capacity, this.effectiveMaxResource = new ResourceInfo(effMaxRes); } + public QueueCapacityVectorInfo getQueueCapacityVectorInfo() { + return queueCapacityVectorInfo; + } + + public void setQueueCapacityVectorInfo(QueueCapacityVectorInfo queueCapacityVectorInfo) { + this.queueCapacityVectorInfo = queueCapacityVectorInfo; + } + public float getCapacity() { return capacity; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java index 3c29f505d8fa1..293966b5b8600 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; /** @@ -39,12 +40,13 @@ public class QueueCapacitiesInfo { public QueueCapacitiesInfo() { } - public QueueCapacitiesInfo(QueueCapacities capacities, - QueueResourceQuotas resourceQuotas, - boolean considerAMUsage) { + public QueueCapacitiesInfo(CSQueue queue, boolean considerAMUsage) { + QueueCapacities capacities = queue.getQueueCapacities(); + QueueResourceQuotas resourceQuotas = queue.getQueueResourceQuotas(); if (capacities == null) { return; } + QueueCapacityVectorInfo queueCapacityVectorInfo; float capacity; float usedCapacity; float maxCapacity; @@ -55,6 +57,8 @@ public QueueCapacitiesInfo(QueueCapacities capacities, float weight; float normalizedWeight; for (String partitionName : capacities.getExistingNodeLabels()) { + queueCapacityVectorInfo = new QueueCapacityVectorInfo( + queue.getConfiguredCapacityVector(partitionName)); usedCapacity = capacities.getUsedCapacity(partitionName) * 100; capacity = capacities.getCapacity(partitionName) * 100; maxCapacity = capacities.getMaximumCapacity(partitionName); @@ -72,7 +76,7 @@ public QueueCapacitiesInfo(QueueCapacities capacities, weight = capacities.getWeight(partitionName); normalizedWeight = capacities.getNormalizedWeight(partitionName); queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo( - partitionName, capacity, usedCapacity, maxCapacity, absCapacity, + partitionName, queueCapacityVectorInfo, capacity, usedCapacity, maxCapacity, absCapacity, absUsedCapacity, absMaxCapacity, considerAMUsage ? maxAMLimitPercentage : 0f, weight, normalizedWeight, @@ -83,11 +87,6 @@ public QueueCapacitiesInfo(QueueCapacities capacities, } } - public QueueCapacitiesInfo(QueueCapacities capacities, - QueueResourceQuotas resourceQuotas) { - this(capacities, resourceQuotas, true); - } - public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) { queueCapacitiesByPartition.add(partitionQueueCapacitiesInfo); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorEntryInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorEntryInfo.java new file mode 100644 index 0000000000000..6582f5aeb2bd5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorEntryInfo.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueCapacityVectorEntryInfo { + private String resourceName; + private String resourceValue; + + public QueueCapacityVectorEntryInfo() { + } + + public QueueCapacityVectorEntryInfo(String resourceName, String resourceValue) { + this.resourceName = resourceName; + this.resourceValue = resourceValue; + } + + public String getResourceName() { + return this.resourceName; + } + + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } + + public String getResourceValue() { + return this.resourceValue; + } + + public void setResourceValue(String resourceValue) { + this.resourceValue = resourceValue; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorInfo.java new file mode 100644 index 0000000000000..0262fdc8cde6b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacityVectorInfo.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import java.util.ArrayList; +import java.util.List; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueCapacityVectorInfo { + private String configuredCapacityVector; + private List capacityVectorEntries; + + public QueueCapacityVectorInfo() { + } + + public QueueCapacityVectorInfo(QueueCapacityVector queueCapacityVector) { + this.configuredCapacityVector = queueCapacityVector.toString(); + this.capacityVectorEntries = new ArrayList<>(); + for (QueueCapacityVector.QueueCapacityVectorEntry + queueCapacityVectorEntry : queueCapacityVector) { + this.capacityVectorEntries.add( + new QueueCapacityVectorEntryInfo(queueCapacityVectorEntry.getResourceName(), + queueCapacityVectorEntry.getResourceWithPostfix())); + } + } + + public String getConfiguredCapacityVector() { + return configuredCapacityVector; + } + + public void setConfiguredCapacityVector(String configuredCapacityVector) { + this.configuredCapacityVector = configuredCapacityVector; + } + + public List getCapacityVectorEntries() { + return capacityVectorEntries; + } + + public void setCapacityVectorEntries(List capacityVectorEntries) { + this.capacityVectorEntries = capacityVectorEntries; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 254db72b63b6e..771f5c76681a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -580,7 +580,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 24, info.length()); + assertEquals("incorrect number of elements", 25, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-0.json index 47720a8c45683..0d0d8c72edb13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4096.0,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4096.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-16.json index c9e36de17c619..f2b2d746eaded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4096.0,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4096.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-32.json index cc246bc90da67..ddcd26d89608e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4096.0,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4096.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-legacy-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-legacy-0.json index 1bd2af2aee203..79ce11895dd1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-legacy-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testAbsoluteMode-legacy-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4096.0,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4096.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-0.json index d2fe9a51d5c98..c10da83f8473e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-16.json index 12dae42119721..81618a56c2363 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-32.json index babf61f3996b9..0c0a3fdb67202 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-legacy-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-legacy-0.json index ef10de274c065..11bc093512088 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-legacy-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testPercentageMode-legacy-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-0.json index 4079bb6563b24..56514f39691e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-16.json index c3a1fea2ef11d..063f3e06bf5df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-32.json index 86295528a12ab..2c9dfce0b015a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-after-aqc.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-after-aqc.json index deafe78ef03e2..200e321a0db08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-after-aqc.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-after-aqc.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -544,6 +564,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=10.0w,vcores=10.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "10.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "10.0w" + } ] + }, "capacity" : 41.666664, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=10.0w,vcores=10.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "10.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "10.0w" + } ] + }, "capacity" : 41.666664, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1498,6 +1538,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1967,6 +2017,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2422,6 +2482,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 4.166667, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2767,6 +2837,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -3222,6 +3302,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 4.166667, "usedCapacity" : 0, "maxCapacity" : 100, @@ -3562,6 +3652,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -4017,6 +4117,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 4.166667, "usedCapacity" : 0, "maxCapacity" : 100, @@ -4373,6 +4483,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -4828,6 +4948,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -5138,6 +5268,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 4.166667, "usedCapacity" : 0, "maxCapacity" : 100, @@ -5448,6 +5588,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -5793,6 +5943,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -6262,6 +6422,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -6731,6 +6901,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -7186,6 +7366,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -7460,6 +7650,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-before-aqc.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-before-aqc.json index 9be270ec5069f..771f0dd48a4ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-before-aqc.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-before-aqc.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -873,6 +903,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1342,6 +1382,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1811,6 +1861,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2266,6 +2326,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2540,6 +2610,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-0.json index edcf8ad08e665..ed79fd278f9c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-16.json index 5e0c3e372083b..2f888d37f3190 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-32.json index 9d0b99f930580..ade5769fe800f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-after-aqc.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-after-aqc.json index 906138d9a346a..94e19f2296b0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-after-aqc.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-after-aqc.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -544,6 +564,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=10.0w,vcores=10.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "10.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "10.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=10.0w,vcores=10.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "10.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "10.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1498,6 +1538,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1967,6 +2017,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2422,6 +2482,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2767,6 +2837,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -3222,6 +3302,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -3562,6 +3652,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -4017,6 +4117,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -4373,6 +4483,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -4828,6 +4948,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -5138,6 +5268,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -5448,6 +5588,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -5793,6 +5943,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -6262,6 +6422,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -6731,6 +6901,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -7186,6 +7366,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -7460,6 +7650,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-before-aqc.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-before-aqc.json index bc7edae0df1bc..61c3e7c6ec41d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-before-aqc.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/dynamic-testWeightMode-legacy-before-aqc.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4.0w,vcores=4.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -873,6 +903,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1342,6 +1382,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2.0w,vcores=2.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1811,6 +1861,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.0w,vcores=12.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2266,6 +2326,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16.0w,vcores=16.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2540,6 +2610,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-0.json index 48abcf26805bb..7b9de95a200ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=25.0%,vcores=25.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "25.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "25.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-16.json index 10a7adc84d574..c6c53479511e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=25.0%,vcores=25.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "25.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "25.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-32.json index f086f0d2cfe1a..fef29b41ef0fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentage-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=25.0%,vcores=25.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "25.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "25.0%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-0.json index dd5c8fce57a23..b6efef59ed608 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-16.json index 72c665592742e..7de012a96d7f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-32.json index c49e974350474..aa9fb386ca603 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeight-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-0.json index bdc260235504d..b01426bbc2f85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=86.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "86.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-16.json index 8202d66ef0bbe..943b673e13587 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=86.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "86.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-32.json index 70123a8c3e11e..43d4986c2df8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndPercentageAndWeightMixed-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=12.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.0" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12288.0,vcores=86.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12288.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "86.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-0.json index 5808662f73235..df917c51891d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=3.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "3.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-16.json index 4831df276e675..29a29ec217041 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=3.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "3.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-32.json index 1ae2f0dfc1b7e..4b8e373bd4080 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerAbsoluteAndWeight-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=3.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "3.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=16384.0,vcores=16.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "16384.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "16.0" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-0.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-0.json index 8f2372e695357..735730c13a66d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-0.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-0.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=3.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "3.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 0, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-16.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-16.json index 18123bbd1834e..0a43df2c97393 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-16.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-16.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=3.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "3.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-32.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-32.json index 37e2bfffd22d4..2b08e31c16e90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-32.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/mixed-testSchedulerPercentageAndWeight-32.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=3.0w,vcores=3.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "3.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "3.0w" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1013,6 +1043,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1482,6 +1522,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1951,6 +2001,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=1.0w,vcores=1.0w]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "1.0w" + }, { + "resourceName" : "vcores", + "resourceValue" : "1.0w" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2406,6 +2466,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -2680,6 +2750,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json index ec79a62695128..bd2e9b468f310 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=28672.0,vcores=28.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "28672.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "28.0" + } ] + }, "capacity" : 87.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -544,6 +564,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=2048.0,vcores=2.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "2048.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "2.0" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -999,6 +1029,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=4096.0,vcores=4.0]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "4096.0" + }, { + "resourceName" : "vcores", + "resourceValue" : "4.0" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1284,6 +1324,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml index 1dac63b7b91fe..d91322721b9de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml @@ -5,6 +5,17 @@ 100.0 -1.0 0.0 + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + root root 2147483647 @@ -57,6 +68,17 @@ + + [memory-mb=12.5%,vcores=12.5%] + + memory-mb + 12.5% + + + vcores + 12.5% + + 12.5 0.0 50.0 @@ -546,6 +568,17 @@ + + [memory-mb=50.0%,vcores=50.0%] + + memory-mb + 50.0% + + + vcores + 50.0% + + 50.0 0.0 100.0 @@ -1035,6 +1068,17 @@ + + [memory-mb=37.5%,vcores=37.5%] + + memory-mb + 37.5% + + + vcores + 37.5% + + 37.5 0.0 100.0 @@ -1481,6 +1525,17 @@ + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + 100.0 0.0 100.0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json index 3aee400df7d12..7869b3868ddb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 50, @@ -748,6 +768,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1437,6 +1467,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1864,6 +1904,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml index 9ef4fda9b385f..d75879059df31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml @@ -5,6 +5,17 @@ 100.0 -1.0 0.0 + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + root root 2147483647 @@ -57,6 +68,17 @@ + + [memory-mb=12.5%,vcores=12.5%] + + memory-mb + 12.5% + + + vcores + 12.5% + + 12.5 0.0 50.0 @@ -774,6 +796,17 @@ + + [memory-mb=50.0%,vcores=50.0%] + + memory-mb + 50.0% + + + vcores + 50.0% + + 50.0 0.0 100.0 @@ -1491,6 +1524,17 @@ + + [memory-mb=37.5%,vcores=37.5%] + + memory-mb + 37.5% + + + vcores + 37.5% + + 37.5 0.0 100.0 @@ -1936,6 +1980,17 @@ + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + 100.0 0.0 100.0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json index 54be63a89f958..5a3b9a4eab93a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=25.0%,vcores=25.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "25.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "25.0%" + } ] + }, "capacity" : 25, "usedCapacity" : 0, "maxCapacity" : 100, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=75.0%,vcores=75.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "75.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "75.0%" + } ] + }, "capacity" : 75, "usedCapacity" : 0, "maxCapacity" : 100, @@ -802,6 +832,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json index 36857273c68e3..f44adf3af9b6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json @@ -7,6 +7,16 @@ "maxCapacity" : 100, "weight" : -1, "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "queueName" : "root", "queuePath" : "root", "maxParallelApps" : 2147483647, @@ -59,6 +69,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, "capacity" : 12.5, "usedCapacity" : 0, "maxCapacity" : 50, @@ -528,6 +548,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, "capacity" : 50, "usedCapacity" : 0, "maxCapacity" : 100, @@ -997,6 +1027,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, "capacity" : 37.5, "usedCapacity" : 0, "maxCapacity" : 100, @@ -1424,6 +1464,16 @@ "capacities" : { "queueCapacitiesByPartition" : [ { "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, "capacity" : 100, "usedCapacity" : 0, "maxCapacity" : 100, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml index a5335a6c3affd..91e06c9551c2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml @@ -5,6 +5,17 @@ 100.0 -1.0 0.0 + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + root root 2147483647 @@ -57,6 +68,17 @@ + + [memory-mb=12.5%,vcores=12.5%] + + memory-mb + 12.5% + + + vcores + 12.5% + + 12.5 0.0 50.0 @@ -545,6 +567,17 @@ + + [memory-mb=50.0%,vcores=50.0%] + + memory-mb + 50.0% + + + vcores + 50.0% + + 50.0 0.0 100.0 @@ -1033,6 +1066,17 @@ + + [memory-mb=37.5%,vcores=37.5%] + + memory-mb + 37.5% + + + vcores + 37.5% + + 37.5 0.0 100.0 @@ -1478,6 +1522,17 @@ + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + 100.0 0.0 100.0 From 3de66f5c4046cf9e44c1f6b14779bc4f6442e92f Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 26 Sep 2023 04:52:57 +0800 Subject: [PATCH 051/155] YARN-11547. [Federation] Router Supports Remove individual application records from FederationStateStore. (#6055) --- .../hadoop/yarn/server/router/Router.java | 96 ++++++++++++++++--- .../router/TestRouterStoreCommands.java | 77 +++++++++++++++ 2 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterStoreCommands.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 4761866253ff7..e4defc308dd7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -28,6 +28,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.MissingArgumentException; import org.apache.commons.lang.time.DurationFormatUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; @@ -37,14 +43,16 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.router.cleaner.SubClusterCleaner; import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; @@ -103,6 +111,9 @@ public class Router extends CompositeService { protected String webAppAddress; private static long clusterTimeStamp = System.currentTimeMillis(); private FedAppReportFetcher fetcher = null; + private static final String CMD_FORMAT_STATE_STORE = "-format-state-store"; + private static final String CMD_REMOVE_APPLICATION_FROM_STATE_STORE = + "-remove-application-from-state-store"; /** * Priority of the Router shutdown hook. @@ -191,7 +202,7 @@ protected void serviceStop() throws Exception { } protected void shutDown() { - new Thread(() -> Router.this.stop()).start(); + new Thread(Router.this::stop).start(); } protected RouterClientRMService createClientRMProxyService() { @@ -292,24 +303,14 @@ public static String getProxyHostAndPort(Configuration conf) { public static void main(String[] argv) { Configuration conf = new YarnConfiguration(); - Thread - .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); + Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(Router.class, argv, LOG); Router router = new Router(); try { GenericOptionsParser hParser = new GenericOptionsParser(conf, argv); argv = hParser.getRemainingArgs(); if (argv.length > 1) { - if (argv[0].equals("-format-state-store")) { - // TODO: YARN-11548. [Federation] Router Supports Format FederationStateStore. - System.err.println("format-state-store is not yet supported."); - } else if (argv[0].equals("-remove-application-from-state-store") && argv.length == 2) { - // TODO: YARN-11547. [Federation] - // Router Supports Remove individual application records from FederationStateStore. - System.err.println("remove-application-from-state-store is not yet supported."); - } else { - printUsage(System.err); - } + executeRouterCommand(conf, argv); } else { // Remove the old hook if we are rebooting. if (null != routerShutdownHook) { @@ -362,6 +363,73 @@ public FedAppReportFetcher getFetcher() { return fetcher; } + @VisibleForTesting + public static void removeApplication(Configuration conf, String applicationId) + throws Exception { + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(conf); + ApplicationId removeAppId = ApplicationId.fromString(applicationId); + LOG.info("Deleting application {} from state store.", removeAppId); + facade.deleteApplicationHomeSubCluster(removeAppId); + LOG.info("Application is deleted from state store"); + } + + private static void handFormatStateStore() { + // TODO: YARN-11548. [Federation] Router Supports Format FederationStateStore. + System.err.println("format-state-store is not yet supported."); + } + + private static void handRemoveApplicationFromStateStore(Configuration conf, + String applicationId) { + try { + removeApplication(conf, applicationId); + System.out.println("Application " + applicationId + " is deleted from state store"); + } catch (Exception e) { + System.err.println("Application " + applicationId + " error, exception = " + e); + } + } + + private static void executeRouterCommand(Configuration conf, String[] args) { + // Step1. Define Options. + Options opts = new Options(); + Option formatStateStoreOpt = new Option("format-state-store", false, + " Formats the FederationStateStore. " + + "This will clear the FederationStateStore and " + + "is useful if past applications are no longer needed. " + + "This should be run only when the Router is not running."); + Option removeApplicationFromStateStoreOpt = new Option("remove-application-from-state-store", + false, " Remove the application from FederationStateStore. " + + " This should be run only when the Router is not running. "); + opts.addOption(formatStateStoreOpt); + opts.addOption(removeApplicationFromStateStoreOpt); + + // Step2. Parse Options. + try { + String cmd = args[0]; + + CommandLine cliParser = new DefaultParser().parse(opts, args); + + if (CMD_FORMAT_STATE_STORE.equals(cmd)) { + handFormatStateStore(); + } else if (CMD_REMOVE_APPLICATION_FROM_STATE_STORE.equals(cmd)) { + if (cliParser.hasOption(removeApplicationFromStateStoreOpt)) { + String applicationId = cliParser.getOptionValue(removeApplicationFromStateStoreOpt); + handRemoveApplicationFromStateStore(conf, applicationId); + } else { + System.err.println("remove-application-from-state-store requires application arg."); + } + } else { + System.out.println("No related commands found."); + printUsage(System.err); + } + } catch (MissingArgumentException ex) { + System.out.println("Missing argument for options."); + printUsage(System.err); + } catch (ParseException e) { + System.out.println("Parsing of a command-line error."); + printUsage(System.err); + } + } + private static void printUsage(PrintStream out) { out.println("Usage: yarn router [-format-state-store] | " + "[-remove-application-from-state-store ]"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterStoreCommands.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterStoreCommands.java new file mode 100644 index 0000000000000..04007ca88dfbf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterStoreCommands.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.junit.Before; +import org.junit.Test; + +public class TestRouterStoreCommands { + + //////////////////////////////// + // Router Constants + //////////////////////////////// + private Configuration conf; + private MemoryFederationStateStore stateStore; + private FederationStateStoreFacade facade; + + @Before + public void setup() throws YarnException { + conf = new YarnConfiguration(); + stateStore = new MemoryFederationStateStore(); + stateStore.init(conf); + facade = FederationStateStoreFacade.getInstance(conf); + facade.reinitialize(stateStore, conf); + } + + @Test + public void testRemoveApplicationFromRouterStateStore() throws Exception { + + // We will design such a unit test. + // We will write the applicationId and subCluster into the stateStore, + // and then remove the application through Router.removeApplication. + // At this time, if we continue to query through the stateStore, + // We will get a prompt that application not exists. + + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + SubClusterId homeSubCluster = SubClusterId.newInstance("SC-1"); + ApplicationHomeSubCluster applicationHomeSubCluster = + ApplicationHomeSubCluster.newInstance(appId, homeSubCluster); + AddApplicationHomeSubClusterRequest request = + AddApplicationHomeSubClusterRequest.newInstance(applicationHomeSubCluster); + stateStore.addApplicationHomeSubCluster(request); + Router.removeApplication(conf, appId.toString()); + + GetApplicationHomeSubClusterRequest request1 = + GetApplicationHomeSubClusterRequest.newInstance(appId); + + LambdaTestUtils.intercept(YarnException.class, "Application " + appId + " does not exist.", + () -> stateStore.getApplicationHomeSubCluster(request1)); + } +} From 26a5f38250de1ac4978427d74c200a968f9a0b65 Mon Sep 17 00:00:00 2001 From: zhangshuyan <81411509+zhangshuyan0@users.noreply.github.com> Date: Tue, 26 Sep 2023 18:49:47 +0800 Subject: [PATCH 052/155] HDFS-17204. EC: Reduce unnecessary log when processing excess redundancy. (#6107). Contributed by Shuyan Zhang. Reviewed-by: Haiyang Hu Signed-off-by: He Xiaoqiao --- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 30c48c4878b14..54b89c813010d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -4201,6 +4201,12 @@ private void chooseExcessRedundancyStriped(BlockCollection bc, storage2index.put(storage, index); } + if (duplicated.isEmpty()) { + LOG.debug("Found no duplicated internal blocks for {}. Maybe it's " + + "because there are stale storages.", storedBlock); + return; + } + // use delHint only if delHint is duplicated final DatanodeStorageInfo delStorageHint = DatanodeStorageInfo.getDatanodeStorageInfo(nonExcess, delNodeHint); From f232eec49095a071ae6c668c18e257c304520091 Mon Sep 17 00:00:00 2001 From: Tamas Domok Date: Wed, 27 Sep 2023 16:43:28 +0200 Subject: [PATCH 053/155] YARN-11522. Update the documentation with the YARN-11000 changes. (#5870) --- .../src/site/markdown/CapacityScheduler.md | 63 +- .../src/site/markdown/ResourceManagerRest.md | 3968 ++++++++++++++--- 2 files changed, 3497 insertions(+), 534 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 3a694e3f2485e..1b94d6ff33e33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -64,8 +64,14 @@ The `CapacityScheduler` supports the following features: * **Priority Scheduling** - This feature allows applications to be submitted and scheduled with different priorities. Higher integer value indicates higher priority for an application. Currently Application priority is supported only for FIFO ordering policy. +* **Percentage Resource Configuration** - Administrators could specify percentages of resources to a queue. + * **Absolute Resource Configuration** - Administrators could specify absolute resources to a queue instead of providing percentage based values. This provides better control for admins to configure required amount of resources for a given queue. +* **Weight Resource Configuration** - Administrators could specify weights to a queue instead of providing percentage based values. This provides better control for admins to configure resources for the queue in a dynamically changing queue hierarchy. + +* **Universal Capacity Vector Resource Configuration** - Administrators could specify resources in a mixed manner to a queue using absolute, weight or percentage modes for each defined resource types. This provides the most flexible way to configure the required amount of resources for a given queue. + * **Dynamic Auto-Creation and Management of Leaf Queues** - This feature supports auto-creation of **leaf queues** in conjunction with **queue-mapping** which currently supports **user-group** based queue mappings for application placement to a queue. The scheduler also supports capacity management for these queues based on a policy configured on the parent queue. Configuration @@ -123,33 +129,75 @@ Configuration | Property | Description | |:---- |:---- | -| `yarn.scheduler.capacity..capacity` | Queue *capacity* in percentage (%) as a float (e.g. 12.5), weight as a float with the postfix *w* (e.g. 2.0w) or as absolute resource queue minimum capacity. When using percentage values the sum of capacities for all queues, at each level, must be equal to 100. If absolute resource is configured, sum of absolute resources of child queues could be less than its parent absolute resource capacity. Applications in the queue may consume more resources than the queue's capacity if there are free resources, providing elasticity. | -| `yarn.scheduler.capacity..maximum-capacity` | Maximum queue capacity in percentage (%) as a float (when the *capacity* property is defined with either percentages or weights) or as absolute resource queue maximum capacity. This limits the *elasticity* for applications in the queue. 1) Value is between 0 and 100. 2) Admin needs to make sure absolute maximum capacity >= absolute capacity for each queue. Also, setting this value to -1 sets maximum capacity to 100%. | +| `yarn.scheduler.capacity.legacy-queue-mode.enabled` | Disabling the legacy-queue mode opens up the possibility to mix different capacity modes and the usage of the Universal Capacity Vector format to allocate resources flexibly for the queues. Default is *true*. | +| `yarn.scheduler.capacity..capacity` | Queue *capacity* in percentage (%) as a float (e.g. 12.5), weight as a float with the postfix *w* (e.g. 2.0w) or as absolute resource queue minimum capacity. When using percentage values the sum of capacities for all queues, at each level, must be equal to 100. If absolute resource is configured, sum of absolute resources of child queues could be less than its parent absolute resource capacity. Applications in the queue may consume more resources than the queue's capacity if there are free resources, providing elasticity. When the legacy-queue-mode is disabled the Universal Capacity Vector format can be used to configure the queue capacities, e.g. `[memory=50%,vcores=2w,gpu=1]`. | +| `yarn.scheduler.capacity..maximum-capacity` | Maximum queue capacity in percentage (%) as a float (when the *capacity* property is defined with either percentages or weights) or as absolute resource queue maximum capacity. This limits the *elasticity* for applications in the queue. 1) Value is between 0 and 100. 2) Admin needs to make sure absolute maximum capacity >= absolute capacity for each queue. Also, setting this value to -1 sets maximum capacity to 100%. When the legacy-queue-mode is disabled the Universal Capacity Vector format can be used to configure the queue capacities, e.g. `[memory=50%,vcores=2w,gpu=1]`. | | `yarn.scheduler.capacity.minimum-user-limit-percent` / `yarn.scheduler.capacity..minimum-user-limit-percent` | Each queue enforces a limit on the percentage of resources allocated to a user at any given time, if there is demand for resources. The user limit can vary between a minimum and maximum value. The former (the minimum value) is set to this property value and the latter (the maximum value) depends on the number of users who have submitted applications. For e.g., suppose the value of this property is 25. If two users have submitted applications to a queue, no single user can use more than 50% of the queue resources. If a third user submits an application, no single user can use more than 33% of the queue resources. With 4 or more users, no user can use more than 25% of the queues resources. A value of 100 implies no user limits are imposed. The default is 100. Value is specified as an integer. This can be set for all queues with `yarn.scheduler.capacity.minimum-user-limit-percent` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..minimum-user-limit-percent`. | | `yarn.scheduler.capacity.user-limit-factor` / `yarn.scheduler.capacity..user-limit-factor` | User limit factor provides a way to control the max amount of resources that a single user can consume. It is the multiple of the queue's capacity. By default this is set to 1 which ensures that a single user can never take more than the queue's configured capacity irrespective of how idle the cluster is. Increasing it means a single user can use more than the minimum capacity of the cluster, while decreasing it results in lower maximum resources. Setting this to -1 will disable the feature. Value is specified as a float. Note: using the flexible auto queue creation (yarn.scheduler.capacity.\.auto-queue-creation-v2) with weights will automatically set this property to -1, as the dynamic queues will be created with the hardcoded weight of 1 and in idle cluster scenarios they should be able to use more resources than calculated. This can be set for all queues with `yarn.scheduler.capacity.user-limit-factor` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..user-limit-factor`. | | `yarn.scheduler.capacity..maximum-allocation-mb` | The per queue maximum limit of memory to allocate to each container request at the Resource Manager. This setting overrides the cluster configuration `yarn.scheduler.maximum-allocation-mb`. This value must be smaller than or equal to the cluster maximum. | | `yarn.scheduler.capacity..maximum-allocation-vcores` | The per queue maximum limit of virtual cores to allocate to each container request at the Resource Manager. This setting overrides the cluster configuration `yarn.scheduler.maximum-allocation-vcores`. This value must be smaller than or equal to the cluster maximum. | | `yarn.scheduler.capacity..user-settings..weight` | This floating point value is used when calculating the user limit resource values for users in a queue. This value will weight each user more or less than the other users in the queue. For example, if user A should receive 50% more resources in a queue than users B and C, this property will be set to 1.5 for user A. Users B and C will default to 1.0. | - * Configuring Resource Allocation + * Configuring Resource Allocation (legacy-queue-mode) `CapacityScheduler` supports three different resource allocation configuration modes: percentage values (*relative mode*), weights and absolute resources. Relative mode provides a way to describe queue's resources as a fraction of its parent's resources. For example if *capacity* is set as 50.0 for the queue `root.users`, users queue has 50% of root's resources set as minimum capacity. - In weight mode the resources are divided based on how the queue's weight relates to the sum of configured weights under the same parent. For example if there are three queues under a parent with weights *3w*, *2w*, *5w*, the sum is 10, so the calculated minimum *capacity* will be 30%, 20% and 50% respectively. The benefit of using this mode is flexibility. When using percentages every time a new queue gets added the percentage values need to be manually recalculated, as the sum under a parent must to be 100%, but with weights this is performed automatically. Using the previous example when a new queue gets added under the same parent as the previous three with weight *10w* the new sum will be 20, so the new calculated *capacities* will be: 15%, 10%, 25%, 50%. Note: `yarn.scheduler.capacity..max-capacity` must be configured with percentages, as there is no weight mode for *maximum-capacity*. + In weight mode the resources are divided based on how the queue's weight relates to the sum of configured weights under the same parent. For example if there are three queues under a parent with weights *3w*, *2w*, *5w*, the sum is 10, so the calculated minimum *capacity* will be 30%, 20% and 50% respectively. The benefit of using this mode is flexibility. When using percentages every time a new queue gets added the percentage values need to be manually recalculated, as the sum under a parent must be 100%, but with weights this is performed automatically. Using the previous example when a new queue gets added under the same parent as the previous three with weight *10w* the new sum will be 20, so the new calculated *capacities* will be: 15%, 10%, 25%, 50%. Note: `yarn.scheduler.capacity..max-capacity` must be configured with percentages, as there is no weight mode for *maximum-capacity*. To use absolute resources mode both `yarn.scheduler.capacity..capacity` and `yarn.scheduler.capacity..max-capacity` should have absolute resource values like `[memory=10240,vcores=12]`. This configuration indicates 10GB Memory and 12 VCores. It is possible to mix weights and percentages in a queue structure, but child queues under one parent must use the same *capacity* mode. + * Configuring Resource Allocation (non-legacy-queue-mode) + + The capacity and maximum-capacity can be set for the queues using the Universal Capacity Vector format, e.g. `[memory=50%,vcores=2w,gpu=1]`. + In this example the Memory is set in percentage mode, the Vcores is set in weight mode and the GPU is set in absolute units. + It is also possible to use to old capacity format, e.g.: `50.0` for percentage, `[memory=1024,vcores=1]` for absolute and `1w` for weight mode. + Different modes can be mixed freely in the queue hierarchy. + + The hierarchy between the resources is calculated based on the queues' capacity configuration and the available cluster resources. + Calculating the hierarchy of resources is done in the following way, for each defined resource type: + 1. the configured absolute capacities are allocated to the queues from the available cluster resources + 2. the remaining cluster resources are allocated to the queues with percentage resource configurations + 3. the rest of the resources are allocated to the queues with weighted resource configurations + + Example for mixed queue resource allocation using the Universal Capacity Vector format: +``` + # Configuration + yarn.scheduler.capacity.legacy-queue-mode.enabled = false + yarn.scheduler.capacity.root.queues = default, test_1, test_2 + yarn.scheduler.capacity.root.test_1.queues = test_1_1, test_1_2, test_1_3 + yarn.scheduler.capacity.root.default.capacity = [memory=1w, vcores=1w] + yarn.scheduler.capacity.root.test_1.capacity = [memory=16384, vcores=16] + yarn.scheduler.capacity.root.test_1.test_1_1.capacity = [memory=50%, vcores=50%] + yarn.scheduler.capacity.root.test_1.test_1_2.capacity = [memory=1w, vcores=1w] + yarn.scheduler.capacity.root.test_1.test_1_3.capacity = [memory=12288, vcores=12] + yarn.scheduler.capacity.root.test_2.capacity = [memory=75%, vcores=75%] + + # ClusterResources=[32GB 32VCores] EffectiveMin AbsoluteCapacity + root.default 4/32 [memory=4096, vcores=4] 12.5% + root.test_1 16/32 [memory=16384, vcores=16] + root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25% + root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25% + root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5% + root.test_2 12/32 [memory=12288, vcores=12] 37.5% +``` + Further examples can be found at `TestRMWebServicesCapacitySchedulerMixedMode.java`. + + The capacity for the root queue cannot be configured, it is fixed to 100% (percentage mode). + + The actual capacity, absoluteCapacity and derived properties like maximumApplications are calculated from the hierarchy between the resources. + If there is no cluster resource, the maximumApplications defaults to the configured values. + * Running and Pending Application Limits - + The `CapacityScheduler` supports the following parameters to control the running and pending applications: | Property | Description | |:---- |:---- | -| `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 10000. This can be set for all queues with `yarn.scheduler.capacity.maximum-applications` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..maximum-applications`. When this property is not set for a specific queue path, the maximum application number is calculated by taking all configured node labels into consideration, and choosing the highest possible value. Integer value expected. | +| `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue absolute capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 10000. This can be set for all queues with `yarn.scheduler.capacity.maximum-applications` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..maximum-applications`. When this property is not set for a specific queue path, the maximum application number is calculated by taking all configured node labels into consideration, and choosing the highest possible value. When the legacy-queue-mode is disabled and no cluster resource is available, it defaults to the configured values. Integer value expected. | | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource-percent` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..maximum-am-resource-percent` | | `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..max-parallel-apps`. Integer value is expected. By default, there is no limit. The maximum parallel application limit is an inherited property in the queue hierarchy, meaning that the lowest value will be selected as the enforced limit in every branch of the hierarchy. | @@ -310,7 +358,8 @@ The property `yarn.scheduler.capacity.mapping-rule-json` takes precedence over ` ####Differences between legacy and flexible queue auto-creation modes -To use the flexible Queue Auto-Creation under a parent the queue capacities must be configured with weights. The flexible mode gives the user much more freedom to automatically create new leaf queues or entire queue hierarchies based on mapping rules. "Legacy" mode refers to either percentage-based configuration or where capacities are defined with absolute resources. +To use the flexible Queue Auto-Creation under a parent the queue capacities must be configured with weights in legacy-queue-mode. The flexible mode gives the user much more freedom to automatically create new leaf queues or entire queue hierarchies based on mapping rules. "Legacy" mode refers to either percentage-based configuration or where capacities are defined with absolute resources. +If legacy-queue-mode is disabled the capacities can be configured using the Universal Capacity Vector format or using weights `1w`, percentages `75` or absolute units `[memory=1024,vcores=1]`. In flexible Queue Auto-Creation mode, every parent queue can have dynamically created parent or leaf queues (if the `yarn.scheduler.capacity..auto-queue-creation-v2.enabled` property is set to true), even if it already has static child queues. This also means that certain settings influence the outcome of the queue placement depending on how the scheduler is configured. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 001fb6b5f24a0..c7e68c2b0666c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -306,20 +306,22 @@ The capacity scheduler supports hierarchical queues. This one request will print | Item | Data Type | Description | |:---- |:---- |:---- | | type | string | Scheduler type - capacityScheduler | -| capacity | float | Configured queue capacity in percentage relative to its parent queue | +| capacity | float | **Configured** queue capacity in percentage relative to its parent queue in legacy-queue-mode, otherwise the **effective** capacity | | usedCapacity | float | Used queue capacity in percentage | -| maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue | +| maxCapacity | float | **Configured** maximum queue capacity in percentage relative to its parent queue in legacy-queue-mode, otherwise the **effective** max capacity | | queueName | string | Name of the queue | | queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of queue resources | | health | A single health object | The health metrics of capacity scheduler. This metrics existed since 2.8.0, but the output was not well formatted. Hence users can not make use of this field cleanly, this is optimized from 3.2.0 onwards. | +| weight | float | The configured weight of the queue | +| normalizedWeight | float | The normalized weight by siblings: `[0.0, 1.0]` in legacy-queue-mode, otherwise 0 | ### Elements of the queues object for a Parent queue | Item | Data Type | Description | |:---- |:---- |:---- | -| capacity | float | Configured queue capacity in percentage relative to its parent queue | +| capacity | float | **Configured** queue capacity in percentage relative to its parent queue in legacy-queue-mode, otherwise the **effective** capacity | | usedCapacity | float | Used queue capacity in percentage | -| maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue | +| maxCapacity | float | **Configured** maximum queue capacity in percentage relative to its parent queue in legacy-queue-mode, otherwise the **effective** max capacity | | absoluteCapacity | float | Absolute capacity percentage this queue can use of entire cluster | | absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster | | absoluteUsedCapacity | float | Absolute used capacity percentage this queue is using of the entire cluster | @@ -330,6 +332,9 @@ The capacity scheduler supports hierarchical queues. This one request will print | state | string of QueueState | The state of the queue | | queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information. Omitted if the queue has no sub-queues. | | resourcesUsed | A single resource object | The total amount of resources used by this queue | +| mode | string | The configured capacity mode: percentage, absolute, weight, mixed | +| weight | float | The configured weight of the queue | +| normalizedWeight | float | The normalized weight by the queue siblings: `[0.0, 1.0]` in legacy-queue-mode, otherwise 0 | ### Elements of the queues object for a Leaf queue - contains all the elements in parent except 'queues' plus the following: @@ -410,328 +415,1735 @@ Response Body: ```json { - "scheduler": { - "schedulerInfo": { - "capacity": 100.0, - "maxCapacity": 100.0, - "queueName": "root", - "queues": { - "queue": [ - { - "absoluteCapacity": 10.5, - "absoluteMaxCapacity": 50.0, - "absoluteUsedCapacity": 0.0, - "capacity": 10.5, - "maxCapacity": 50.0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "queueName": "a", - "queues": { - "queue": [ - { - "absoluteCapacity": 3.15, - "absoluteMaxCapacity": 25.0, - "absoluteUsedCapacity": 0.0, - "capacity": 30.000002, - "maxCapacity": 50.0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "queueName": "a1", - "queues": { - "queue": [ - { - "absoluteCapacity": 2.6775, - "absoluteMaxCapacity": 25.0, - "absoluteUsedCapacity": 0.0, - "capacity": 85.0, - "maxActiveApplications": 1, - "maxActiveApplicationsPerUser": 1, - "maxApplications": 267, - "maxApplicationsPerUser": 267, - "maxCapacity": 100.0, - "numActiveApplications": 0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "numContainers": 0, - "numPendingApplications": 0, - "queueName": "a1a", - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "type": "capacitySchedulerLeafQueueInfo", - "usedCapacity": 0.0, - "usedResources": "", - "userLimit": 100, - "userLimitFactor": 1.0, - "users": null - }, - { - "absoluteCapacity": 0.47250003, - "absoluteMaxCapacity": 25.0, - "absoluteUsedCapacity": 0.0, - "capacity": 15.000001, - "maxActiveApplications": 1, - "maxActiveApplicationsPerUser": 1, - "maxApplications": 47, - "maxApplicationsPerUser": 47, - "maxCapacity": 100.0, - "numActiveApplications": 0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "numContainers": 0, - "numPendingApplications": 0, - "queueName": "a1b", - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "type": "capacitySchedulerLeafQueueInfo", - "usedCapacity": 0.0, - "usedResources": "", - "userLimit": 100, - "userLimitFactor": 1.0, - "users": null - } - ] - }, - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "usedCapacity": 0.0, - "usedResources": "" - }, - { - "absoluteCapacity": 7.35, - "absoluteMaxCapacity": 50.0, - "absoluteUsedCapacity": 0.0, - "capacity": 70.0, - "maxActiveApplications": 1, - "maxActiveApplicationsPerUser": 100, - "maxApplications": 735, - "maxApplicationsPerUser": 73500, - "maxCapacity": 100.0, - "numActiveApplications": 0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "numContainers": 0, - "numPendingApplications": 0, - "queueName": "a2", - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "type": "capacitySchedulerLeafQueueInfo", - "usedCapacity": 0.0, - "usedResources": "", - "userLimit": 100, - "userLimitFactor": 100.0, - "users": null - } - ] - }, - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "usedCapacity": 0.0, - "usedResources": "" - }, - { - "absoluteCapacity": 89.5, - "absoluteMaxCapacity": 100.0, - "absoluteUsedCapacity": 0.0, - "capacity": 89.5, - "maxCapacity": 100.0, - "numApplications": 2, - "maxParallelApps": 2147483647, - "queueName": "b", - "queues": { - "queue": [ - { - "absoluteCapacity": 53.7, - "absoluteMaxCapacity": 100.0, - "absoluteUsedCapacity": 0.0, - "capacity": 60.000004, - "maxActiveApplications": 1, - "maxActiveApplicationsPerUser": 100, - "maxApplications": 5370, - "maxApplicationsPerUser": 537000, - "maxCapacity": 100.0, - "numActiveApplications": 1, - "numApplications": 2, - "maxParallelApps": 2147483647, - "numContainers": 0, - "numPendingApplications": 1, - "queueName": "b1", - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "type": "capacitySchedulerLeafQueueInfo", - "usedCapacity": 0.0, - "usedResources": "", - "userLimit": 100, - "userLimitFactor": 100.0, - "users": { - "user": [ - { - "numActiveApplications": 0, - "numPendingApplications": 1, - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "username": "user2" - }, - { - "numActiveApplications": 1, - "numPendingApplications": 0, - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "username": "user1" - } - ] - } - }, - { - "absoluteCapacity": 35.3525, - "absoluteMaxCapacity": 100.0, - "absoluteUsedCapacity": 0.0, - "capacity": 39.5, - "maxActiveApplications": 1, - "maxActiveApplicationsPerUser": 100, - "maxApplications": 3535, - "maxApplicationsPerUser": 353500, - "maxCapacity": 100.0, - "numActiveApplications": 0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "numContainers": 0, - "numPendingApplications": 0, - "queueName": "b2", - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "type": "capacitySchedulerLeafQueueInfo", - "usedCapacity": 0.0, - "usedResources": "", - "userLimit": 100, - "userLimitFactor": 100.0, - "users": null - }, - { - "absoluteCapacity": 0.4475, - "absoluteMaxCapacity": 100.0, - "absoluteUsedCapacity": 0.0, - "capacity": 0.5, - "maxActiveApplications": 1, - "maxActiveApplicationsPerUser": 100, - "maxApplications": 44, - "maxApplicationsPerUser": 4400, - "maxCapacity": 100.0, - "numActiveApplications": 0, - "numApplications": 0, - "maxParallelApps": 2147483647, - "numContainers": 0, - "numPendingApplications": 0, - "queueName": "b3", - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "type": "capacitySchedulerLeafQueueInfo", - "usedCapacity": 0.0, - "usedResources": "", - "userLimit": 100, - "userLimitFactor": 100.0, - "users": null - } - ] - }, - "resourcesUsed": { - "memory": 0, - "vCores": 0 - }, - "state": "RUNNING", - "usedCapacity": 0.0, - "usedResources": "" - } - ] - }, - "health": { - "lastrun": 1326381444693, - "operationsInfo": [ - { - "operation": "last-allocation", - "nodeId": "N/A", - "containerId": "N/A", - "queue": "N/A" - }, - { - "operation": "last-release", - "nodeId": "host.domain.com:8041", - "containerId": "container_1326821518301_0005_01_000001", - "queue": "root.default" - }, - { - "operation": "last-preemption", - "nodeId": "N/A", - "containerId": "N/A", - "queue": "N/A" - }, - { - "operation": "last-reservation", - "nodeId": "N/A", - "containerId": "N/A", - "queue": "N/A" - } - ], - "lastRunDetails": [ - { - "operation": "releases", - "count": 0, - "resources": { - "memory": 0, - "vCores": 0 - } - }, - { - "operation": "allocations", - "count": 0, - "resources": { - "memory": 0, - "vCores": 0 - } - }, - { - "operation": "reservations", - "count": 0, - "resources": { - "memory": 0, - "vCores": 0 - } - } - ] - }, - "type": "capacityScheduler", - "usedCapacity": 0.0 + "scheduler" : { + "schedulerInfo" : { + "type" : "capacityScheduler", + "capacity" : 100, + "usedCapacity" : 0, + "maxCapacity" : 100, + "weight" : -1, + "normalizedWeight" : 0, + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, + "queueName" : "root", + "queuePath" : "root", + "maxParallelApps" : 2147483647, + "isAbsoluteResource" : false, + "queues" : { + "queue" : [ { + "type" : "capacitySchedulerLeafQueueInfo", + "queuePath" : "root.a", + "capacity" : 12.5, + "usedCapacity" : 0, + "maxCapacity" : 50, + "absoluteCapacity" : 12.5, + "absoluteMaxCapacity" : 50, + "absoluteUsedCapacity" : 0, + "weight" : -1, + "normalizedWeight" : 0, + "numApplications" : 0, + "maxParallelApps" : 2147483647, + "queueName" : "a", + "isAbsoluteResource" : false, + "state" : "RUNNING", + "resourcesUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "hideReservationQueues" : false, + "nodeLabels" : [ "*" ], + "allocatedContainers" : 0, + "reservedContainers" : 0, + "pendingContainers" : 0, + "capacities" : { + "queueCapacitiesByPartition" : [ { + "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=12.5%,vcores=12.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "12.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "12.5%" + } ] + }, + "capacity" : 12.5, + "usedCapacity" : 0, + "maxCapacity" : 50, + "absoluteCapacity" : 12.5, + "absoluteUsedCapacity" : 0, + "absoluteMaxCapacity" : 50, + "maxAMLimitPercentage" : 10, + "weight" : -1, + "normalizedWeight" : 0, + "configuredMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "configuredMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMinResource" : { + "memory" : 4096, + "vCores" : 4, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 4 + } ] + } + }, + "effectiveMaxResource" : { + "memory" : 16384, + "vCores" : 16, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 16384 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 16 + } ] + } + } + } ] + }, + "resources" : { + "resourceUsagesByPartition" : [ { + "partitionName" : "", + "used" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "reserved" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "pending" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amLimit" : { + "memory" : 2048, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 2048 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "userAmLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } + } ] + }, + "minEffectiveCapacity" : { + "memory" : 4096, + "vCores" : 4, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 4 + } ] + } + }, + "maxEffectiveCapacity" : { + "memory" : 16384, + "vCores" : 16, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 16384 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 16 + } ] + } + }, + "maximumAllocation" : { + "memory" : 8192, + "vCores" : 4, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 8192 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 4 + } ] + } + }, + "queueAcls" : { + "queueAcl" : [ { + "accessType" : "ADMINISTER_QUEUE", + "accessControlList" : " " + }, { + "accessType" : "APPLICATION_MAX_PRIORITY", + "accessControlList" : "*" + }, { + "accessType" : "SUBMIT_APP", + "accessControlList" : " " + } ] + }, + "queuePriority" : 0, + "orderingPolicyInfo" : "fifo", + "autoCreateChildQueueEnabled" : false, + "leafQueueTemplate" : { }, + "mode" : "percentage", + "queueType" : "leaf", + "creationMethod" : "static", + "autoCreationEligibility" : "off", + "autoQueueTemplateProperties" : { }, + "autoQueueParentTemplateProperties" : { }, + "autoQueueLeafTemplateProperties" : { }, + "numActiveApplications" : 0, + "numPendingApplications" : 0, + "numContainers" : 0, + "maxApplications" : 1250, + "maxApplicationsPerUser" : 1250, + "userLimit" : 100, + "users" : { }, + "userLimitFactor" : 1, + "configuredMaxAMResourceLimit" : 0.1, + "AMResourceLimit" : { + "memory" : 2048, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 2048 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "usedAMResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "userAMResourceLimit" : { + "memory" : 2048, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 2048 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "preemptionDisabled" : true, + "intraQueuePreemptionDisabled" : true, + "defaultPriority" : 0, + "isAutoCreatedLeafQueue" : false, + "maxApplicationLifetime" : -1, + "defaultApplicationLifetime" : -1 + }, { + "type" : "capacitySchedulerLeafQueueInfo", + "queuePath" : "root.b", + "capacity" : 50, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 50, + "absoluteMaxCapacity" : 100, + "absoluteUsedCapacity" : 0, + "weight" : -1, + "normalizedWeight" : 0, + "numApplications" : 0, + "maxParallelApps" : 2147483647, + "queueName" : "b", + "isAbsoluteResource" : false, + "state" : "RUNNING", + "resourcesUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "hideReservationQueues" : false, + "nodeLabels" : [ "*" ], + "allocatedContainers" : 0, + "reservedContainers" : 0, + "pendingContainers" : 0, + "capacities" : { + "queueCapacitiesByPartition" : [ { + "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=50.0%,vcores=50.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "50.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "50.0%" + } ] + }, + "capacity" : 50, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 50, + "absoluteUsedCapacity" : 0, + "absoluteMaxCapacity" : 100, + "maxAMLimitPercentage" : 10, + "weight" : -1, + "normalizedWeight" : 0, + "configuredMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "configuredMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMinResource" : { + "memory" : 16384, + "vCores" : 16, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 16384 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 16 + } ] + } + }, + "effectiveMaxResource" : { + "memory" : 32768, + "vCores" : 32, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 32768 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 32 + } ] + } + } + } ] + }, + "resources" : { + "resourceUsagesByPartition" : [ { + "partitionName" : "", + "used" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "reserved" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "pending" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amLimit" : { + "memory" : 4096, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "userAmLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } + } ] + }, + "minEffectiveCapacity" : { + "memory" : 16384, + "vCores" : 16, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 16384 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 16 + } ] + } + }, + "maxEffectiveCapacity" : { + "memory" : 32768, + "vCores" : 32, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 32768 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 32 + } ] + } + }, + "maximumAllocation" : { + "memory" : 8192, + "vCores" : 4, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 8192 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 4 + } ] + } + }, + "queueAcls" : { + "queueAcl" : [ { + "accessType" : "ADMINISTER_QUEUE", + "accessControlList" : " " + }, { + "accessType" : "APPLICATION_MAX_PRIORITY", + "accessControlList" : "*" + }, { + "accessType" : "SUBMIT_APP", + "accessControlList" : " " + } ] + }, + "queuePriority" : 0, + "orderingPolicyInfo" : "fifo", + "autoCreateChildQueueEnabled" : false, + "leafQueueTemplate" : { }, + "mode" : "percentage", + "queueType" : "leaf", + "creationMethod" : "static", + "autoCreationEligibility" : "off", + "autoQueueTemplateProperties" : { }, + "autoQueueParentTemplateProperties" : { }, + "autoQueueLeafTemplateProperties" : { }, + "numActiveApplications" : 0, + "numPendingApplications" : 0, + "numContainers" : 0, + "maxApplications" : 5000, + "maxApplicationsPerUser" : 5000, + "userLimit" : 100, + "users" : { }, + "userLimitFactor" : 1, + "configuredMaxAMResourceLimit" : 0.1, + "AMResourceLimit" : { + "memory" : 4096, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "usedAMResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "userAMResourceLimit" : { + "memory" : 4096, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "preemptionDisabled" : true, + "intraQueuePreemptionDisabled" : true, + "defaultPriority" : 0, + "isAutoCreatedLeafQueue" : false, + "maxApplicationLifetime" : -1, + "defaultApplicationLifetime" : -1 + }, { + "type" : "capacitySchedulerLeafQueueInfo", + "queuePath" : "root.c", + "capacity" : 37.5, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 37.5, + "absoluteMaxCapacity" : 100, + "absoluteUsedCapacity" : 0, + "weight" : -1, + "normalizedWeight" : 0, + "numApplications" : 0, + "maxParallelApps" : 2147483647, + "queueName" : "c", + "isAbsoluteResource" : false, + "state" : "RUNNING", + "resourcesUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "hideReservationQueues" : false, + "nodeLabels" : [ "*" ], + "allocatedContainers" : 0, + "reservedContainers" : 0, + "pendingContainers" : 0, + "capacities" : { + "queueCapacitiesByPartition" : [ { + "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=37.5%,vcores=37.5%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "37.5%" + }, { + "resourceName" : "vcores", + "resourceValue" : "37.5%" + } ] + }, + "capacity" : 37.5, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 37.5, + "absoluteUsedCapacity" : 0, + "absoluteMaxCapacity" : 100, + "maxAMLimitPercentage" : 10, + "weight" : -1, + "normalizedWeight" : 0, + "configuredMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "configuredMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMinResource" : { + "memory" : 12288, + "vCores" : 12, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 12288 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 12 + } ] + } + }, + "effectiveMaxResource" : { + "memory" : 32768, + "vCores" : 32, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 32768 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 32 + } ] + } + } + } ] + }, + "resources" : { + "resourceUsagesByPartition" : [ { + "partitionName" : "", + "used" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "reserved" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "pending" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amLimit" : { + "memory" : 4096, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "userAmLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } + } ] + }, + "minEffectiveCapacity" : { + "memory" : 12288, + "vCores" : 12, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 12288 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 12 + } ] + } + }, + "maxEffectiveCapacity" : { + "memory" : 32768, + "vCores" : 32, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 32768 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 32 + } ] + } + }, + "maximumAllocation" : { + "memory" : 8192, + "vCores" : 4, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 8192 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 4 + } ] + } + }, + "queueAcls" : { + "queueAcl" : [ { + "accessType" : "ADMINISTER_QUEUE", + "accessControlList" : " " + }, { + "accessType" : "APPLICATION_MAX_PRIORITY", + "accessControlList" : "*" + }, { + "accessType" : "SUBMIT_APP", + "accessControlList" : " " + } ] + }, + "queuePriority" : 0, + "orderingPolicyInfo" : "fifo", + "autoCreateChildQueueEnabled" : false, + "leafQueueTemplate" : { }, + "mode" : "percentage", + "queueType" : "leaf", + "creationMethod" : "static", + "autoCreationEligibility" : "off", + "autoQueueTemplateProperties" : { }, + "autoQueueParentTemplateProperties" : { }, + "autoQueueLeafTemplateProperties" : { }, + "numActiveApplications" : 0, + "numPendingApplications" : 0, + "numContainers" : 0, + "maxApplications" : 3750, + "maxApplicationsPerUser" : 3750, + "userLimit" : 100, + "users" : { }, + "userLimitFactor" : 1, + "configuredMaxAMResourceLimit" : 0.1, + "AMResourceLimit" : { + "memory" : 4096, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "usedAMResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "userAMResourceLimit" : { + "memory" : 4096, + "vCores" : 1, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 4096 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 1 + } ] + } + }, + "preemptionDisabled" : true, + "intraQueuePreemptionDisabled" : true, + "defaultPriority" : 0, + "isAutoCreatedLeafQueue" : false, + "maxApplicationLifetime" : -1, + "defaultApplicationLifetime" : -1 + } ] + }, + "capacities" : { + "queueCapacitiesByPartition" : [ { + "partitionName" : "", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[memory-mb=100.0%,vcores=100.0%]", + "capacityVectorEntries" : [ { + "resourceName" : "memory-mb", + "resourceValue" : "100.0%" + }, { + "resourceName" : "vcores", + "resourceValue" : "100.0%" + } ] + }, + "capacity" : 100, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 100, + "absoluteUsedCapacity" : 0, + "absoluteMaxCapacity" : 100, + "maxAMLimitPercentage" : 0, + "weight" : -1, + "normalizedWeight" : 0, + "configuredMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "configuredMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMinResource" : { + "memory" : 32768, + "vCores" : 32, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 32768 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 32 + } ] + } + }, + "effectiveMaxResource" : { + "memory" : 32768, + "vCores" : 32, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 32768 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 32 + } ] + } + } + } ] + }, + "health" : { + "lastrun" : 0, + "operationsInfo" : [ { + "operation" : "last-allocation", + "nodeId" : "N/A", + "containerId" : "N/A", + "queue" : "N/A" + }, { + "operation" : "last-release", + "nodeId" : "N/A", + "containerId" : "N/A", + "queue" : "N/A" + }, { + "operation" : "last-preemption", + "nodeId" : "N/A", + "containerId" : "N/A", + "queue" : "N/A" + }, { + "operation" : "last-reservation", + "nodeId" : "N/A", + "containerId" : "N/A", + "queue" : "N/A" + } ], + "lastRunDetails" : [ { + "operation" : "releases", + "count" : 0, + "resources" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } + }, { + "operation" : "allocations", + "count" : 0, + "resources" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } + }, { + "operation" : "reservations", + "count" : 0, + "resources" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } + } ] + }, + "maximumAllocation" : { + "memory" : 8192, + "vCores" : 4, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 8192 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 4 + } ] } + }, + "queueAcls" : { + "queueAcl" : [ { + "accessType" : "ADMINISTER_QUEUE", + "accessControlList" : "*" + }, { + "accessType" : "APPLICATION_MAX_PRIORITY", + "accessControlList" : "*" + }, { + "accessType" : "SUBMIT_APP", + "accessControlList" : "*" + } ] + }, + "queuePriority" : 0, + "orderingPolicyInfo" : "utilization", + "mode" : "percentage", + "queueType" : "parent", + "creationMethod" : "static", + "autoCreationEligibility" : "off", + "autoQueueTemplateProperties" : { }, + "autoQueueParentTemplateProperties" : { }, + "autoQueueLeafTemplateProperties" : { } } + } } -```json +``` **XML response** @@ -750,255 +2162,1650 @@ Response Header: Response Body: ```xml - - + 100.0 0.0 100.0 + -1.0 + 0.0 + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + root + root + 2147483647 + false - - 10.5 + + root.a + 12.5 0.0 50.0 - 10.5 + 12.5 50.0 0.0 + -1.0 + 0.0 0 2147483647 - <memory:0, vCores:0> a + false RUNNING - - - 30.000002 + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + false + * + 0 + 0 + 0 + + + + + [memory-mb=12.5%,vcores=12.5%] + + memory-mb + 12.5% + + + vcores + 12.5% + + + 12.5 0.0 50.0 - 3.15 - 25.0 + 12.5 0.0 - 0 - 2147483647 - <memory:0, vCores:0> - a1 - RUNNING - - - 85.0 - 0.0 - 100.0 - 2.6775 - 25.0 - 0.0 - 0 - 2147483647 - <memory:0, vCores:0> - a1a - RUNNING - - 0 - 0 - - 0 - 0 - 0 - 267 - 267 - 1 - 1 - 100 - - 1.0 - - - 15.000001 - 0.0 - 100.0 - 0.47250003 - 25.0 - 0.0 - 0 - 2147483647 - <memory:0, vCores:0> - a1b - RUNNING - - 0 - 0 - - 0 - 0 - 0 - 47 - 47 - 1 - 1 - 100 - - 1.0 - - - + 50.0 + 10.0 + -1.0 + 0.0 + 0 0 - - - - 70.0 - 0.0 - 100.0 - 7.35 - 50.0 - 0.0 - 0 - 2147483647 - <memory:0, vCores:0> - a2 - RUNNING - + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + 0 0 - - 0 - 0 - 0 - 735 - 73500 - 1 - 100 - 100 - - 100.0 - - - + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 4096 + 4 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 4 + + + + + 16384 + 16 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 16384 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 16 + + + + + + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 2048 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 2048 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + + + 4096 + 4 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 4 + + + + + 16384 + 16 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 16384 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 16 + + + + + 8192 + 4 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 8192 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 4 + + + + + + ADMINISTER_QUEUE + + + + APPLICATION_MAX_PRIORITY + * + + + SUBMIT_APP + + + + 0 + fifo + false + + percentage + leaf + static + off + + + + 0 + 0 + 0 + 1250 + 1250 + 100.0 + + 1.0 + 0.1 + + 2048 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 2048 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + 0 0 - + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 2048 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 2048 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + true + true + 0 + false + -1 + -1 - - 89.5 + + root.b + 50.0 0.0 100.0 - 89.5 + 50.0 100.0 0.0 - 2 + -1.0 + 0.0 + 0 2147483647 - <memory:0, vCores:0> b + false RUNNING - - - 60.000004 + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + false + * + 0 + 0 + 0 + + + + + [memory-mb=50.0%,vcores=50.0%] + + memory-mb + 50.0% + + + vcores + 50.0% + + + 50.0 0.0 100.0 - 53.7 - 100.0 + 50.0 0.0 - 2 - 2147483647 - <memory:0, vCores:0> - b1 - RUNNING - + 100.0 + 10.0 + -1.0 + 0.0 + 0 0 - - 1 - 1 - 0 - 5370 - 537000 - 1 - 100 - 100 - - - user2 - - 0 - 0 - - 1 - 0 - - - user1 - - 0 - 0 - - 0 - 1 - - - 100.0 - - - 39.5 - 0.0 - 100.0 - 35.3525 - 100.0 - 0.0 - 0 - 2147483647 - <memory:0, vCores:0> - b2 - RUNNING - + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 16384 + 16 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 16384 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 16 + + + + + 32768 + 32 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 32768 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 32 + + + + + + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 4096 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + 0 0 - - 0 - 0 - 0 - 3535 - 353500 - 1 - 100 - 100 - - 100.0 - - - 0.5 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + + + 16384 + 16 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 16384 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 16 + + + + + 32768 + 32 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 32768 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 32 + + + + + 8192 + 4 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 8192 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 4 + + + + + + ADMINISTER_QUEUE + + + + APPLICATION_MAX_PRIORITY + * + + + SUBMIT_APP + + + + 0 + fifo + false + + percentage + leaf + static + off + + + + 0 + 0 + 0 + 5000 + 5000 + 100.0 + + 1.0 + 0.1 + + 4096 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 4096 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + true + true + 0 + false + -1 + -1 + + + root.c + 37.5 + 0.0 + 100.0 + 37.5 + 100.0 + 0.0 + -1.0 + 0.0 + 0 + 2147483647 + c + false + RUNNING + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + false + * + 0 + 0 + 0 + + + + + [memory-mb=37.5%,vcores=37.5%] + + memory-mb + 37.5% + + + vcores + 37.5% + + + 37.5 0.0 100.0 - 0.4475 - 100.0 + 37.5 0.0 - 0 - 2147483647 - <memory:0, vCores:0> - b3 - RUNNING - + 100.0 + 10.0 + -1.0 + 0.0 + 0 0 - - 0 - 0 - 0 - 44 - 4400 - 1 - 100 - 100 - - 100.0 - - - + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 12288 + 12 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 12288 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 12 + + + + + 32768 + 32 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 32768 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 32 + + + + + + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 4096 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + + + 12288 + 12 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 12288 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 12 + + + + + 32768 + 32 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 32768 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 32 + + + + + 8192 + 4 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 8192 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 4 + + + + + + ADMINISTER_QUEUE + + + + APPLICATION_MAX_PRIORITY + * + + + SUBMIT_APP + + + + 0 + fifo + false + + percentage + leaf + static + off + + + + 0 + 0 + 0 + 3750 + 3750 + 100.0 + + 1.0 + 0.1 + + 4096 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + 0 0 - + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 4096 + 1 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 4096 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 1 + + + + true + true + 0 + false + -1 + -1 + + + + + [memory-mb=100.0%,vcores=100.0%] + + memory-mb + 100.0% + + + vcores + 100.0% + + + 100.0 + 0.0 + 100.0 + 100.0 + 0.0 + 100.0 + 0.0 + -1.0 + 0.0 + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 32768 + 32 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 32768 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 32 + + + + + 32768 + 32 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 32768 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 32 + + + + + - 1326381444693 + 0 last-allocation N/A @@ -1007,9 +3814,9 @@ Response Body: last-release - host.domain.com:8041 - container_1326821518301_0005_01_000001 - root.default + N/A + N/A + N/A last-preemption @@ -1029,6 +3836,26 @@ Response Body: 0 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + @@ -1037,6 +3864,26 @@ Response Body: 0 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + @@ -1045,9 +3892,76 @@ Response Body: 0 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + 8192 + 4 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 8192 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 4 + + + + + + ADMINISTER_QUEUE + * + + + APPLICATION_MAX_PRIORITY + * + + + SUBMIT_APP + * + + + 0 + utilization + percentage + parent + static + off + + + ``` From d9cb76ac985a1f28208e29a6f1efa6d161720fb0 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 27 Sep 2023 18:21:45 -0400 Subject: [PATCH 054/155] YARN-11468. Zookeeper SSL/TLS support. Contributed by Ferenc Erdelyi --- .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++++ .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 6 ++++++ .../hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bbb1ed6f8a7de..13d4c209110e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -857,6 +857,10 @@ public static boolean isAclEnabled(Configuration conf) { /** Zookeeper interaction configs */ public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-"; + /** Enable Zookeeper SSL/TLS communication. */ + public static final String RM_ZK_CLIENT_SSL_ENABLED = RM_ZK_PREFIX + "client-ssl.enabled"; + public static final boolean DEFAULT_RM_ZK_CLIENT_SSL_ENABLED = false; + public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9fa600db4b039..2259b73fb651b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -741,6 +741,12 @@ 1048576 + + Enable SSL/TLS encryption for the ZooKeeper communication. + yarn.resourcemanager.zk-client-ssl.enabled + false + + Name of the cluster. In a HA setting, this is used to ensure the RM participates in leader diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 2730dde72fc17..90eaed3d8a02a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -427,7 +427,8 @@ public ZKCuratorManager createAndStartZKManager(Configuration authInfos.add(authInfo); } - manager.start(authInfos); + manager.start(authInfos, config.getBoolean(YarnConfiguration.RM_ZK_CLIENT_SSL_ENABLED, + YarnConfiguration.DEFAULT_RM_ZK_CLIENT_SSL_ENABLED)); return manager; } From 0c153fe46507874c2aab12d00bd3bcec0ebfaaed Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Thu, 28 Sep 2023 10:17:29 +0900 Subject: [PATCH 055/155] YARN-11558. Fix dependency convergence error on hbase2 profile. (#6017) --- hadoop-project/pom.xml | 18 ++++++++++++++++++ .../pom.xml | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e95e7471c5533..2f487a439e4df 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1749,6 +1749,24 @@ org.apache.hbase hbase-server ${hbase.version} + + + org.osgi + org.osgi.core + + + javax.ws.rs + javax.ws.rs-api + + + org.javassist + javassist + + + javax.servlet + servlet-api + +     org.apache.hbase diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml index 6c0a635bc8370..0d61513889a5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml @@ -126,6 +126,11 @@ test-jar test + + junit + junit + test + org.junit.jupiter junit-jupiter-api From 2d871fab7888aa531a199aa52bc1e0875f6a85f3 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 27 Sep 2023 22:33:21 -0400 Subject: [PATCH 056/155] MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17 nodes. Contributed by Peter Szucs --- .../hadoop/yarn/conf/YarnConfiguration.java | 8 +++++ .../src/main/resources/yarn-default.xml | 11 ++++++ .../launcher/ContainerLaunch.java | 8 +++-- .../localizer/ContainerLocalizer.java | 15 ++++++++ .../launcher/TestContainerLaunch.java | 8 +++-- .../localizer/TestContainerLocalizer.java | 35 +++++++++++++++++++ 6 files changed, 81 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 13d4c209110e0..90a8978a228b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2232,6 +2232,14 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = "-Xmx256m"; + /* + * Flag to indicate whether JDK17's required add-exports flags should be added to + * container localizers regardless of the user specified JAVA_OPTS. + */ + public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY = + NM_PREFIX + "container-localizer.java.opts.add-exports-as-default"; + public static final boolean NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT = true; + /** The log level of container localizer process. */ public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL= NM_PREFIX + "container-localizer.log.level"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2259b73fb651b..9991e841d74b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1484,6 +1484,17 @@ -Xmx256m + + yarn.nodemanager.container-localizer.java.opts.add-exports-as-default + true + Since on JDK17 it is no longer possible to use the reflection API to + access public fields and methods, add-exports flags should be added to container + localizers regardless of the user specified JAVA_OPTS. Setting this to true will + add the flags to the container localizer on nodes with JDK17 or higher. + Defaults to true, but the setting has no effect on nodes using JDK16 and before. + + + The log level for container localizer while it is an independent process. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 143086db2aec1..5466182b715d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -123,6 +123,11 @@ public class ContainerLaunch implements Callable { private static final String PID_FILE_NAME_FMT = "%s.pid"; static final String EXIT_CODE_FILE_SUFFIX = ".exitcode"; + private static final String ADDITIONAL_JDK17_PLUS_OPTIONS = + "--add-opens=java.base/java.lang=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.util=ALL-UNNAMED"; + protected final Dispatcher dispatcher; protected final ContainerExecutor exec; protected final Application app; @@ -171,8 +176,7 @@ public static String expandEnvironment(String var, File.pathSeparator); if (Shell.isJavaVersionAtLeast(17)) { - var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, - "--add-opens=java.base/java.lang=ALL-UNNAMED"); + var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, ADDITIONAL_JDK17_PLUS_OPTIONS); } else { var = var.replace(ApplicationConstants.JVM_ADD_OPENS_VAR, ""); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 604a810ec9468..c928b19874edb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -103,6 +103,14 @@ public class ContainerLocalizer { new FsPermission((short) 0755); public static final String CSI_VOLIUME_MOUNTS_ROOT = "csivolumes"; + /* + * Testing discovered that these Java options are needed for Spark service + * running on JDK17 and Isilon clusters. + */ + private static final String ADDITIONAL_JDK17_PLUS_OPTIONS = + "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.util=ALL-UNNAMED"; + private final String user; private final String appId; private final List localDirs; @@ -400,6 +408,13 @@ private LocalizerStatus createStatus() throws InterruptedException { public static List getJavaOpts(Configuration conf) { String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY, YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT); + boolean isExtraJDK17OptionsConfigured = + conf.getBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY, + YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT); + + if (Shell.isJavaVersionAtLeast(17) && isExtraJDK17OptionsConfigured) { + opts = opts.trim().concat(" " + ADDITIONAL_JDK17_PLUS_OPTIONS); + } return Arrays.asList(opts.split(" ")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 6971d34b9d814..6a9dd6b7476bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -580,8 +580,12 @@ public void testEnvExpansion() throws IOException { String res = ContainerLaunch.expandEnvironment(input, logPath); - String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ? - "--add-opens=java.base/java.lang=ALL-UNNAMED" : ""; + String additionalJdk17PlusOptions = + "--add-opens=java.base/java.lang=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.util=ALL-UNNAMED"; + String expectedAddOpens = Shell.isJavaVersionAtLeast(17) ? additionalJdk17PlusOptions : ""; + if (Shell.WINDOWS) { Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;" + "%HADOOP_HOME%/share/hadoop/common/lib/*;" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java index 8d8f748347c9f..76aa73142e0cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java @@ -702,4 +702,39 @@ public void testUserCacheDirPermission() throws Exception { lfs.getFileStatus(destDirPath.getParent().getParent()).getPermission()); } + @Test + public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreConfigured() throws Exception { + ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper(); + ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest(); + + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY, + true); + + List javaOpts = localizer.getJavaOpts(conf); + + if (Shell.isJavaVersionAtLeast(17)) { + Assert.assertTrue(javaOpts.contains("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + Assert.assertTrue(javaOpts.contains("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } + Assert.assertTrue(javaOpts.contains("-Xmx256m")); + } + + @Test + public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreNotConfigured() throws Exception { + ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper(); + ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest(); + + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY, + false); + + List javaOpts = localizer.getJavaOpts(conf); + + if (Shell.isJavaVersionAtLeast(17)) { + Assert.assertFalse(javaOpts.contains("--add-exports=java.base/sun.net.dns=ALL-UNNAMED")); + Assert.assertFalse(javaOpts.contains("--add-exports=java.base/sun.net.util=ALL-UNNAMED")); + } + Assert.assertTrue(javaOpts.contains("-Xmx256m")); + } } From 35c42e4039a66c565a60d4c18905afa1887db92f Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 28 Sep 2023 06:52:31 +0100 Subject: [PATCH 057/155] HADOOP-18912. upgrade snappy-java to 1.1.10.4 (#6115). Contributed by PJ Fanning. Signed-off-by: Ayush Saxena --- LICENSE-binary | 2 +- hadoop-project/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 5a653da811602..1eecdf7dd11f6 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -357,7 +357,7 @@ org.ehcache:ehcache:3.3.1 org.ini4j:ini4j:0.5.4 org.lz4:lz4-java:1.7.1 org.objenesis:objenesis:2.6 -org.xerial.snappy:snappy-java:1.1.10.1 +org.xerial.snappy:snappy-java:1.1.10.4 org.yaml:snakeyaml:2.0 org.wildfly.openssl:wildfly-openssl:1.1.3.Final software.amazon.awssdk:bundle:jar:2.20.128 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 2f487a439e4df..4322a51cd2d47 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -138,7 +138,7 @@ 2.9.0 3.2.4 4.1.94.Final - 1.1.10.1 + 1.1.10.4 1.7.1 From 1d2afc5cf6f816461b76ec2bdbab8209052cd129 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 29 Sep 2023 07:15:53 +0800 Subject: [PATCH 058/155] YARN-8862. [BackPort] [GPG] Add Yarn Registry cleanup in ApplicationCleaner. (#6083) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../utils/FederationRegistryClient.java | 60 +++++++++++-------- .../utils/TestFederationRegistryClient.java | 27 +++++++++ .../globalpolicygenerator/GPGContext.java | 5 ++ .../globalpolicygenerator/GPGContextImpl.java | 12 ++++ .../GlobalPolicyGenerator.java | 21 +++++++ .../ApplicationCleaner.java | 30 +++++++--- .../DefaultApplicationCleaner.java | 2 + .../TestDefaultApplicationCleaner.java | 34 +++++++++++ 8 files changed, 159 insertions(+), 32 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java index fa64188a608b9..9e4d1e6ed0e81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.collections.MapUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.registry.client.api.BindFlags; @@ -142,9 +143,7 @@ public synchronized boolean writeAMRMTokenForUAM(ApplicationId appId, // Then update the subClusterTokenMap subClusterTokenMap.put(subClusterId, token); } catch (YarnException | IOException e) { - LOG.error( - "Failed writing AMRMToken to registry for subcluster " + subClusterId, - e); + LOG.error("Failed writing AMRMToken to registry for subcluster {}.", subClusterId, e); } return update; } @@ -189,8 +188,7 @@ public synchronized boolean writeAMRMTokenForUAM(ApplicationId appId, retMap.put(scId, amrmToken); } catch (Exception e) { - LOG.error("Failed reading registry key " + key - + ", skipping subcluster " + scId, e); + LOG.error("Failed reading registry key {}, skipping subcluster {}.", key, scId, e); } } @@ -202,24 +200,39 @@ public synchronized boolean writeAMRMTokenForUAM(ApplicationId appId, /** * Remove an application from registry. * - * @param appId application id + * @param appId application id. */ public synchronized void removeAppFromRegistry(ApplicationId appId) { + removeAppFromRegistry(appId, false); + } + + /** + * Remove an application from registry. + * + * @param appId application id + * @param ignoreMemoryState whether to ignore the memory data in terms of + * known application + */ + public synchronized void removeAppFromRegistry(ApplicationId appId, + boolean ignoreMemoryState) { Map> subClusterTokenMap = this.appSubClusterTokenMap.get(appId); - LOG.info("Removing all registry entries for {}", appId); - - if (subClusterTokenMap == null || subClusterTokenMap.size() == 0) { - return; + if (!ignoreMemoryState) { + if (MapUtils.isEmpty(subClusterTokenMap)) { + return; + } } + LOG.info("Removing all registry entries for {}.", appId); // Lastly remove the application directory String key = getRegistryKey(appId, null); try { removeKeyRegistry(this.registry, this.user, key, true, true); - subClusterTokenMap.clear(); + if (subClusterTokenMap != null) { + subClusterTokenMap.clear(); + } } catch (YarnException e) { - LOG.error("Failed removing registry directory key " + key, e); + LOG.error("Failed removing registry directory key {}.", key, e); } } @@ -247,7 +260,7 @@ public String run() { } } catch (Throwable e) { if (throwIfFails) { - LOG.error("Registry resolve key " + key + " failed", e); + LOG.error("Registry resolve key {} failed.", key, e); } } return null; @@ -271,7 +284,7 @@ public Boolean run() { return true; } catch (Throwable e) { if (throwIfFails) { - LOG.error("Registry remove key " + key + " failed", e); + LOG.error("Registry remove key {} failed.", key, e); } } return false; @@ -300,7 +313,7 @@ public Boolean run() { return true; } catch (Throwable e) { if (throwIfFails) { - LOG.error("Registry write key " + key + " failed", e); + LOG.error("Registry write key {} failed.", key, e); } } return false; @@ -317,18 +330,15 @@ public Boolean run() { private List listDirRegistry(final RegistryOperations registryImpl, UserGroupInformation ugi, final String key, final boolean throwIfFails) throws YarnException { - List result = ugi.doAs(new PrivilegedAction>() { - @Override - public List run() { - try { - return registryImpl.list(key); - } catch (Throwable e) { - if (throwIfFails) { - LOG.error("Registry list key " + key + " failed", e); - } + List result = ugi.doAs((PrivilegedAction>) () -> { + try { + return registryImpl.list(key); + } catch (Throwable e) { + if (throwIfFails) { + LOG.error("Registry list key {} failed.", key, e); } - return null; } + return null; }); if (result == null && throwIfFails) { throw new YarnException("Registry list key " + key + " failed"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java index 42be851512af3..cccfbb4613c0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java @@ -87,4 +87,31 @@ public void testBasicCase() { this.registryClient.loadStateFromRegistry(appId).size()); } + @Test + public void testRemoveWithMemoryState() { + ApplicationId appId1 = ApplicationId.newInstance(0, 0); + ApplicationId appId2 = ApplicationId.newInstance(0, 1); + String scId0 = "subcluster0"; + + this.registryClient.writeAMRMTokenForUAM(appId1, scId0, new Token<>()); + this.registryClient.writeAMRMTokenForUAM(appId2, scId0, new Token<>()); + Assert.assertEquals(2, this.registryClient.getAllApplications().size()); + + // Create a new client instance + this.registryClient = + new FederationRegistryClient(this.conf, this.registry, this.user); + + this.registryClient.loadStateFromRegistry(appId2); + // Should remove app2 + this.registryClient.removeAppFromRegistry(appId2, false); + Assert.assertEquals(1, this.registryClient.getAllApplications().size()); + + // Should not remove app1 since memory state don't have it + this.registryClient.removeAppFromRegistry(appId1, false); + Assert.assertEquals(1, this.registryClient.getAllApplications().size()); + + // Should remove app1 + this.registryClient.removeAppFromRegistry(appId1, true); + Assert.assertEquals(0, this.registryClient.getAllApplications().size()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java index 6b0a5a43112b0..e54244d7133d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; +import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; /** @@ -32,4 +33,8 @@ public interface GPGContext { GPGPolicyFacade getPolicyFacade(); void setPolicyFacade(GPGPolicyFacade facade); + + FederationRegistryClient getRegistryClient(); + + void setRegistryClient(FederationRegistryClient client); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java index bb498448fae81..b14f502990182 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator; +import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; /** @@ -27,6 +28,7 @@ public class GPGContextImpl implements GPGContext { private FederationStateStoreFacade facade; private GPGPolicyFacade policyFacade; + private FederationRegistryClient registryClient; @Override public FederationStateStoreFacade getStateStoreFacade() { @@ -48,4 +50,14 @@ public GPGPolicyFacade getPolicyFacade(){ public void setPolicyFacade(GPGPolicyFacade gpgPolicyfacade){ policyFacade = gpgPolicyfacade; } + + @Override + public FederationRegistryClient getRegistryClient() { + return registryClient; + } + + @Override + public void setRegistryClient(FederationRegistryClient client) { + registryClient = client; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index ba8ce856cdaa5..7ea2f5f27277b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -37,6 +37,7 @@ import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; @@ -46,6 +47,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.ApplicationCleaner; import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator; import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner; @@ -81,6 +83,7 @@ public class GlobalPolicyGenerator extends CompositeService { // Federation Variables private GPGContext gpgContext; + private RegistryOperations registry; // Scheduler service that runs tasks periodically private ScheduledThreadPoolExecutor scheduledExecutorService; @@ -123,6 +126,17 @@ protected void serviceInit(Configuration conf) throws Exception { new GPGPolicyFacade(this.gpgContext.getStateStoreFacade(), conf); this.gpgContext.setPolicyFacade(gpgPolicyFacade); + this.registry = FederationStateStoreFacade.createInstance(conf, + YarnConfiguration.YARN_REGISTRY_CLASS, + YarnConfiguration.DEFAULT_YARN_REGISTRY_CLASS, + RegistryOperations.class); + this.registry.init(conf); + + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + FederationRegistryClient registryClient = + new FederationRegistryClient(conf, this.registry, user); + this.gpgContext.setRegistryClient(registryClient); + this.scheduledExecutorService = new ScheduledThreadPoolExecutor( conf.getInt(YarnConfiguration.GPG_SCHEDULED_EXECUTOR_THREADS, YarnConfiguration.DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS)); @@ -157,6 +171,8 @@ protected void serviceStart() throws Exception { super.serviceStart(); + this.registry.start(); + // Schedule SubClusterCleaner service Configuration config = getConfig(); long scCleanerIntervalMs = config.getTimeDuration( @@ -214,6 +230,11 @@ protected void serviceStart() throws Exception { @Override protected void serviceStop() throws Exception { + if (this.registry != null) { + this.registry.stop(); + this.registry = null; + } + try { if (this.scheduledExecutorService != null && !this.scheduledExecutorService.isShutdown()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java index cd3f7618558e9..af0bd6184b797 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.commons.lang3.time.DurationFormatUtils; @@ -27,9 +28,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext; import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils; import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -46,6 +49,7 @@ public abstract class ApplicationCleaner implements Runnable { private Configuration conf; private GPGContext gpgContext; + private FederationRegistryClient registryClient; private int minRouterSuccessCount; private int maxRouterRetry; @@ -56,6 +60,7 @@ public void init(Configuration config, GPGContext context) this.gpgContext = context; this.conf = config; + this.registryClient = context.getRegistryClient(); String routerSpecString = this.conf.get(YarnConfiguration.GPG_APPCLEANER_CONTACT_ROUTER_SPEC, @@ -80,10 +85,9 @@ public void init(Configuration config, GPGContext context) + this.minRouterSuccessCount + " should be positive"); } - LOG.info( - "Initialized AppCleaner with Router query with min success {}, " - + "max retry {}, retry interval {}", - this.minRouterSuccessCount, this.maxRouterRetry, + LOG.info("Initialized AppCleaner with Router query with min success {}, " + + "max retry {}, retry interval {}.", this.minRouterSuccessCount, + this.maxRouterRetry, DurationFormatUtils.formatDurationISO(this.routerQueryIntevalMillis)); } @@ -100,9 +104,9 @@ public GPGContext getGPGContext() { public Set getAppsFromRouter() throws YarnRuntimeException { String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); - LOG.info(String.format("Contacting router at: %s", webAppAddress)); - AppsInfo appsInfo = GPGUtils.invokeRMWebService(webAppAddress, "apps", AppsInfo.class, conf, - DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString()); + LOG.info("Contacting router at: {}.", webAppAddress); + AppsInfo appsInfo = GPGUtils.invokeRMWebService(webAppAddress, RMWSConsts.APPS, + AppsInfo.class, conf, DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString()); Set appSet = new HashSet<>(); for (AppInfo appInfo : appsInfo.getApps()) { @@ -148,6 +152,18 @@ public Set getRouterKnownApplications() throws YarnException { + " success Router queries after " + totalAttemptCount + " retries"); } + protected void cleanupAppRecordInRegistry(Set knownApps) { + List allApps = this.registryClient.getAllApplications(); + LOG.info("Got {} existing apps in registry.", allApps.size()); + for (String app : allApps) { + ApplicationId appId = ApplicationId.fromString(app); + if (!knownApps.contains(appId)) { + LOG.info("removing finished application entry for {}", app); + this.registryClient.removeAppFromRegistry(appId, true); + } + } + } + @Override public abstract void run(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java index 857d2e645d4c4..5b2ff26fcfb4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java @@ -70,6 +70,8 @@ public void run() { LOG.error("deleteApplicationHomeSubCluster failed at application {}.", appId, e); } } + // Clean up registry entries + cleanupAppRecordInRegistry(routerApps); } catch (Throwable e) { LOG.error("Application cleaner started at time {} fails. ", now, e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java index 2d63c48236fb5..1e703b51960e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java @@ -24,15 +24,21 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext; import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContextImpl; @@ -50,6 +56,8 @@ public class TestDefaultApplicationCleaner { private FederationStateStoreFacade facade; private ApplicationCleaner appCleaner; private GPGContext gpgContext; + private RegistryOperations registry; + private FederationRegistryClient registryClient; private List appIds; // The list of applications returned by mocked router @@ -68,8 +76,18 @@ public void setup() throws Exception { facade = FederationStateStoreFacade.getInstance(); facade.reinitialize(stateStore, conf); + registry = new FSRegistryOperationsService(); + registry.init(conf); + registry.start(); + + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + registryClient = new FederationRegistryClient(conf, registry, user); + registryClient.cleanAllApplications(); + Assert.assertEquals(0, registryClient.getAllApplications().size()); + gpgContext = new GPGContextImpl(); gpgContext.setStateStoreFacade(facade); + gpgContext.setRegistryClient(registryClient); appCleaner = new TestableDefaultApplicationCleaner(); appCleaner.init(conf, gpgContext); @@ -87,7 +105,12 @@ public void setup() throws Exception { stateStore.addApplicationHomeSubCluster( AddApplicationHomeSubClusterRequest.newInstance( ApplicationHomeSubCluster.newInstance(appId, subClusterId))); + + // Write some registry entries for the app + registryClient.writeAMRMTokenForUAM(appId, subClusterId.toString(), + new Token()); } + Assert.assertEquals(3, registryClient.getAllApplications().size()); } @After @@ -96,6 +119,14 @@ public void breakDown() { stateStore.close(); stateStore = null; } + if (registryClient != null) { + registryClient.cleanAllApplications(); + registryClient = null; + } + if (registry != null) { + registry.stop(); + registry = null; + } } @Test @@ -116,6 +147,9 @@ public void testFederationStateStoreAppsCleanUp() throws YarnException { .getApplicationsHomeSubCluster( GetApplicationsHomeSubClusterRequest.newInstance()) .getAppsHomeSubClusters().size()); + + // The known app should not be cleaned in registry + Assert.assertEquals(1, registryClient.getAllApplications().size()); } /** From 390cd294f8c698058656cddc68a30882a090dd0e Mon Sep 17 00:00:00 2001 From: xiaojunxiang <65019264+hellosrc@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:25:59 +0800 Subject: [PATCH 059/155] HDFS-17211. Fix comments in the RemoteParam class. (#6124). Contributed hellosrc. Reviewed-by: Xing Lin Signed-off-by: Ayush Saxena --- .../hadoop/hdfs/server/federation/router/RemoteParam.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java index 8b216d919ed0c..b6b4ef731c277 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteParam.java @@ -35,7 +35,7 @@ public class RemoteParam { /** * Constructs a default remote parameter. Always maps the value to the - * destination of the provided RemoveLocationContext. + * destination of the provided RemoteLocationContext. */ public RemoteParam() { this.paramMap = null; From 8931393302c67053e7d8d8c57c5970732cea4f8d Mon Sep 17 00:00:00 2001 From: ConfX <114765570+teamconfx@users.noreply.github.com> Date: Fri, 29 Sep 2023 01:36:24 -0500 Subject: [PATCH 060/155] HDFS-17133: TestFsDatasetImpl missing null check when cleaning up (#6079). Contributed by ConfX. Signed-off-by: Ayush Saxena --- .../server/datanode/fsdataset/impl/TestFsDatasetImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index 1c94e351c2b52..585c36fa995e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -1173,7 +1173,7 @@ private void testMoveBlockFailure(Configuration config) { LOG.info("Exception in testMoveBlockFailure ", ex); fail("Exception while testing testMoveBlockFailure "); } finally { - if (cluster.isClusterUp()) { + if (cluster != null && cluster.isClusterUp()) { cluster.shutdown(); } } @@ -1984,4 +1984,4 @@ public void tesInvalidateMissingBlock() throws Exception { cluster.shutdown(); } } -} \ No newline at end of file +} From 5f47f091a227ecd1d9b1b375e9c71e01d31a264f Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 29 Sep 2023 16:59:01 +0800 Subject: [PATCH 061/155] YARN-11537. [Addendum][Federation] Router CLI Supports List SubClusterPolicyConfiguration Of Queues. (#6121) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../rmadmin/FederationRMAdminInterceptor.java | 79 +++++++++++++++++-- .../TestFederationRMAdminInterceptor.java | 23 ++++++ 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index 268522c1b1921..b7c1462a60d56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -130,7 +130,7 @@ public void init(String userName) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("RPC Router RMAdminClient-" + userName + "-%d ").build(); - BlockingQueue workQueue = new LinkedBlockingQueue(); + BlockingQueue workQueue = new LinkedBlockingQueue<>(); this.executorService = new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.MILLISECONDS, workQueue, threadFactory); @@ -1032,7 +1032,7 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( } try { - QueryFederationQueuePoliciesResponse response = null; + QueryFederationQueuePoliciesResponse response; long startTime = clock.getTime(); String queue = request.getQueue(); @@ -1056,9 +1056,15 @@ public QueryFederationQueuePoliciesResponse listFederationQueuePolicies( // We filter by pagination. response = filterPoliciesConfigurationsByQueues(queues, policiesConfigurations, pageSize, currentPage); + } else { + // If we don't have any filtering criteria, we should also support paginating the results. + response = filterPoliciesConfigurations(policiesConfigurations, pageSize, currentPage); } long stopTime = clock.getTime(); routerMetrics.succeededListFederationQueuePoliciesRetrieved(stopTime - startTime); + if (response == null) { + response = QueryFederationQueuePoliciesResponse.newInstance(); + } return response; } catch (Exception e) { routerMetrics.incrListFederationQueuePoliciesFailedRetrieved(); @@ -1137,12 +1143,75 @@ private QueryFederationQueuePoliciesResponse filterPoliciesConfigurationsByQueue } } + // Step3. To paginate the returned results. + return queryFederationQueuePoliciesPagination(federationQueueWeights, pageSize, currentPage); + } + + /** + * Filter PoliciesConfigurations, and we paginate Policies within this method. + * + * @param policiesConfigurations policy configurations. + * @param pageSize Items per page. + * @param currentPage The number of pages to be queried. + * @return federation queue policies response. + * @throws YarnException indicates exceptions from yarn servers. + */ + private QueryFederationQueuePoliciesResponse filterPoliciesConfigurations( + Map policiesConfigurations, + int pageSize, int currentPage) throws YarnException { + + // Step1. Check the parameters, if the policy list is empty, return empty directly. + if (MapUtils.isEmpty(policiesConfigurations)) { + return null; + } + + // Step2. Traverse policiesConfigurations and obtain the FederationQueueWeight list. + List federationQueueWeights = new ArrayList<>(); + for (Map.Entry entry : + policiesConfigurations.entrySet()) { + String queue = entry.getKey(); + SubClusterPolicyConfiguration policyConf = entry.getValue(); + if (policyConf == null) { + continue; + } + FederationQueueWeight federationQueueWeight = parseFederationQueueWeight(queue, policyConf); + if (federationQueueWeight != null) { + federationQueueWeights.add(federationQueueWeight); + } + } + + // Step3. To paginate the returned results. + return queryFederationQueuePoliciesPagination(federationQueueWeights, pageSize, currentPage); + } + + /** + * Pagination for FederationQueuePolicies. + * + * @param queueWeights List Of FederationQueueWeight. + * @param pageSize Items per page. + * @param currentPage The number of pages to be queried. + * @return federation queue policies response. + * @throws YarnException indicates exceptions from yarn servers. + */ + private QueryFederationQueuePoliciesResponse queryFederationQueuePoliciesPagination( + List queueWeights, int pageSize, int currentPage) + throws YarnException { + if (CollectionUtils.isEmpty(queueWeights)) { + return null; + } + int startIndex = (currentPage - 1) * pageSize; - int endIndex = Math.min(startIndex + pageSize, federationQueueWeights.size()); + int endIndex = Math.min(startIndex + pageSize, queueWeights.size()); + + if (startIndex > endIndex) { + throw new YarnException("The index of the records to be retrieved " + + "has exceeded the maximum index."); + } + List subFederationQueueWeights = - federationQueueWeights.subList(startIndex, endIndex); + queueWeights.subList(startIndex, endIndex); - int totalSize = federationQueueWeights.size(); + int totalSize = queueWeights.size(); int totalPage = (totalSize % pageSize == 0) ? totalSize / pageSize : (totalSize / pageSize) + 1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java index 8439b01222e2c..520c25d22cb09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java @@ -965,5 +965,28 @@ public void testFilterPoliciesConfigurationsByQueues() throws Exception { List federationQueueWeights6 = response6.getFederationQueueWeights(); assertNotNull(federationQueueWeights6); assertEquals(1, federationQueueWeights6.size()); + + // Queue7: We design such a test case, we do not set any filter conditions, + // but we need to get the return results + QueryFederationQueuePoliciesRequest request7 = + QueryFederationQueuePoliciesRequest.newInstance(10, 1, null, null); + QueryFederationQueuePoliciesResponse response7 = + interceptor.listFederationQueuePolicies(request7); + assertNotNull(response7); + assertEquals(1, response7.getCurrentPage()); + assertEquals(10, response7.getPageSize()); + assertEquals(3, response7.getTotalPage()); + assertEquals(26, response7.getTotalSize()); + List federationQueueWeights7 = response7.getFederationQueueWeights(); + assertNotNull(federationQueueWeights7); + assertEquals(10, federationQueueWeights7.size()); + + // Queue8: We are designing a unit test where the number of records + // we need to retrieve exceeds the maximum number of Policies available. + QueryFederationQueuePoliciesRequest request8 = + QueryFederationQueuePoliciesRequest.newInstance(10, 10, null, null); + LambdaTestUtils.intercept(YarnException.class, + "The index of the records to be retrieved has exceeded the maximum index.", + () -> interceptor.listFederationQueuePolicies(request8)); } } From b8815fe68bddd670174b6f1fa19aff178dec7b59 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sun, 1 Oct 2023 19:55:32 +0800 Subject: [PATCH 062/155] MAPREDUCE-7453. Revert HADOOP-18649. (#6102). Contributed by zhengchenyu. In container-log4j.properties, log4j.appender.{APPENDER}.MaxFileSize is set to ${yarn.app.container.log.filesize}, but yarn.app.container.log.filesize is 0 in default. So log is missing. This log is always rolling and only show the latest log. --- .../hadoop/mapreduce/v2/util/MRApps.java | 5 +- .../src/main/resources/mapred-default.xml | 15 +- .../hadoop/yarn/ContainerLogAppender.java | 128 ++++++++++++++++++ .../yarn/ContainerRollingLogAppender.java | 75 ++++++++++ .../hadoop/yarn/TestContainerLogAppender.java | 48 +++++++ .../main/resources/container-log4j.properties | 39 +++--- 6 files changed, 286 insertions(+), 24 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 72dd48b09c2cb..a3ccfd72d8ce8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -60,6 +60,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.util.ApplicationClassLoader; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.ContainerLogAppender; +import org.apache.hadoop.yarn.ContainerRollingLogAppender; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -586,7 +588,8 @@ public static String getChildLogLevel(Configuration conf, boolean isMap) { /** * Add the JVM system properties necessary to configure - * {@link org.apache.log4j.RollingFileAppender}. + * {@link ContainerLogAppender} or + * {@link ContainerRollingLogAppender}. * * @param task for map/reduce, or null for app master * @param vargs the argument list to append to diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a6d68acda34ab..9b0d8b563d7bd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -840,8 +840,11 @@ yarn.app.mapreduce.task.container.log.backups 0 Number of backup files for task logs when using - RollingFileAppender (RFA). See - org.apache.log4j.RollingFileAppender.maxBackupIndex. + ContainerRollingLogAppender (CRLA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, + ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA + is enabled for tasks when both mapreduce.task.userlog.limit.kb and + yarn.app.mapreduce.task.container.log.backups are greater than zero. @@ -849,8 +852,12 @@ yarn.app.mapreduce.am.container.log.backups 0 Number of backup files for the ApplicationMaster logs when using - RollingFileAppender (RFA). See - org.apache.log4j.RollingFileAppender.maxBackupIndex. + ContainerRollingLogAppender (CRLA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, + ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA + is enabled for the ApplicationMaster when both + yarn.app.mapreduce.am.container.log.limit.kb and + yarn.app.mapreduce.am.container.log.backups are greater than zero. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java new file mode 100644 index 0000000000000..03a0078167f80 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import java.io.File; +import java.io.Flushable; +import java.util.ArrayDeque; +import java.util.Deque; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.log4j.FileAppender; +import org.apache.log4j.spi.LoggingEvent; + +/** + * A simple log4j-appender for container's logs. + */ +@Public +@Unstable +public class ContainerLogAppender extends FileAppender + implements Flushable { + + private String containerLogDir; + private String containerLogFile; + private int maxEvents; + private Deque eventBuffer; + private boolean closed = false; + + @Override + public synchronized void activateOptions() { + if (maxEvents > 0) { + this.eventBuffer = new ArrayDeque<>(); + } + setFile(new File(this.containerLogDir, containerLogFile).toString()); + setAppend(true); + super.activateOptions(); + } + + @Override + public synchronized void append(LoggingEvent event) { + if (closed) { + return; + } + if (eventBuffer != null) { + if (eventBuffer.size() == maxEvents) { + eventBuffer.removeFirst(); + } + eventBuffer.addLast(event); + } else { + super.append(event); + } + } + + @Override + public void flush() { + if (qw != null) { + qw.flush(); + } + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + if (eventBuffer != null) { + for (LoggingEvent event : eventBuffer) { + super.append(event); + } + // let garbage collection do its work + eventBuffer = null; + } + super.close(); + } + } + + /** + * Getter/Setter methods for log4j. + * + * @return containerLogDir. + */ + public String getContainerLogDir() { + return this.containerLogDir; + } + + public void setContainerLogDir(String containerLogDir) { + this.containerLogDir = containerLogDir; + } + + public String getContainerLogFile() { + return containerLogFile; + } + + public void setContainerLogFile(String containerLogFile) { + this.containerLogFile = containerLogFile; + } + + private static final long EVENT_SIZE = 100; + + public long getTotalLogFileSize() { + return maxEvents * EVENT_SIZE; + } + + /** + * Setter so that log4j can configure it from the + * configuration(log4j.properties). + * + * @param logSize log size. + */ + public void setTotalLogFileSize(long logSize) { + maxEvents = (int)(logSize / EVENT_SIZE); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java new file mode 100644 index 0000000000000..acb47d9dd002b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.log4j.RollingFileAppender; + +import java.io.File; +import java.io.Flushable; + +/** + * A simple log4j-appender for container's logs. + * + */ +@Public +@Unstable +public class ContainerRollingLogAppender extends RollingFileAppender implements Flushable { + private String containerLogDir; + private String containerLogFile; + + @Override + public void activateOptions() { + synchronized (this) { + setFile(new File(this.containerLogDir, containerLogFile).toString()); + setAppend(true); + super.activateOptions(); + } + } + + @Override + public void flush() { + if (qw != null) { + qw.flush(); + } + } + + /** + * Getter/Setter methods for log4j. + * + * @return containerLogDir. + */ + + public String getContainerLogDir() { + return this.containerLogDir; + } + + public void setContainerLogDir(String containerLogDir) { + this.containerLogDir = containerLogDir; + } + + public String getContainerLogFile() { + return containerLogFile; + } + + public void setContainerLogFile(String containerLogFile) { + this.containerLogFile = containerLogFile; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java new file mode 100644 index 0000000000000..26acfd7bad87b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import org.junit.jupiter.api.Test; + +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; + +public class TestContainerLogAppender { + + @Test + void testAppendInClose() throws Exception { + final ContainerLogAppender claAppender = new ContainerLogAppender(); + claAppender.setName("testCLA"); + claAppender.setLayout(new PatternLayout("%-5p [%t]: %m%n")); + claAppender.setContainerLogDir("target/testAppendInClose/logDir"); + claAppender.setContainerLogFile("syslog"); + claAppender.setTotalLogFileSize(1000); + claAppender.activateOptions(); + final Logger claLog = Logger.getLogger("testAppendInClose-catergory"); + claLog.setAdditivity(false); + claLog.addAppender(claAppender); + claLog.info(new Object() { + public String toString() { + claLog.info("message1"); + return "return message1"; + } + }); + claAppender.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index c5371c6d9ef4f..678e3a74c897a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -26,35 +26,36 @@ log4j.threshold=ALL # #Default values -yarn.app.container.log.filesize=100MB -yarn.app.container.log.backups=1 -yarn.app.mapreduce.shuffle.log.backups=1 +yarn.app.container.log.dir=null +yarn.app.container.log.filesize=100 -log4j.appender.CLA=org.apache.log4j.RollingFileAppender -log4j.appender.CLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} -log4j.appender.CLA.MaxFileSize=${yarn.app.container.log.filesize} -log4j.appender.CLA.MaxBackupIndex=${yarn.app.container.log.backups} +log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender +log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.CLA.containerLogFile=${hadoop.root.logfile} +log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize} log4j.appender.CLA.layout=org.apache.log4j.PatternLayout log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.CRLA=org.apache.log4j.RollingFileAppender -log4j.appender.CRLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} -log4j.appender.CRLA.MaxFileSize=${yarn.app.container.log.filesize} -log4j.appender.CRLA.MaxBackupIndex=${yarn.app.container.log.backups} +log4j.appender.CRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender +log4j.appender.CRLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.CRLA.containerLogFile=${hadoop.root.logfile} +log4j.appender.CRLA.maximumFileSize=${yarn.app.container.log.filesize} +log4j.appender.CRLA.maxBackupIndex=${yarn.app.container.log.backups} log4j.appender.CRLA.layout=org.apache.log4j.PatternLayout log4j.appender.CRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.shuffleCLA=org.apache.log4j.RollingFileAppender -log4j.appender.shuffleCLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile} -log4j.appender.shuffleCLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize} -log4j.appender.shuffleCLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} +log4j.appender.shuffleCLA=org.apache.hadoop.yarn.ContainerLogAppender +log4j.appender.shuffleCLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.shuffleCLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile} +log4j.appender.shuffleCLA.totalLogFileSize=${yarn.app.mapreduce.shuffle.log.filesize} log4j.appender.shuffleCLA.layout=org.apache.log4j.PatternLayout log4j.appender.shuffleCLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.shuffleCRLA=org.apache.log4j.RollingFileAppender -log4j.appender.shuffleCRLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile} -log4j.appender.shuffleCRLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize} -log4j.appender.shuffleCRLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} +log4j.appender.shuffleCRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender +log4j.appender.shuffleCRLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.shuffleCRLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile} +log4j.appender.shuffleCRLA.maximumFileSize=${yarn.app.mapreduce.shuffle.log.filesize} +log4j.appender.shuffleCRLA.maxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} log4j.appender.shuffleCRLA.layout=org.apache.log4j.PatternLayout log4j.appender.shuffleCRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n From b87180568b261a617dc9645197615efd4c753786 Mon Sep 17 00:00:00 2001 From: Wang Yu Date: Sun, 1 Oct 2023 20:00:59 +0800 Subject: [PATCH 063/155] HDFS-17209. Correct comments to align with the code (#6110). Contributed by Yu Wang. Reviewed-by: Inigo Goiri Signed-off-by: Ayush Saxena --- .../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 42082019bad20..4ade6ada72d04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -2569,7 +2569,7 @@ public void shutdown() { if (this.threadGroup != null) { int sleepMs = 2; while (true) { - // When shutting down for restart, wait 2.5 seconds before forcing + // When shutting down for restart, wait 1 second before forcing // termination of receiver threads. if (!this.shutdownForUpgrade || (this.shutdownForUpgrade && (Time.monotonicNow() - timeNotified From a04a9e107b57afb37740ac73ca239d3c335f8695 Mon Sep 17 00:00:00 2001 From: Tamas Domok Date: Mon, 2 Oct 2023 15:20:47 +0200 Subject: [PATCH 064/155] YARN-11578. Cache fs supports chmod in LogAggregationFileController. (#6120) --- .../LogAggregationFileController.java | 73 +++++++++++++----- .../TestLogAggregationFileController.java | 76 +++++++++++++++---- 2 files changed, 119 insertions(+), 30 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 7803c6215f209..8fe4f828ebf20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -33,7 +33,9 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -111,6 +113,35 @@ public abstract class LogAggregationFileController { protected boolean fsSupportsChmod = true; + private static class FsLogPathKey { + private Class fsType; + private Path logPath; + + FsLogPathKey(Class fsType, Path logPath) { + this.fsType = fsType; + this.logPath = logPath; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FsLogPathKey that = (FsLogPathKey) o; + return Objects.equals(fsType, that.fsType) && Objects.equals(logPath, that.logPath); + } + + @Override + public int hashCode() { + return Objects.hash(fsType, logPath); + } + } + private static final ConcurrentHashMap FS_CHMOD_CACHE + = new ConcurrentHashMap<>(); + public LogAggregationFileController() {} /** @@ -429,26 +460,34 @@ public void verifyAndCreateRemoteLogDir() { + remoteRootLogDir + "]", e); } } else { - //Check if FS has capability to set/modify permissions - Path permissionCheckFile = new Path(qualified, String.format("%s.permission_check", - RandomStringUtils.randomAlphanumeric(8))); + final FsLogPathKey key = new FsLogPathKey(remoteFS.getClass(), qualified); + FileSystem finalRemoteFS = remoteFS; + fsSupportsChmod = FS_CHMOD_CACHE.computeIfAbsent(key, + k -> checkFsSupportsChmod(finalRemoteFS, remoteRootLogDir, qualified)); + } + } + + private boolean checkFsSupportsChmod(FileSystem remoteFS, Path logDir, Path qualified) { + //Check if FS has capability to set/modify permissions + Path permissionCheckFile = new Path(qualified, String.format("%s.permission_check", + RandomStringUtils.randomAlphanumeric(8))); + try { + remoteFS.createNewFile(permissionCheckFile); + remoteFS.setPermission(permissionCheckFile, new FsPermission(TLDIR_PERMISSIONS)); + return true; + } catch (UnsupportedOperationException use) { + LOG.info("Unable to set permissions for configured filesystem since" + + " it does not support this {}", remoteFS.getScheme()); + } catch (IOException e) { + LOG.warn("Failed to check if FileSystem supports permissions on " + + "remoteLogDir [{}]", logDir, e); + } finally { try { - remoteFS.createNewFile(permissionCheckFile); - remoteFS.setPermission(permissionCheckFile, new FsPermission(TLDIR_PERMISSIONS)); - } catch (UnsupportedOperationException use) { - LOG.info("Unable to set permissions for configured filesystem since" - + " it does not support this {}", remoteFS.getScheme()); - fsSupportsChmod = false; - } catch (IOException e) { - LOG.warn("Failed to check if FileSystem supports permissions on " - + "remoteLogDir [" + remoteRootLogDir + "]", e); - } finally { - try { - remoteFS.delete(permissionCheckFile, false); - } catch (IOException ignored) { - } + remoteFS.delete(permissionCheckFile, false); + } catch (IOException ignored) { } } + return false; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java index fe1c5f2fa7328..a7c653b8187c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatcher; @@ -35,12 +37,14 @@ import static org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController.TLDIR_PERMISSIONS; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; /** @@ -116,30 +120,76 @@ public String toString() { @Test void testRemoteDirCreationWithCustomUser() throws Exception { + LogAggregationFileController controller = mock( + LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS); FileSystem fs = mock(FileSystem.class); - doReturn(new URI("")).when(fs).getUri(); - doReturn(new FileStatus(128, false, 0, 64, System.currentTimeMillis(), - System.currentTimeMillis(), new FsPermission(TLDIR_PERMISSIONS), - "not_yarn_user", "yarn_group", new Path("/tmp/logs"))).when(fs) - .getFileStatus(any(Path.class)); + setupCustomUserMocks(controller, fs, "/tmp/logs"); - Configuration conf = new Configuration(); + controller.initialize(new Configuration(), "TFile"); + controller.fsSupportsChmod = false; + + controller.verifyAndCreateRemoteLogDir(); + assertPermissionFileWasUsedOneTime(fs); + assertTrue(controller.fsSupportsChmod); + + doThrow(UnsupportedOperationException.class).when(fs).setPermission(any(), any()); + controller.verifyAndCreateRemoteLogDir(); + assertPermissionFileWasUsedOneTime(fs); // still once -> cached + assertTrue(controller.fsSupportsChmod); + + controller.fsSupportsChmod = false; + controller.verifyAndCreateRemoteLogDir(); + assertPermissionFileWasUsedOneTime(fs); // still once -> cached + assertTrue(controller.fsSupportsChmod); + } + + @Test + void testRemoteDirCreationWithCustomUserFsChmodNotSupported() throws Exception { LogAggregationFileController controller = mock( LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS); + FileSystem fs = mock(FileSystem.class); + setupCustomUserMocks(controller, fs, "/tmp/logs2"); + doThrow(UnsupportedOperationException.class).when(fs).setPermission(any(), any()); + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, "/tmp/logs2"); + controller.initialize(conf, "TFile"); + controller.verifyAndCreateRemoteLogDir(); + assertPermissionFileWasUsedOneTime(fs); + assertFalse(controller.fsSupportsChmod); + + controller.verifyAndCreateRemoteLogDir(); + assertPermissionFileWasUsedOneTime(fs); // still once -> cached + assertFalse(controller.fsSupportsChmod); + controller.fsSupportsChmod = true; + controller.verifyAndCreateRemoteLogDir(); + assertPermissionFileWasUsedOneTime(fs); // still once -> cached + assertFalse(controller.fsSupportsChmod); + } + + private static void setupCustomUserMocks(LogAggregationFileController controller, + FileSystem fs, String path) + throws URISyntaxException, IOException { + doReturn(new URI("")).when(fs).getUri(); + doReturn(new FileStatus(128, false, 0, 64, System.currentTimeMillis(), + System.currentTimeMillis(), new FsPermission(TLDIR_PERMISSIONS), + "not_yarn_user", "yarn_group", new Path(path))).when(fs) + .getFileStatus(any(Path.class)); doReturn(fs).when(controller).getFileSystem(any(Configuration.class)); UserGroupInformation ugi = UserGroupInformation.createUserForTesting( "yarn_user", new String[]{"yarn_group", "other_group"}); UserGroupInformation.setLoginUser(ugi); + } - controller.initialize(conf, "TFile"); - controller.verifyAndCreateRemoteLogDir(); - - verify(fs).createNewFile(argThat(new PathContainsString(".permission_check"))); - verify(fs).setPermission(argThat(new PathContainsString(".permission_check")), + private static void assertPermissionFileWasUsedOneTime(FileSystem fs) throws IOException { + verify(fs, times(1)) + .createNewFile(argThat(new PathContainsString(".permission_check"))); + verify(fs, times(1)) + .setPermission(argThat(new PathContainsString(".permission_check")), eq(new FsPermission(TLDIR_PERMISSIONS))); - verify(fs).delete(argThat(new PathContainsString(".permission_check")), eq(false)); - assertTrue(controller.fsSupportsChmod); + verify(fs, times(1)) + .delete(argThat(new PathContainsString(".permission_check")), eq(false)); } } From fe3984aa010b7a470bf3dcb00029369435382d1b Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:21:52 +0800 Subject: [PATCH 065/155] YARN-11580. YARN Router Web supports displaying information for Non-Federation. (#6127) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../store/records/SubClusterInfo.java | 7 + .../yarn/server/router/RouterServerUtil.java | 1 + .../yarn/server/router/webapp/AppsBlock.java | 10 +- .../server/router/webapp/FederationBlock.java | 203 +++++++++++------- .../router/webapp/MetricsOverviewTable.java | 158 +++++++++++--- .../yarn/server/router/webapp/NavBlock.java | 5 +- .../server/router/webapp/NodeLabelsBlock.java | 32 ++- .../yarn/server/router/webapp/NodesBlock.java | 11 +- .../server/router/webapp/RouterBlock.java | 133 ++++++++++-- .../webapp/dao/RouterSchedulerMetrics.java | 13 +- .../router/webapp/TestFederationWebApp.java | 2 + .../src/test/resources/yarn-site.xml | 4 + 12 files changed, 439 insertions(+), 140 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java index 40b87c7eb094e..f5a8f04f167d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java @@ -76,6 +76,13 @@ public static SubClusterInfo newInstance(SubClusterId subClusterId, return subClusterInfo; } + public static SubClusterInfo newInstance(SubClusterId subClusterId, + String rmWebServiceAddress, SubClusterState state, long lastStartTime, long lastHeartBeat, + String capability) { + return newInstance(subClusterId, null, null, null, + rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability); + } + /** * Get the {@link SubClusterId} representing the unique identifier of the * subcluster. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index 94dbcdce1ae2f..096d8dac0a76c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -661,6 +661,7 @@ public static ReservationDefinition convertReservationDefinition( * - if its size is within limits. * * @param appContext the app context to check. + * @param conf Configuration. * @throws IOException if an IO error occurred. * @throws YarnException yarn exception. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java index 49ad710bb11de..4fa07b70fa71b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java @@ -94,11 +94,13 @@ private static String escape(String str) { } private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) { + String webAddress = null; if (isEnabled) { - String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf); - return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY); + webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf); + } else { + webAddress = WebAppUtils.getRMWebAppURLWithScheme(this.conf); } - return null; + return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY); } private AppsInfo getSubClusterAppsInfo(String subCluster, String states) { @@ -110,7 +112,7 @@ private AppsInfo getSubClusterAppsInfo(String subCluster, String states) { if (subClusterInfo != null) { // Prepare webAddress String webAddress = subClusterInfo.getRMWebServiceAddress(); - String herfWebAppAddress = ""; + String herfWebAppAddress; if (webAddress != null && !webAddress.isEmpty()) { herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress; return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java index 44f8d7407bf20..7876d5d6f1f97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java @@ -26,6 +26,7 @@ import java.util.Date; import com.google.gson.Gson; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; @@ -67,7 +68,7 @@ public void render(Block html) { * @param capability metric json obtained from RM. * @return ClusterMetricsInfo Object */ - private ClusterMetricsInfo getClusterMetricsInfo(String capability) { + protected ClusterMetricsInfo getClusterMetricsInfo(String capability) { try { if (capability != null && !capability.isEmpty()) { JSONJAXBContext jc = new JSONJAXBContext( @@ -125,79 +126,10 @@ private void initHtmlPageFederation(Block html, boolean isEnabled) { .__().__().tbody(); try { - - // Sort the SubClusters - List subclusters = getSubClusterInfoList(); - - for (SubClusterInfo subcluster : subclusters) { - - Map subclusterMap = new HashMap<>(); - - // Prepare subCluster - SubClusterId subClusterId = subcluster.getSubClusterId(); - String subClusterIdText = subClusterId.getId(); - - // Prepare WebAppAddress - String webAppAddress = subcluster.getRMWebServiceAddress(); - String herfWebAppAddress = ""; - if (webAppAddress != null && !webAppAddress.isEmpty()) { - herfWebAppAddress = - WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAppAddress; - } - - // Prepare Capability - String capability = subcluster.getCapability(); - ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability); - - // Prepare LastStartTime & LastHeartBeat - Date lastStartTime = new Date(subcluster.getLastStartTime()); - Date lastHeartBeat = new Date(subcluster.getLastHeartBeat()); - - // Prepare Resource - long totalMB = subClusterInfo.getTotalMB(); - String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB); - long totalVirtualCores = subClusterInfo.getTotalVirtualCores(); - String resources = String.format("", totalMBDesc, totalVirtualCores); - - // Prepare Node - long totalNodes = subClusterInfo.getTotalNodes(); - long activeNodes = subClusterInfo.getActiveNodes(); - String nodes = String.format("", totalNodes, activeNodes); - - // Prepare HTML Table - String stateStyle = "color:#dc3545;font-weight:bolder"; - SubClusterState state = subcluster.getState(); - if (SubClusterState.SC_RUNNING == state) { - stateStyle = "color:#28a745;font-weight:bolder"; - } - - tbody.tr().$id(subClusterIdText) - .td().$class("details-control").a(herfWebAppAddress, subClusterIdText).__() - .td().$style(stateStyle).__(state.name()).__() - .td().__(lastStartTime).__() - .td().__(lastHeartBeat).__() - .td(resources) - .td(nodes) - .__(); - - // Formatted memory information - long allocatedMB = subClusterInfo.getAllocatedMB(); - String allocatedMBDesc = StringUtils.byteDesc(allocatedMB * BYTES_IN_MB); - long availableMB = subClusterInfo.getAvailableMB(); - String availableMBDesc = StringUtils.byteDesc(availableMB * BYTES_IN_MB); - long pendingMB = subClusterInfo.getPendingMB(); - String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB); - long reservedMB = subClusterInfo.getReservedMB(); - String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB); - - subclusterMap.put("totalmemory", totalMBDesc); - subclusterMap.put("allocatedmemory", allocatedMBDesc); - subclusterMap.put("availablememory", availableMBDesc); - subclusterMap.put("pendingmemory", pendingMBDesc); - subclusterMap.put("reservedmemory", reservedMBDesc); - subclusterMap.put("subcluster", subClusterId.getId()); - subclusterMap.put("capability", capability); - lists.add(subclusterMap); + if (isEnabled) { + initSubClusterPage(tbody, lists); + } else { + initLocalClusterPage(tbody, lists); } } catch (Exception e) { LOG.error("Cannot render Router Federation.", e); @@ -210,4 +142,127 @@ private void initHtmlPageFederation(Block html, boolean isEnabled) { tbody.__().__().div().p().$style("color:red") .__("*The application counts are local per subcluster").__().__(); } + + /** + * Initialize the Federation page of the local-cluster. + * + * @param tbody HTML tbody. + * @param lists subCluster page data list. + */ + private void initLocalClusterPage(TBODY> tbody, List> lists) { + Configuration config = this.router.getConfig(); + SubClusterInfo localCluster = getSubClusterInfoByLocalCluster(config); + if (localCluster != null) { + try { + initSubClusterPageItem(tbody, localCluster, lists); + } catch (Exception e) { + LOG.error("init LocalCluster = {} page data error.", localCluster, e); + } + } + } + + /** + * Initialize the Federation page of the sub-cluster. + * + * @param tbody HTML tbody. + * @param lists subCluster page data list. + */ + private void initSubClusterPage(TBODY> tbody, List> lists) { + // Sort the SubClusters + List subClusters = getSubClusterInfoList(); + + // Iterate through the sub-clusters and display data for each sub-cluster. + // If a sub-cluster cannot display data, skip it. + for (SubClusterInfo subCluster : subClusters) { + try { + initSubClusterPageItem(tbody, subCluster, lists); + } catch (Exception e) { + LOG.error("init subCluster = {} page data error.", subCluster, e); + } + } + } + + /** + * We will initialize the specific SubCluster's data within this method. + * + * @param tbody HTML TBody. + * @param subClusterInfo Sub-cluster information. + * @param lists Used to record data that needs to be displayed in JS. + */ + private void initSubClusterPageItem(TBODY> tbody, + SubClusterInfo subClusterInfo, List> lists) { + + Map subClusterMap = new HashMap<>(); + + // Prepare subCluster + SubClusterId subClusterId = subClusterInfo.getSubClusterId(); + String subClusterIdText = subClusterId.getId(); + + // Prepare WebAppAddress + String webAppAddress = subClusterInfo.getRMWebServiceAddress(); + String herfWebAppAddress = ""; + if (webAppAddress != null && !webAppAddress.isEmpty()) { + herfWebAppAddress = + WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAppAddress; + } + + // Prepare Capability + String capability = subClusterInfo.getCapability(); + ClusterMetricsInfo subClusterMetricsInfo = getClusterMetricsInfo(capability); + + if (subClusterMetricsInfo == null) { + return; + } + + // Prepare LastStartTime & LastHeartBeat + Date lastStartTime = new Date(subClusterInfo.getLastStartTime()); + Date lastHeartBeat = new Date(subClusterInfo.getLastHeartBeat()); + + // Prepare Resource + long totalMB = subClusterMetricsInfo.getTotalMB(); + String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB); + long totalVirtualCores = subClusterMetricsInfo.getTotalVirtualCores(); + String resources = String.format("", totalMBDesc, totalVirtualCores); + + // Prepare Node + long totalNodes = subClusterMetricsInfo.getTotalNodes(); + long activeNodes = subClusterMetricsInfo.getActiveNodes(); + String nodes = String.format("", totalNodes, activeNodes); + + // Prepare HTML Table + String stateStyle = "color:#dc3545;font-weight:bolder"; + SubClusterState state = subClusterInfo.getState(); + if (SubClusterState.SC_RUNNING == state) { + stateStyle = "color:#28a745;font-weight:bolder"; + } + + tbody.tr().$id(subClusterIdText) + .td().$class("details-control").a(herfWebAppAddress, subClusterIdText).__() + .td().$style(stateStyle).__(state.name()).__() + .td().__(lastStartTime).__() + .td().__(lastHeartBeat).__() + .td(resources) + .td(nodes) + .__(); + + // Formatted memory information + long allocatedMB = subClusterMetricsInfo.getAllocatedMB(); + String allocatedMBDesc = StringUtils.byteDesc(allocatedMB * BYTES_IN_MB); + long availableMB = subClusterMetricsInfo.getAvailableMB(); + String availableMBDesc = StringUtils.byteDesc(availableMB * BYTES_IN_MB); + long pendingMB = subClusterMetricsInfo.getPendingMB(); + String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB); + long reservedMB = subClusterMetricsInfo.getReservedMB(); + String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB); + + subClusterMap.put("totalmemory", totalMBDesc); + subClusterMap.put("allocatedmemory", allocatedMBDesc); + subClusterMap.put("availablememory", availableMBDesc); + subClusterMap.put("pendingmemory", pendingMBDesc); + subClusterMap.put("reservedmemory", reservedMBDesc); + subClusterMap.put("subcluster", subClusterId.getId()); + subClusterMap.put("capability", capability); + lists.add(subClusterMap); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java index 1a157a10ce04e..f27692b5ad562 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java @@ -19,7 +19,9 @@ import com.google.inject.Inject; import com.sun.jersey.api.client.Client; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; @@ -206,28 +208,53 @@ private void initFederationClusterSchedulersMetrics(Hamlet.DIV div, // If Federation mode is not enabled or there is currently no SubCluster available, // each column in the list should be displayed as N/A - if (!isEnabled || subclusters == null || subclusters.isEmpty()) { - fsMetricsScheduleTr.tr(). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE). - td(UNAVAILABLE) - .__(); - } else { + if (!isEnabled) { + initLocalClusterOverViewTable(fsMetricsScheduleTr); + } else if (subclusters != null && !subclusters.isEmpty()) { initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters); + } else { + showRouterSchedulerMetricsData(UNAVAILABLE, fsMetricsScheduleTr); } fsMetricsScheduleTr.__().__(); } + /** + * We display Scheduler information for local cluster. + * + * @param fsMetricsScheduleTr MetricsScheduleTr. + */ + private void initLocalClusterOverViewTable( + Hamlet.TBODY>> fsMetricsScheduleTr) { + // configuration + Configuration config = this.router.getConfig(); + Client client = RouterWebServiceUtil.createJerseyClient(config); + String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(config); + + // Get the name of the local cluster. + String localClusterName = config.get(YarnConfiguration.RM_CLUSTER_ID, UNAVAILABLE); + SchedulerOverviewInfo schedulerOverviewInfo = + getSchedulerOverviewInfo(webAppAddress, config, client); + if (schedulerOverviewInfo != null) { + RouterSchedulerMetrics rsMetrics = + new RouterSchedulerMetrics(localClusterName, schedulerOverviewInfo); + // Basic information + showRouterSchedulerMetricsData(rsMetrics, fsMetricsScheduleTr); + } else { + showRouterSchedulerMetricsData(localClusterName, fsMetricsScheduleTr); + } + } + + /** + * We display Scheduler information for multiple subClusters. + * + * @param metrics RouterClusterMetrics. + * @param fsMetricsScheduleTr MetricsScheduleTr. + * @param subClusters subCluster list. + */ private void initSubClusterOverViewTable(RouterClusterMetrics metrics, Hamlet.TBODY>> fsMetricsScheduleTr, - Collection subclusters) { + Collection subClusters) { // configuration Configuration config = this.router.getConfig(); @@ -235,30 +262,93 @@ private void initSubClusterOverViewTable(RouterClusterMetrics metrics, Client client = RouterWebServiceUtil.createJerseyClient(config); // Traverse all SubClusters to get cluster information. - for (SubClusterInfo subcluster : subclusters) { + for (SubClusterInfo subcluster : subClusters) { + // We need to make sure subCluster is not null + if (subcluster != null && subcluster.getSubClusterId() != null) { + // Call the RM interface to obtain schedule information + String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) + + subcluster.getRMWebServiceAddress(); + SchedulerOverviewInfo schedulerOverviewInfo = + getSchedulerOverviewInfo(webAppAddress, config, client); - // Call the RM interface to obtain schedule information - String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) + - subcluster.getRMWebServiceAddress(); + // If schedulerOverviewInfo is not null, + // We will display information from rsMetrics, otherwise we will not display information. + if (schedulerOverviewInfo != null) { + RouterSchedulerMetrics rsMetrics = + new RouterSchedulerMetrics(subcluster, metrics, schedulerOverviewInfo); + // Basic information + showRouterSchedulerMetricsData(rsMetrics, fsMetricsScheduleTr); + } + } + } - SchedulerOverviewInfo typeInfo = RouterWebServiceUtil + client.destroy(); + } + + /** + * Get SchedulerOverview information based on webAppAddress. + * + * @param webAppAddress webAppAddress. + * @param config configuration. + * @param client jersey Client. + * @return SchedulerOverviewInfo. + */ + private SchedulerOverviewInfo getSchedulerOverviewInfo(String webAppAddress, + Configuration config, Client client) { + try { + SchedulerOverviewInfo schedulerOverviewInfo = RouterWebServiceUtil .genericForward(webAppAddress, null, SchedulerOverviewInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW, null, null, - config, client); - RouterSchedulerMetrics rsMetrics = new RouterSchedulerMetrics(subcluster, metrics, typeInfo); - - // Basic information - fsMetricsScheduleTr.tr(). - td(rsMetrics.getSubCluster()). - td(rsMetrics.getSchedulerType()). - td(rsMetrics.getSchedulingResourceType()). - td(rsMetrics.getMinimumAllocation()). - td(rsMetrics.getMaximumAllocation()). - td(rsMetrics.getApplicationPriority()). - td(rsMetrics.getSchedulerBusy()). - td(rsMetrics.getRmDispatcherEventQueueSize()). - td(rsMetrics.getSchedulerDispatcherEventQueueSize()). - __(); + config, client); + return schedulerOverviewInfo; + } catch (Exception e) { + LOG.error("get SchedulerOverviewInfo from webAppAddress = {} error.", + webAppAddress, e); + return null; } } + + /** + * Show RouterSchedulerMetricsData. + * + * @param rsMetrics routerSchedulerMetrics. + * @param fsMetricsScheduleTr MetricsScheduleTr. + */ + private void showRouterSchedulerMetricsData(RouterSchedulerMetrics rsMetrics, + Hamlet.TBODY>> fsMetricsScheduleTr) { + // Basic information + fsMetricsScheduleTr.tr(). + td(rsMetrics.getSubCluster()). + td(rsMetrics.getSchedulerType()). + td(rsMetrics.getSchedulingResourceType()). + td(rsMetrics.getMinimumAllocation()). + td(rsMetrics.getMaximumAllocation()). + td(rsMetrics.getApplicationPriority()). + td(rsMetrics.getSchedulerBusy()). + td(rsMetrics.getRmDispatcherEventQueueSize()). + td(rsMetrics.getSchedulerDispatcherEventQueueSize()). + __(); + } + + /** + * Show RouterSchedulerMetricsData. + * + * @param subClusterId subClusterId. + * @param fsMetricsScheduleTr MetricsScheduleTr. + */ + private void showRouterSchedulerMetricsData(String subClusterId, + Hamlet.TBODY>> fsMetricsScheduleTr) { + String subCluster = StringUtils.isNotBlank(subClusterId) ? subClusterId : UNAVAILABLE; + fsMetricsScheduleTr.tr(). + td(subCluster). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE) + .__(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java index 2266370ee95f6..aa04096688fc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java @@ -40,11 +40,14 @@ public NavBlock(Router router, ViewContext ctx) { @Override public void render(Block html) { + + String federationText = isYarnFederationEnabled() ? "Federation" : "LocalCluster"; + Hamlet.UL> mainList = html.div("#nav"). h3("Cluster"). ul(). li().a(url(""), "About").__(). - li().a(url("federation"), "Federation").__(); + li().a(url("federation"), federationText).__(); List subClusterIds = getActiveSubClusterIds(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java index 4b77164bec7f9..37d1d3c95a6dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java @@ -93,14 +93,34 @@ private NodeLabelsInfo getSubClusterNodeLabelsInfo(String subCluster) { return null; } + /** + * We will obtain the NodeLabel information of multiple sub-clusters. + * + * If Federation mode is enabled, get the NodeLabels of multiple sub-clusters, + * otherwise get the NodeLabels of the local cluster. + * + * @param isEnabled Whether to enable Federation mode, + * true, Federation mode; false, Non-Federation mode. + * + * @return NodeLabelsInfo. + */ private NodeLabelsInfo getYarnFederationNodeLabelsInfo(boolean isEnabled) { + Configuration config = this.router.getConfig(); + String webAddress; if (isEnabled) { - String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig()); - return getSubClusterNodeLabelsByWebAddress(webAddress); + webAddress = WebAppUtils.getRouterWebAppURLWithScheme(config); + } else { + webAddress = WebAppUtils.getRMWebAppURLWithScheme(config); } - return null; + return getSubClusterNodeLabelsByWebAddress(webAddress); } + /** + * Get NodeLabels based on WebAddress. + * + * @param webAddress RM WebAddress. + * @return NodeLabelsInfo. + */ private NodeLabelsInfo getSubClusterNodeLabelsByWebAddress(String webAddress) { Configuration conf = this.router.getConfig(); Client client = RouterWebServiceUtil.createJerseyClient(conf); @@ -112,6 +132,12 @@ private NodeLabelsInfo getSubClusterNodeLabelsByWebAddress(String webAddress) { return nodes; } + /** + * Initialize the Router page based on NodeLabels. + * + * @param nodeLabelsInfo NodeLabelsInfo. + * @param html html Block. + */ private void initYarnFederationNodeLabelsOfCluster(NodeLabelsInfo nodeLabelsInfo, Block html) { Hamlet.TBODY> tbody = html.table("#nodelabels"). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index 8d0fa5322b916..f5c0f9f9d1dbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -65,7 +65,7 @@ protected void render(Block html) { // We will try to get the subClusterName. // If the subClusterName is not empty, // it means that we need to get the Node list of a subCluster. - NodesInfo nodesInfo = null; + NodesInfo nodesInfo; if (subClusterName != null && !subClusterName.isEmpty()) { initSubClusterMetricsOverviewTable(html, subClusterName); nodesInfo = getSubClusterNodesInfo(subClusterName); @@ -80,11 +80,14 @@ protected void render(Block html) { } private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) { + Configuration config = this.router.getConfig(); + String webAddress; if (isEnabled) { - String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig()); - return getSubClusterNodesInfoByWebAddress(webAddress); + webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig()); + } else { + webAddress = WebAppUtils.getRMWebAppURLWithScheme(config); } - return null; + return getSubClusterNodesInfoByWebAddress(webAddress); } private NodesInfo getSubClusterNodesInfo(String subCluster) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java index 7fd8d246a337d..c811c4d48c685 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java @@ -18,21 +18,29 @@ package org.apache.hadoop.yarn.server.router.webapp; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import java.io.StringWriter; import java.util.List; import java.util.ArrayList; import java.util.Map; @@ -62,36 +70,58 @@ public RouterBlock(Router router, ViewContext ctx) { */ protected ClusterMetricsInfo getRouterClusterMetricsInfo() { boolean isEnabled = isYarnFederationEnabled(); + String webAppAddress; if(isEnabled) { - String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); - Client client = RouterWebServiceUtil.createJerseyClient(conf); - ClusterMetricsInfo metrics = RouterWebServiceUtil - .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, - conf, client); - client.destroy(); - return metrics; + webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); + } else { + webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf); } - return null; + return getClusterMetricsInfo(webAppAddress); + } + + /** + * Get RouterClusterMetrics Info. + * + * @param webAppAddress webAppAddress. + * @return ClusterMetricsInfo. + */ + protected ClusterMetricsInfo getClusterMetricsInfo(String webAppAddress) { + // If webAppAddress is empty, we will return NULL. + if (StringUtils.isBlank(webAppAddress)) { + return null; + } + + // We will get ClusterMetricsInfo By webAppAddress. + Client client = RouterWebServiceUtil.createJerseyClient(conf); + ClusterMetricsInfo metrics = RouterWebServiceUtil + .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, + conf, client); + client.destroy(); + return metrics; } /** * Get a list of subclusters. * * @return subcluster List. - * @throws YarnException if the call to the getSubClusters is unsuccessful. */ - protected List getSubClusterInfoList() throws YarnException { - - Map subClustersInfo = facade.getSubClusters(true); + protected List getSubClusterInfoList() { + List subClusters = new ArrayList<>(); + try { + Map subClustersInfo = facade.getSubClusters(true); - // Sort the SubClusters. - List subclusters = new ArrayList<>(); - subclusters.addAll(subClustersInfo.values()); - Comparator cmp = Comparator.comparing(o -> o.getSubClusterId()); - Collections.sort(subclusters, cmp); + // Sort the SubClusters. + subClusters.addAll(subClustersInfo.values()); + Comparator cmp = Comparator.comparing(o -> o.getSubClusterId()); + Collections.sort(subClusters, cmp); - return subclusters; + // Return results + return subClusters; + } catch (YarnException e) { + LOG.error("getSubClusterInfoList error.", e); + return subClusters; + } } /** @@ -163,6 +193,12 @@ protected ClusterMetricsInfo getClusterMetricsInfoBySubClusterId(String subclust return null; } + /** + * Get SubClusterInfo based on subclusterId. + * + * @param subclusterId subCluster Id + * @return SubClusterInfo Collection + */ protected Collection getSubClusterInfoList(String subclusterId) { try { SubClusterId subClusterId = SubClusterId.newInstance(subclusterId); @@ -257,4 +293,63 @@ protected void initNodeLabelsMenu(Hamlet.UL> mainList, mainList.li().a(url("nodelabels"), "Node Labels").__(); } } + + /** + * Generate SubClusterInfo based on local cluster information. + * + * @param config Configuration. + * @return SubClusterInfo. + */ + protected SubClusterInfo getSubClusterInfoByLocalCluster(Configuration config) { + + Client client = null; + try { + + // Step1. Retrieve the name of the local cluster and ClusterMetricsInfo. + String localClusterName = config.get(YarnConfiguration.RM_CLUSTER_ID, UNAVAILABLE); + String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(config); + String rmWebAppURLWithoutScheme = WebAppUtils.getRMWebAppURLWithoutScheme(config); + client = RouterWebServiceUtil.createJerseyClient(config); + ClusterMetricsInfo clusterMetricsInfos = RouterWebServiceUtil + .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, + config, client); + + if (clusterMetricsInfos == null) { + return null; + } + + // Step2. Retrieve cluster information for the local cluster to obtain its startup time. + ClusterInfo clusterInfo = RouterWebServiceUtil.genericForward(webAppAddress, null, + ClusterInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.INFO, + null, null, config, client); + + if (clusterInfo == null) { + return null; + } + + // Step3. Get Local-Cluster Capability + JSONJAXBContext jc = new JSONJAXBContext( + JSONConfiguration.mapped().rootUnwrapping(false).build(), ClusterMetricsInfo.class); + JSONMarshaller marshaller = jc.createJSONMarshaller(); + StringWriter writer = new StringWriter(); + marshaller.marshallToJSON(clusterMetricsInfos, writer); + String capability = writer.toString(); + + // Step4. Generate SubClusterInfo. + SubClusterId subClusterId = SubClusterId.newInstance(localClusterName); + SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId, + rmWebAppURLWithoutScheme, SubClusterState.SC_RUNNING, clusterInfo.getStartedOn(), + Time.now(), capability); + + return subClusterInfo; + } catch (Exception e) { + LOG.error("An error occurred while parsing the local YARN cluster.", e); + } finally { + if (client != null) { + client.destroy(); + } + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java index 4a3af1ba43ddc..714642cec832d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java @@ -51,9 +51,20 @@ public RouterSchedulerMetrics() { public RouterSchedulerMetrics(SubClusterInfo subClusterInfo, RouterClusterMetrics metrics, SchedulerOverviewInfo overview) { + if (subClusterInfo != null) { + initRouterSchedulerMetrics(subClusterInfo.getSubClusterId().getId(), overview); + } + } + + public RouterSchedulerMetrics(String localClusterName, SchedulerOverviewInfo overview) { + initRouterSchedulerMetrics(localClusterName, overview); + } + + private void initRouterSchedulerMetrics(String subClusterName, + SchedulerOverviewInfo overview) { try { // Parse Scheduler Information. - this.subCluster = subClusterInfo.getSubClusterId().getId(); + this.subCluster = subClusterName; this.schedulerType = overview.getSchedulerType(); this.schedulingResourceType = overview.getSchedulingResourceType(); this.minimumAllocation = overview.getMinimumAllocation().toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java index f1501fe1e7a24..f703dab955201 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java @@ -116,6 +116,7 @@ public void testFederationAppViewNotEnable() @Test public void testNodeLabelAppViewNotEnable() throws InterruptedException, YarnException, IOException { + LOG.info("testNodeLabelAppViewNotEnable - NotEnable Federation."); // Test Federation Not Enabled Configuration config = new YarnConfiguration(); config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false); @@ -125,6 +126,7 @@ public void testNodeLabelAppViewNotEnable() @Test public void testNodeLabelAppViewEnable() throws InterruptedException, YarnException, IOException { + LOG.info("testNodeLabelAppViewEnable - Enable Federation."); // Test Federation Not Enabled Configuration config = new YarnConfiguration(); config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml index 55461680b5b84..4a28627a9a194 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml @@ -43,4 +43,8 @@ yarn.federation.policy-manager-params {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} + + yarn.resourcemanager.cluster-id + local-cluster + From 0cfffb30121f4fb5f396d6bf4547f60eb75daa3b Mon Sep 17 00:00:00 2001 From: xiaojunxiang <65019264+xiaojunxiang2023@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:26:03 +0800 Subject: [PATCH 066/155] HDFS-17214. RBF: The Quota class' andByStorageType method res has an incorrect initial value. (#6135) Co-authored-by: xiaojunxiang --- .../hdfs/server/federation/router/Quota.java | 2 +- .../federation/router/TestRouterQuota.java | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java index 825118fed3c3c..8e984d65c33e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java @@ -347,7 +347,7 @@ public static boolean orByStorageType(Predicate predicate) { * @return true if bitwise AND by all storage type returns true, false otherwise. */ public static boolean andByStorageType(Predicate predicate) { - boolean res = false; + boolean res = true; for (StorageType type : StorageType.values()) { res &= predicate.test(type); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index aa3d547056134..3c5be3cf98de6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -31,6 +31,8 @@ import java.util.EnumSet; import java.util.List; import java.util.UUID; +import java.util.Arrays; +import java.util.function.Predicate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; @@ -1168,6 +1170,34 @@ public void testQuotaUpdateWhenDestinationNotPresent() throws Exception { assertEquals(0, quotaUsage.getSpaceConsumed()); } + @Test + public void testAndByStorageType() { + long[] typeQuota = new long[StorageType.values().length]; + Arrays.fill(typeQuota, HdfsConstants.QUOTA_DONT_SET); + + Predicate predicate = new Predicate() { + @Override + public boolean test(StorageType storageType) { + return typeQuota[storageType.ordinal()] == HdfsConstants.QUOTA_DONT_SET; + } + }; + + assertTrue(Quota.andByStorageType(predicate)); + + // This is a value to test for, + // as long as it is not equal to HdfsConstants.QUOTA_DONT_SET + typeQuota[0] = HdfsConstants.QUOTA_RESET; + assertFalse(Quota.andByStorageType(predicate)); + + Arrays.fill(typeQuota, HdfsConstants.QUOTA_DONT_SET); + typeQuota[1] = HdfsConstants.QUOTA_RESET; + assertFalse(Quota.andByStorageType(predicate)); + + Arrays.fill(typeQuota, HdfsConstants.QUOTA_DONT_SET); + typeQuota[typeQuota.length-1] = HdfsConstants.QUOTA_RESET; + assertFalse(Quota.andByStorageType(predicate)); + } + /** * Add three mount tables. * /dir-1 --> ns0---/dir-1 [nsQuota, ssQuota] From 5edd21bc85ed159417f4366dd6a3813e9d1beb02 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Thu, 5 Oct 2023 02:22:59 +0800 Subject: [PATCH 067/155] HDFS-17194. Enhance the log message for striped block recovery (#6094) --- .../server/datanode/BlockRecoveryWorker.java | 112 ++++++++---------- 1 file changed, 51 insertions(+), 61 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java index 249888c4642ae..b24849fb38c8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java @@ -142,33 +142,27 @@ protected void recover() throws IOException { ReplicaState.RWR.getValue()) { syncList.add(new BlockRecord(id, proxyDN, info)); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Block recovery: Ignored replica with invalid " + - "original state: " + info + " from DataNode: " + id); - } + LOG.debug("Block recovery: Ignored replica with invalid " + + "original state: {} from DataNode: {}", info, id); } } else { - if (LOG.isDebugEnabled()) { - if (info == null) { - LOG.debug("Block recovery: DataNode: " + id + " does not have " - + "replica for block: " + block); - } else { - LOG.debug("Block recovery: Ignored replica with invalid " - + "generation stamp or length: " + info + " from " + - "DataNode: " + id); - } + if (info == null) { + LOG.debug("Block recovery: DataNode: {} does not have " + + "replica for block: {}", id, block); + } else { + LOG.debug("Block recovery: Ignored replica with invalid " + + "generation stamp or length: {} from DataNode: {}", info, id); } } } catch (RecoveryInProgressException ripE) { InterDatanodeProtocol.LOG.warn( - "Recovery for replica " + block + " on data-node " + id - + " is already in progress. Recovery id = " - + rBlock.getNewGenerationStamp() + " is aborted.", ripE); + "Recovery for replica {} on data-node {} is already in progress. " + + "Recovery id = {} is aborted.", block, id, rBlock.getNewGenerationStamp(), ripE); return; } catch (IOException e) { ++errorCount; - InterDatanodeProtocol.LOG.warn("Failed to recover block (block=" - + block + ", datanode=" + id + ")", e); + InterDatanodeProtocol.LOG.warn("Failed to recover block (block={}, datanode={})", + block, id, e); } } @@ -206,11 +200,9 @@ void syncBlock(List syncList) throws IOException { // or their replicas have 0 length. // The block can be deleted. if (syncList.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("syncBlock for block " + block + ", all datanodes don't " + - "have the block or their replicas have 0 length. The block can " + - "be deleted."); - } + LOG.debug("syncBlock for block {}, all datanodes don't " + + "have the block or their replicas have 0 length. The block can " + + "be deleted.", block); nn.commitBlockSynchronization(block, recoveryId, 0, true, true, DatanodeID.EMPTY_ARRAY, null); return; @@ -249,12 +241,9 @@ void syncBlock(List syncList) throws IOException { r.rInfo.getNumBytes() == finalizedLength) { participatingList.add(r); } - if (LOG.isDebugEnabled()) { - LOG.debug("syncBlock replicaInfo: block=" + block + - ", from datanode " + r.id + ", receivedState=" + rState.name() + - ", receivedLength=" + r.rInfo.getNumBytes() + - ", bestState=FINALIZED, finalizedLength=" + finalizedLength); - } + LOG.debug("syncBlock replicaInfo: block={}, from datanode {}, receivedState={}, " + + "receivedLength={}, bestState=FINALIZED, finalizedLength={}", + block, r.id, rState.name(), r.rInfo.getNumBytes(), finalizedLength); } newBlock.setNumBytes(finalizedLength); break; @@ -267,12 +256,9 @@ void syncBlock(List syncList) throws IOException { minLength = Math.min(minLength, r.rInfo.getNumBytes()); participatingList.add(r); } - if (LOG.isDebugEnabled()) { - LOG.debug("syncBlock replicaInfo: block=" + block + - ", from datanode " + r.id + ", receivedState=" + rState.name() + - ", receivedLength=" + r.rInfo.getNumBytes() + ", bestState=" + - bestState.name()); - } + LOG.debug("syncBlock replicaInfo: block={}, from datanode {}, receivedState={}, " + + "receivedLength={}, bestState={}", block, r.id, rState.name(), + r.rInfo.getNumBytes(), bestState.name()); } // recover() guarantees syncList will have at least one replica with RWR // or better state. @@ -325,11 +311,8 @@ void syncBlock(List syncList) throws IOException { storages[i] = r.storageID; } - if (LOG.isDebugEnabled()) { - LOG.debug("Datanode triggering commitBlockSynchronization, block=" + - block + ", newGs=" + newBlock.getGenerationStamp() + - ", newLength=" + newBlock.getNumBytes()); - } + LOG.debug("Datanode triggering commitBlockSynchronization, block={}, newGs={}, " + + "newLength={}", block, newBlock.getGenerationStamp(), newBlock.getNumBytes()); nn.commitBlockSynchronization(block, newBlock.getGenerationStamp(), newBlock.getNumBytes(), true, false, @@ -406,14 +389,15 @@ protected void recover() throws IOException { //check generation stamps for (int i = 0; i < locs.length; i++) { DatanodeID id = locs[i]; + ExtendedBlock internalBlk = null; try { DatanodeID bpReg = getDatanodeID(bpid); + internalBlk = new ExtendedBlock(block); + final long blockId = block.getBlockId() + blockIndices[i]; + internalBlk.setBlockId(blockId); InterDatanodeProtocol proxyDN = bpReg.equals(id) ? datanode : DataNode.createInterDataNodeProtocolProxy(id, conf, dnConf.socketTimeout, dnConf.connectToDnViaHostname); - ExtendedBlock internalBlk = new ExtendedBlock(block); - final long blockId = block.getBlockId() + blockIndices[i]; - internalBlk.setBlockId(blockId); ReplicaRecoveryInfo info = callInitReplicaRecovery(proxyDN, new RecoveringBlock(internalBlk, null, recoveryId)); @@ -427,27 +411,36 @@ protected void recover() throws IOException { // simply choose the one with larger length. // TODO: better usage of redundant replicas syncBlocks.put(blockId, new BlockRecord(id, proxyDN, info)); + } else { + LOG.debug("Block recovery: Ignored replica with invalid " + + "original state: {} from DataNode: {} by block: {}", info, id, block); + } + } else { + if (info == null) { + LOG.debug("Block recovery: DataNode: {} does not have " + + "replica for block: (block={}, internalBlk={})", id, block, internalBlk); + } else { + LOG.debug("Block recovery: Ignored replica with invalid " + + "generation stamp or length: {} from DataNode: {} by block: {}", + info, id, block); } } } catch (RecoveryInProgressException ripE) { InterDatanodeProtocol.LOG.warn( - "Recovery for replica " + block + " on data-node " + id - + " is already in progress. Recovery id = " - + rBlock.getNewGenerationStamp() + " is aborted.", ripE); + "Recovery for replica (block={}, internalBlk={}) on data-node {} is already " + + "in progress. Recovery id = {} is aborted.", block, internalBlk, id, + rBlock.getNewGenerationStamp(), ripE); return; } catch (IOException e) { - InterDatanodeProtocol.LOG.warn("Failed to recover block (block=" - + block + ", datanode=" + id + ")", e); + InterDatanodeProtocol.LOG.warn("Failed to recover block (block={}, internalBlk={}, " + + "datanode={})", block, internalBlk, id, e); } } checkLocations(syncBlocks.size()); final long safeLength = getSafeLength(syncBlocks); - if (LOG.isDebugEnabled()) { - LOG.debug("Recovering block " + block - + ", length=" + block.getNumBytes() + ", safeLength=" + safeLength - + ", syncList=" + syncBlocks); - } + LOG.debug("Recovering block {}, length={}, safeLength={}, syncList={}", block, + block.getNumBytes(), safeLength, syncBlocks); // If some internal blocks reach the safe length, convert them to RUR List rurList = new ArrayList<>(locs.length); @@ -498,8 +491,8 @@ private void truncatePartialBlock(List rurList, r.updateReplicaUnderRecovery(bpid, recoveryId, r.rInfo.getBlockId(), newSize); } catch (IOException e) { - InterDatanodeProtocol.LOG.warn("Failed to updateBlock (newblock=" - + ", datanode=" + r.id + ")", e); + InterDatanodeProtocol.LOG.warn("Failed to updateBlock (block={}, internalBlk={}, " + + "datanode={})", block, r.rInfo, r.id, e); failedList.add(r.id); } } @@ -552,12 +545,9 @@ private static void logRecoverBlock(String who, RecoveringBlock rb) { ExtendedBlock block = rb.getBlock(); DatanodeInfo[] targets = rb.getLocations(); - LOG.info("BlockRecoveryWorker: " + who + " calls recoverBlock(" + block - + ", targets=[" + Joiner.on(", ").join(targets) + "]" - + ", newGenerationStamp=" + rb.getNewGenerationStamp() - + ", newBlock=" + rb.getNewBlock() - + ", isStriped=" + rb.isStriped() - + ")"); + LOG.info("BlockRecoveryWorker: {} calls recoverBlock({}, targets=[{}], newGenerationStamp={}" + + ", newBlock={}, isStriped={})", who, block, Joiner.on(", ").join(targets), + rb.getNewGenerationStamp(), rb.getNewBlock(), rb.isStriped()); } /** From ababe3d9b05b375678bb7ae8f197cd77ec4e19af Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 5 Oct 2023 00:25:03 +0530 Subject: [PATCH 068/155] HADOOP-18875. ABFS: Add sendMs and recvMs information for each AbfsHttpOperation by default. (#6008) Contributed By: Anmol Asrani --- .../azurebfs/services/AbfsHttpOperation.java | 70 ++++++------------- .../services/TestAbfsPerfTracker.java | 4 +- 2 files changed, 24 insertions(+), 50 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 67ac0c31665d6..7f5df6066f1b2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -79,8 +79,6 @@ public class AbfsHttpOperation implements AbfsPerfLoggable { private int expectedBytesToBeSent; private long bytesReceived; - // optional trace enabled metrics - private final boolean isTraceEnabled; private long connectionTimeMs; private long sendRequestTimeMs; private long recvResponseTimeMs; @@ -104,7 +102,6 @@ public static AbfsHttpOperation getAbfsHttpOperationWithFixedResult( protected AbfsHttpOperation(final URL url, final String method, final int httpStatus) { - this.isTraceEnabled = LOG.isTraceEnabled(); this.url = url; this.method = method; this.statusCode = httpStatus; @@ -188,14 +185,12 @@ public String toString() { sb.append(getClientRequestId()); sb.append(",rid="); sb.append(requestId); - if (isTraceEnabled) { - sb.append(",connMs="); - sb.append(connectionTimeMs); - sb.append(",sendMs="); - sb.append(sendRequestTimeMs); - sb.append(",recvMs="); - sb.append(recvResponseTimeMs); - } + sb.append(",connMs="); + sb.append(connectionTimeMs); + sb.append(",sendMs="); + sb.append(sendRequestTimeMs); + sb.append(",recvMs="); + sb.append(recvResponseTimeMs); sb.append(",sent="); sb.append(bytesSent); sb.append(",recv="); @@ -218,18 +213,16 @@ public String getLogString() { .append(" ci=") .append(getClientRequestId()) .append(" ri=") - .append(requestId); - - if (isTraceEnabled) { - sb.append(" ct=") - .append(connectionTimeMs) - .append(" st=") - .append(sendRequestTimeMs) - .append(" rt=") - .append(recvResponseTimeMs); - } + .append(requestId) - sb.append(" bs=") + .append(" ct=") + .append(connectionTimeMs) + .append(" st=") + .append(sendRequestTimeMs) + .append(" rt=") + .append(recvResponseTimeMs) + + .append(" bs=") .append(bytesSent) .append(" br=") .append(bytesReceived) @@ -271,7 +264,6 @@ public String getMaskedEncodedUrl() { */ public AbfsHttpOperation(final URL url, final String method, final List requestHeaders) throws IOException { - this.isTraceEnabled = LOG.isTraceEnabled(); this.url = url; this.method = method; @@ -319,9 +311,7 @@ public void sendRequest(byte[] buffer, int offset, int length) throws IOExceptio // send the request body long startTime = 0; - if (this.isTraceEnabled) { - startTime = System.nanoTime(); - } + startTime = System.nanoTime(); OutputStream outputStream = null; // Updates the expected bytes to be sent based on length. this.expectedBytesToBeSent = length; @@ -360,9 +350,7 @@ public void sendRequest(byte[] buffer, int offset, int length) throws IOExceptio if (outputStream != null) { outputStream.close(); } - if (this.isTraceEnabled) { - this.sendRequestTimeMs = elapsedTimeMs(startTime); - } + this.sendRequestTimeMs = elapsedTimeMs(startTime); } } @@ -379,15 +367,10 @@ public void processResponse(final byte[] buffer, final int offset, final int len // get the response long startTime = 0; - if (this.isTraceEnabled) { - startTime = System.nanoTime(); - } + startTime = System.nanoTime(); this.statusCode = getConnResponseCode(); - - if (this.isTraceEnabled) { - this.recvResponseTimeMs = elapsedTimeMs(startTime); - } + this.recvResponseTimeMs = elapsedTimeMs(startTime); this.statusDescription = getConnResponseMessage(); @@ -404,15 +387,11 @@ public void processResponse(final byte[] buffer, final int offset, final int len return; } - if (this.isTraceEnabled) { - startTime = System.nanoTime(); - } + startTime = System.nanoTime(); if (statusCode >= HttpURLConnection.HTTP_BAD_REQUEST) { processStorageErrorResponse(); - if (this.isTraceEnabled) { - this.recvResponseTimeMs += elapsedTimeMs(startTime); - } + this.recvResponseTimeMs += elapsedTimeMs(startTime); this.bytesReceived = this.connection.getHeaderFieldLong(HttpHeaderConfigurations.CONTENT_LENGTH, 0); } else { // consume the input stream to release resources @@ -454,9 +433,7 @@ public void processResponse(final byte[] buffer, final int offset, final int len LOG.debug("IO Error: ", ex); throw ex; } finally { - if (this.isTraceEnabled) { - this.recvResponseTimeMs += elapsedTimeMs(startTime); - } + this.recvResponseTimeMs += elapsedTimeMs(startTime); this.bytesReceived = totalBytesRead; } } @@ -472,9 +449,6 @@ public void setRequestProperty(String key, String value) { * @throws IOException if an error occurs. */ private HttpURLConnection openConnection() throws IOException { - if (!isTraceEnabled) { - return (HttpURLConnection) url.openConnection(); - } long start = System.nanoTime(); try { return (HttpURLConnection) url.openConnection(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java index 191d6e77ae09b..ef52f244f7e49 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsPerfTracker.java @@ -113,7 +113,7 @@ public void verifyTrackingForSingletonLatencyRecords() throws Exception { assertThat(latencyDetails).describedAs("AbfsPerfTracker should return non-null record").isNotNull(); assertThat(latencyDetails).describedAs("Latency record should be in the correct format") .containsPattern("h=[^ ]* t=[^ ]* a=bogusFilesystemName c=bogusAccountName cr=oneOperationCaller" - + " ce=oneOperationCallee r=Succeeded l=[0-9]+ s=0 e= ci=[^ ]* ri=[^ ]* bs=0 br=0 m=GET" + + " ce=oneOperationCallee r=Succeeded l=[0-9]+ s=0 e= ci=[^ ]* ri=[^ ]* ct=[^ ]* st=[^ ]* rt=[^ ]* bs=0 br=0 m=GET" + " u=http%3A%2F%2Fwww.microsoft.com%2FbogusFile"); } @@ -154,7 +154,7 @@ public void verifyTrackingForAggregateLatencyRecords() throws Exception { assertThat(latencyDetails).describedAs("Latency record should be in the correct format") .containsPattern("h=[^ ]* t=[^ ]* a=bogusFilesystemName c=bogusAccountName cr=oneOperationCaller" + " ce=oneOperationCallee r=Succeeded l=[0-9]+ ls=[0-9]+ lc=" + TEST_AGGREGATE_COUNT - + " s=0 e= ci=[^ ]* ri=[^ ]* bs=0 br=0 m=GET u=http%3A%2F%2Fwww.microsoft.com%2FbogusFile"); + + " s=0 e= ci=[^ ]* ri=[^ ]* ct=[^ ]* st=[^ ]* rt=[^ ]* bs=0 br=0 m=GET u=http%3A%2F%2Fwww.microsoft.com%2FbogusFile"); } latencyDetails = abfsPerfTracker.getClientLatency(); From f3a27f2b228972aacb9f233597143a6d9f359d60 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:20:40 +0800 Subject: [PATCH 069/155] YARN-11579. Fix 'Physical Mem Used' and 'Physical VCores Used' are not displaying data. (#6123) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../webapp/dao/ClusterMetricsInfo.java | 19 +++++++++++++++++++ .../webapp/TestRMWebServices.java | 2 +- .../router/webapp/RouterWebServiceUtil.java | 12 ++++++++++++ .../webapp/TestRouterWebServiceUtil.java | 10 ++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index e188fa0526894..cbb9b9fc16d05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -54,7 +54,9 @@ public class ClusterMetricsInfo { private int containersPending; private long totalMB; + private long utilizedMB; private long totalVirtualCores; + private long utilizedVirtualCores; private int utilizedMBPercent; private int utilizedVirtualCoresPercent; private int rmSchedulerBusyPercent; @@ -167,6 +169,7 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { .getContainerAssignedPerSecond(); this.rmEventQueueSize = clusterMetrics.getRmEventQueueSize(); this.schedulerEventQueueSize = clusterMetrics.getSchedulerEventQueueSize(); + this.utilizedVirtualCores = clusterMetrics.getUtilizedVirtualCores(); } public int getAppsSubmitted() { @@ -432,4 +435,20 @@ public int getRmEventQueueSize() { public int getSchedulerEventQueueSize() { return schedulerEventQueueSize; } + + public long getUtilizedVirtualCores() { + return utilizedVirtualCores; + } + + public void setUtilizedVirtualCores(long utilizedVirtualCores) { + this.utilizedVirtualCores = utilizedVirtualCores; + } + + public long getUtilizedMB() { + return utilizedMB; + } + + public void setUtilizedMB(long utilizedMB) { + this.utilizedMB = utilizedMB; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 8322553a9a5b5..5a4de80df7100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -481,7 +481,7 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 35, clusterinfo.length()); + assertEquals("incorrect number of elements", 37, clusterinfo.length()); verifyClusterMetrics( clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 724fa329f610c..02913a43cbf1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -506,10 +506,14 @@ public static void mergeMetrics(ClusterMetricsInfo metrics, metrics.setTotalMB(metrics.getTotalMB() + metricsResponse.getTotalMB()); + metrics.setUtilizedMB(metrics.getUtilizedMB() + + metricsResponse.getUtilizedMB()); metrics.setTotalVirtualCores(metrics.getTotalVirtualCores() + metricsResponse.getTotalVirtualCores()); metrics.setTotalNodes(metrics.getTotalNodes() + metricsResponse.getTotalNodes()); + metrics.setUtilizedVirtualCores(metrics.getUtilizedVirtualCores() + + metricsResponse.getUtilizedVirtualCores()); metrics.setLostNodes(metrics.getLostNodes() + metricsResponse.getLostNodes()); metrics.setUnhealthyNodes(metrics.getUnhealthyNodes() @@ -524,6 +528,14 @@ public static void mergeMetrics(ClusterMetricsInfo metrics, + metricsResponse.getActiveNodes()); metrics.setShutdownNodes(metrics.getShutdownNodes() + metricsResponse.getShutdownNodes()); + + int utilizedVirtualCoresPercent = metrics.getTotalVirtualCores() <= 0 ? 0 : + (int) (metrics.getUtilizedVirtualCores() * 100 / metrics.getTotalVirtualCores()); + metrics.setUtilizedVirtualCoresPercent(utilizedVirtualCoresPercent); + + int utilizedMBPercent = metrics.getTotalMB() <= 0 ? 0 : + (int) (metrics.getUtilizedMB() * 100 / metrics.getTotalMB()); + metrics.setUtilizedMBPercent(utilizedMBPercent); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java index 9ce44250eb5b4..5e480e7714ac5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java @@ -489,10 +489,16 @@ public void testMergeMetrics() { Assert.assertEquals( metricsResponse.getTotalMB() + metricsClone.getTotalMB(), metrics.getTotalMB()); + Assert.assertEquals( + metricsResponse.getUtilizedMB() + metricsClone.getUtilizedMB(), + metrics.getUtilizedMB()); Assert.assertEquals( metricsResponse.getTotalVirtualCores() + metricsClone.getTotalVirtualCores(), metrics.getTotalVirtualCores()); + Assert.assertEquals( + metricsResponse.getUtilizedVirtualCores() + metricsClone.getUtilizedVirtualCores(), + metrics.getUtilizedVirtualCores()); Assert.assertEquals( metricsResponse.getTotalNodes() + metricsClone.getTotalNodes(), metrics.getTotalNodes()); @@ -544,7 +550,9 @@ private ClusterMetricsInfo createClusterMetricsClone( metricsClone.setContainersPending(metrics.getPendingContainers()); metricsClone.setTotalMB(metrics.getTotalMB()); + metricsClone.setUtilizedMB(metrics.getUtilizedMB()); metricsClone.setTotalVirtualCores(metrics.getTotalVirtualCores()); + metricsClone.setUtilizedVirtualCores(metrics.getUtilizedVirtualCores()); metricsClone.setTotalNodes(metrics.getTotalNodes()); metricsClone.setLostNodes(metrics.getLostNodes()); metricsClone.setUnhealthyNodes(metrics.getUnhealthyNodes()); @@ -580,7 +588,9 @@ private void setUpClusterMetrics(ClusterMetricsInfo metrics, long seed) { metrics.setContainersPending(rand.nextInt(1000)); metrics.setTotalMB(rand.nextInt(1000)); + metrics.setUtilizedMB(metrics.getTotalMB() - rand.nextInt(100)); metrics.setTotalVirtualCores(rand.nextInt(1000)); + metrics.setUtilizedVirtualCores(metrics.getUtilizedVirtualCores() - rand.nextInt(100)); metrics.setTotalNodes(rand.nextInt(1000)); metrics.setLostNodes(rand.nextInt(1000)); metrics.setUnhealthyNodes(rand.nextInt(1000)); From 2bf5a9ed118e5830669db473447a7fd4a4037d97 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 5 Oct 2023 21:28:21 +0100 Subject: [PATCH 070/155] HADOOP-18917. Upgrade to commons-io 2.14.0 (#6133). Contributed by PJ Fanning Signed-off-by: Ayush Saxena --- LICENSE-binary | 2 +- hadoop-project/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 1eecdf7dd11f6..4916fda109239 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -247,7 +247,7 @@ commons-cli:commons-cli:1.5.0 commons-codec:commons-codec:1.11 commons-collections:commons-collections:3.2.2 commons-daemon:commons-daemon:1.0.13 -commons-io:commons-io:2.8.0 +commons-io:commons-io:2.14.0 commons-net:commons-net:3.9.0 de.ruedigermoeller:fst:2.50 io.grpc:grpc-api:1.26.0 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 4322a51cd2d47..3fe8f8fa8f4d0 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -118,7 +118,7 @@ 3.2.2 1.24.0 1.9.0 - 2.11.0 + 2.14.0 3.12.0 1.1.3 3.6.1 From 57100bba1bfd6963294181a2521396dc30c295f7 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Fri, 6 Oct 2023 05:10:32 +0100 Subject: [PATCH 071/155] HADOOP-18917. Addendum: Upgrade to commons-io 2.14.0 (#6152). Contributed by PJ Fanning Co-authored-by: Ayush Saxena --- .../azurebfs/services/TestTextFileBasedIdentityHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestTextFileBasedIdentityHandler.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestTextFileBasedIdentityHandler.java index f9950faf944df..b0a72b2131cdd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestTextFileBasedIdentityHandler.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestTextFileBasedIdentityHandler.java @@ -19,9 +19,9 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.NoSuchFileException; import org.junit.Assert; import org.junit.BeforeClass; @@ -114,7 +114,7 @@ public void testLookupForUser() throws IOException { public void testLookupForUserFileNotFound() throws Exception { TextFileBasedIdentityHandler handler = new TextFileBasedIdentityHandler(userMappingFile.getPath() + ".test", groupMappingFile.getPath()); - intercept(FileNotFoundException.class, "FileNotFoundException", + intercept(NoSuchFileException.class, "NoSuchFileException", () -> handler.lookupForLocalUserIdentity(testUserDataLine3.split(":")[0])); } @@ -143,7 +143,7 @@ public void testLookupForGroup() throws IOException { public void testLookupForGroupFileNotFound() throws Exception { TextFileBasedIdentityHandler handler = new TextFileBasedIdentityHandler(userMappingFile.getPath(), groupMappingFile.getPath() + ".test"); - intercept(FileNotFoundException.class, "FileNotFoundException", + intercept(NoSuchFileException.class, "NoSuchFileException", () -> handler.lookupForLocalGroupIdentity(testGroupDataLine2.split(":")[0])); } } From 4c408a557f149dfb91ae4ea187d18638cd12dd77 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Fri, 6 Oct 2023 15:09:23 +0800 Subject: [PATCH 072/155] HDFS-17205. HdfsServerConstants.MIN_BLOCKS_FOR_WRITE should be configurable (#6112). Contributed by Haiyang Hu Reviewed-by: He Xiaoqiao Signed-off-by: Ayush Saxena --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +++ .../server/blockmanagement/BlockManager.java | 12 +++++ .../blockmanagement/BlockPlacementPolicy.java | 10 ++++ .../BlockPlacementPolicyDefault.java | 20 +++++++- .../blockmanagement/DatanodeDescriptor.java | 7 ++- .../server/common/HdfsServerConstants.java | 2 + .../hadoop/hdfs/server/namenode/NameNode.java | 19 +++++++- .../src/main/resources/hdfs-default.xml | 9 ++++ .../TestReplicationPolicy.java | 23 +++++++++ .../namenode/TestNameNodeReconfigure.java | 47 +++++++++++++++++++ .../hadoop/hdfs/tools/TestDFSAdmin.java | 12 +++-- 11 files changed, 155 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f0450b0778dad..8a3b9b9bda476 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.ReservedSpaceCalculator; import org.apache.hadoop.hdfs.web.URLConnectionFactory; @@ -1270,6 +1271,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT = false; + public static final String DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY = + "dfs.namenode.block-placement.min-blocks-for.write"; + public static final int DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_DEFAULT = + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE; + public static final String DFS_NAMENODE_GC_TIME_MONITOR_ENABLE = "dfs.namenode.gc.time.monitor.enable"; public static final boolean DFS_NAMENODE_GC_TIME_MONITOR_ENABLE_DEFAULT = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 54b89c813010d..783000bbef2f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -5689,4 +5689,16 @@ public void setExcludeSlowNodesEnabled(boolean enable) { public boolean getExcludeSlowNodesEnabled(BlockType blockType) { return placementPolicies.getPolicy(blockType).getExcludeSlowNodesEnabled(); } + + public void setMinBlocksForWrite(int minBlocksForWrite) { + ensurePositiveInt(minBlocksForWrite, + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY); + placementPolicies.getPolicy(CONTIGUOUS).setMinBlocksForWrite(minBlocksForWrite); + placementPolicies.getPolicy(STRIPED).setMinBlocksForWrite(minBlocksForWrite); + } + + @VisibleForTesting + public int getMinBlocksForWrite(BlockType blockType) { + return placementPolicies.getPolicy(blockType).getMinBlocksForWrite(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 85468a57bde47..a37202630ac68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -274,4 +274,14 @@ public void splitNodesWithRack( public abstract void setExcludeSlowNodesEnabled(boolean enable); public abstract boolean getExcludeSlowNodesEnabled(); + + /** + * Updates the value used for minBlocksForWrite, which is set by + * {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY}. + * + * @param minBlocksForWrite the minimum number of blocks required for write operations. + */ + public abstract void setMinBlocksForWrite(int minBlocksForWrite); + + public abstract int getMinBlocksForWrite(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 60a2a8676adab..8020d7c45b37a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -19,6 +19,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOADBYSTORAGETYPE_KEY; import static org.apache.hadoop.util.Time.monotonicNow; @@ -111,7 +113,8 @@ private String getText() { private FSClusterStats stats; protected long heartbeatInterval; // interval for DataNode heartbeats private long staleInterval; // interval used to identify stale DataNodes - + private volatile int minBlocksForWrite; // minimum number of blocks required for write operations. + /** * A miss of that many heartbeats is tolerated for replica deletion policy. */ @@ -161,6 +164,9 @@ public void initialize(Configuration conf, FSClusterStats stats, this.excludeSlowNodesEnabled = conf.getBoolean( DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT); + this.minBlocksForWrite = conf.getInt( + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY, + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_DEFAULT); } @Override @@ -959,7 +965,7 @@ DatanodeStorageInfo chooseStorage4Block(DatanodeDescriptor dnd, List results, StorageType storageType) { DatanodeStorageInfo storage = - dnd.chooseStorage4Block(storageType, blockSize); + dnd.chooseStorage4Block(storageType, blockSize, minBlocksForWrite); if (storage != null) { results.add(storage); } else { @@ -1386,4 +1392,14 @@ public void setExcludeSlowNodesEnabled(boolean enable) { public boolean getExcludeSlowNodesEnabled() { return excludeSlowNodesEnabled; } + + @Override + public void setMinBlocksForWrite(int minBlocksForWrite) { + this.minBlocksForWrite = minBlocksForWrite; + } + + @Override + public int getMinBlocksForWrite() { + return minBlocksForWrite; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 352238b7f70d1..a11fa1bac2598 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -812,11 +811,11 @@ public boolean containsInvalidateBlock(Block block) { * * @param t requested storage type * @param blockSize requested block size + * @param minBlocksForWrite requested the minimum number of blocks */ public DatanodeStorageInfo chooseStorage4Block(StorageType t, - long blockSize) { - final long requiredSize = - blockSize * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE; + long blockSize, int minBlocksForWrite) { + final long requiredSize = blockSize * minBlocksForWrite; final long scheduledSize = blockSize * getBlocksScheduled(t); long remaining = 0; DatanodeStorageInfo storage = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index ce788e78703cb..3a00c8a790cee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -42,6 +42,8 @@ @InterfaceAudience.Private public interface HdfsServerConstants { + // Will be set by + // {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY}. int MIN_BLOCKS_FOR_WRITE = 1; long LEASE_RECOVER_PERIOD = 10 * 1000; // in ms diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 388f06f0e33ec..a4618c000778a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -135,6 +135,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT; @@ -362,7 +364,8 @@ public enum OperationCategory { DFS_DATANODE_MAX_NODES_TO_REPORT_KEY, DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, - DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK)); + DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" @@ -2362,6 +2365,8 @@ protected String reconfigurePropertyImpl(String property, String newVal) (property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK))) { return reconfigureDecommissionBackoffMonitorParameters(datanodeManager, property, newVal); + } else if (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY)) { + return reconfigureMinBlocksForWrite(property, newVal); } else { throw new ReconfigurationException(property, newVal, getConf().get( property)); @@ -2671,6 +2676,18 @@ private String reconfigureDecommissionBackoffMonitorParameters( } } + private String reconfigureMinBlocksForWrite(String property, String newValue) + throws ReconfigurationException { + try { + int newSetting = adjustNewVal( + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_DEFAULT, newValue); + namesystem.getBlockManager().setMinBlocksForWrite(newSetting); + return String.valueOf(newSetting); + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newValue, getConf().get(property), e); + } + } + @Override // ReconfigurableBase protected Configuration getNewConf() { return new HdfsConfiguration(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index c8f5993adbbb7..1135b98c6bacb 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2445,6 +2445,15 @@ + + dfs.namenode.block-placement.min-blocks-for.write + 1 + + Setting the minimum number of blocks for write operations is used to calculate the space required + for write operations. + + + dfs.namenode.max.slowpeer.collect.nodes 5 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 20163cc5fa531..b99e060ee387c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1734,4 +1734,27 @@ public void testReduceChooseTimesIfNOStaleNode() { assertFalse(dnManager.shouldAvoidStaleDataNodesForWrite()); resetHeartbeatForStorages(); } + + @Test + public void testChosenFailureForNotEnoughStorageSpace() { + final LogVerificationAppender appender = new LogVerificationAppender(); + final Logger logger = Logger.getRootLogger(); + logger.addAppender(appender); + + // Set all datanode storage remaining space is 1 * BLOCK_SIZE. + for(int i = 0; i < dataNodes.length; i++) { + updateHeartbeatWithUsage(dataNodes[i], BLOCK_SIZE, 0L, BLOCK_SIZE, + 0L, 0L, 0L, 0, 0); + } + + // Set chooseStorage4Block required the minimum number of blocks is 2. + replicator.setMinBlocksForWrite(2); + DatanodeStorageInfo[] targets = chooseTarget(1, dataNodes[1], + new ArrayList(), null); + assertEquals(0, targets.length); + assertNotEquals(0, + appender.countLinesWithMessage("NOT_ENOUGH_STORAGE_SPACE")); + + resetHeartbeatForStorages(); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java index ec7717e503a6b..63d3a45fff81e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminBackoffMonitor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminMonitorInterface; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Test; import org.junit.Before; import org.junit.After; @@ -654,6 +655,52 @@ public void testReconfigureDecommissionBackoffMonitorParameters() } } + @Test + public void testReconfigureMinBlocksForWrite() throws Exception { + final NameNode nameNode = cluster.getNameNode(0); + final BlockManager bm = nameNode.getNamesystem().getBlockManager(); + String key = DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY; + int defaultVal = DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_DEFAULT; + + // Ensure we cannot set any of the parameters negative + ReconfigurationException reconfigurationException = + LambdaTestUtils.intercept(ReconfigurationException.class, + () -> nameNode.reconfigurePropertyImpl(key, "-20")); + assertTrue(reconfigurationException.getCause() instanceof IllegalArgumentException); + assertEquals(key + " = '-20' is invalid. It should be a " + +"positive, non-zero integer value.", reconfigurationException.getCause().getMessage()); + + // Ensure none of the values were updated from the defaults + assertEquals(defaultVal, bm.getMinBlocksForWrite(BlockType.CONTIGUOUS)); + assertEquals(defaultVal, bm.getMinBlocksForWrite(BlockType.STRIPED)); + + reconfigurationException = LambdaTestUtils.intercept(ReconfigurationException.class, + () -> nameNode.reconfigurePropertyImpl(key, "0")); + assertTrue(reconfigurationException.getCause() instanceof IllegalArgumentException); + assertEquals(key + " = '0' is invalid. It should be a " + +"positive, non-zero integer value.", reconfigurationException.getCause().getMessage()); + + // Ensure none of the values were updated from the defaults + assertEquals(defaultVal, bm.getMinBlocksForWrite(BlockType.CONTIGUOUS)); + assertEquals(defaultVal, bm.getMinBlocksForWrite(BlockType.STRIPED)); + + + // Ensure none of the parameters can be set to a string value + reconfigurationException = LambdaTestUtils.intercept(ReconfigurationException.class, + () -> nameNode.reconfigurePropertyImpl(key, "str")); + assertTrue(reconfigurationException.getCause() instanceof NumberFormatException); + + // Ensure none of the values were updated from the defaults + assertEquals(defaultVal, bm.getMinBlocksForWrite(BlockType.CONTIGUOUS)); + assertEquals(defaultVal, bm.getMinBlocksForWrite(BlockType.STRIPED)); + + nameNode.reconfigurePropertyImpl(key, "3"); + + // Ensure none of the values were updated from the new value. + assertEquals(3, bm.getMinBlocksForWrite(BlockType.CONTIGUOUS)); + assertEquals(3, bm.getMinBlocksForWrite(BlockType.STRIPED)); + } + @After public void shutDown() throws IOException { if (cluster != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 682e033bf298e..70a8bab8b0905 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -38,6 +38,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY; @@ -441,7 +442,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("namenode", address, outs, errs); - assertEquals(22, outs.size()); + assertEquals(23, outs.size()); assertTrue(outs.get(0).contains("Reconfigurable properties:")); assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1)); assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2)); @@ -452,10 +453,11 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(7)); assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(8)); assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(9)); - assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, outs.get(10)); - assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, outs.get(11)); - assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(12)); - assertEquals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, outs.get(13)); + assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY, outs.get(10)); + assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, outs.get(11)); + assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, outs.get(12)); + assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(13)); + assertEquals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, outs.get(14)); assertEquals(errs.size(), 0); } From daa78adc888704e5688b84b404573ed1e28012db Mon Sep 17 00:00:00 2001 From: huangzhaobo Date: Fri, 6 Oct 2023 15:10:44 +0800 Subject: [PATCH 073/155] HDFS-17200. Add some datanode related metrics to Metrics.md. (#6099). Contributed by huangzhaobo99 Signed-off-by: Ayush Saxena --- .../hadoop-common/src/site/markdown/Metrics.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 01d89b81356e4..45c323aa9d7e0 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -508,8 +508,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i | `PacketsSlowWriteToMirror` | Total number of packets whose write to other Datanodes in the pipeline takes more than a certain time (300ms by default) | | `PacketsSlowWriteToDisk` | Total number of packets whose write to disk takes more than a certain time (300ms by default) | | `PacketsSlowWriteToOsCache` | Total number of packets whose write to os cache takes more than a certain time (300ms by default) | -| `slowFlushOrSyncCount` | Total number of packets whose sync/flush takes more than a certain time (300ms by default) | -| `slowAckToUpstreamCount` | Total number of packets whose upstream ack takes more than a certain time (300ms by default) | +| `SlowFlushOrSyncCount` | Total number of packets whose sync/flush takes more than a certain time (300ms by default) | +| `SlowAckToUpstreamCount` | Total number of packets whose upstream ack takes more than a certain time (300ms by default) | +| `SumOfActorCommandQueueLength` | Sum of all BPServiceActors command queue length | +| `NumProcessedCommands` | Num of processed commands of all BPServiceActors | +| `ProcessedCommandsOpNumOps` | Total number of processed commands operations | +| `ProcessedCommandsOpAvgTime` | Average time of processed commands operations in milliseconds | FsVolume -------- From ee1ebbe5f99eb034a330c41e9da8118fb1ab0eb8 Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Sat, 7 Oct 2023 18:20:38 +0100 Subject: [PATCH 074/155] HADOOP-18923. Switch to SPDX identifier for license name (#6149). Contributed by Colm O hEigeartaigh. Signed-off-by: Ayush Saxena --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 13f738d0f7838..b86af01172f42 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x - Apache License, Version 2.0 + Apache-2.0 https://www.apache.org/licenses/LICENSE-2.0.txt From 42b32fbbdc93259480d3eb7cea5ce22a5193fa68 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sun, 8 Oct 2023 08:20:11 +0800 Subject: [PATCH 075/155] YARN-11583. Improve Node Link for YARN Federation Web Page. (#6145) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../router/webapp/MetricsOverviewTable.java | 20 ++++++++---- .../server/router/webapp/NodeLabelsBlock.java | 9 ++++++ .../server/router/webapp/NodeLabelsPage.java | 11 +++++-- .../yarn/server/router/webapp/NodesBlock.java | 32 ++++++++++++++++--- .../yarn/server/router/webapp/NodesPage.java | 9 ++++-- .../server/router/webapp/RouterBlock.java | 2 ++ .../router/webapp/RouterWebServiceUtil.java | 9 ++++++ .../router/webapp/TestRouterWebServices.java | 2 +- 8 files changed, 78 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java index f27692b5ad562..aeb953915af55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java @@ -160,13 +160,19 @@ private void initFederationClusterNodesMetrics(Hamlet.DIV div, // Initialize table data information tbody().$class("ui-widget-content"). tr(). - td(String.valueOf(metrics.getActiveNodes())). - td(String.valueOf(metrics.getDecommissioningNodes())). - td(String.valueOf(metrics.getDecommissionedNodes())). - td(String.valueOf(metrics.getLostNodes())). - td(String.valueOf(metrics.getUnhealthyNodes())). - td(String.valueOf(metrics.getRebootedNodes())). - td(String.valueOf(metrics.getShutdownNodes())). + td().a(url("nodes"), String.valueOf(metrics.getActiveNodes())).__(). + td().a(url("nodes/router/?node.state=decommissioning"), + String.valueOf(metrics.getDecommissioningNodes())).__(). + td().a(url("nodes/router/?node.state=decommissioned"), + String.valueOf(metrics.getDecommissionedNodes())).__(). + td().a(url("nodes/router/?node.state=lost"), + String.valueOf(metrics.getLostNodes())).__(). + td().a(url("nodes/router/?node.state=unhealthy"), + String.valueOf(metrics.getUnhealthyNodes())).__(). + td().a(url("nodes/router/?node.state=rebooted"), + String.valueOf(metrics.getRebootedNodes())).__(). + td().a(url("nodes/router/?node.state=shutdown"), + String.valueOf(metrics.getShutdownNodes())).__(). __(). __().__(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java index 37d1d3c95a6dd..2ee813934f751 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -158,6 +159,14 @@ private void initYarnFederationNodeLabelsOfCluster(NodeLabelsInfo nodeLabelsInfo String type = (info.getExclusivity()) ? "Exclusive Partition" : "Non Exclusive Partition"; row = row.td(type); int nActiveNMs = info.getActiveNMs(); + if (nActiveNMs > 0) { + row = row.td().a(url("nodes", + "?" + YarnWebParams.NODE_LABEL + "=" + info.getName()), String.valueOf(nActiveNMs)) + .__(); + } else { + row = row.td(String.valueOf(nActiveNMs)); + } + row = row.td(String.valueOf(nActiveNMs)); PartitionInfo partitionInfo = info.getPartitionInfo(); ResourceInfo available = partitionInfo.getResourceAvailable(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java index 9b3cea468172a..5ab462ee0f14c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java @@ -21,7 +21,9 @@ import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; +import static org.apache.hadoop.yarn.server.router.webapp.RouterWebServiceUtil.generateWebTitle; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; /** @@ -33,10 +35,15 @@ public class NodeLabelsPage extends RouterView { protected void preHead(Hamlet.HTML<__> html) { commonPreHead(html); String type = $(NODE_SC); + String nodeLabel = $(NODE_LABEL); String title = "Node labels of the cluster"; - if (type != null && !type.isEmpty()) { - title = title + " (" + type + ")"; + + if (nodeLabel != null && !nodeLabel.isEmpty()) { + title = generateWebTitle(title, nodeLabel); + } else if (type != null && !type.isEmpty()) { + title = generateWebTitle(title, type); } + setTitle(title); set(DATATABLES_ID, "nodelabels"); setTableStyles(html, "nodelabels", ".healthStatus {width:10em}", ".healthReport {width:10em}"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index f5c0f9f9d1dbe..4544ed4ed4466 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; @@ -40,6 +41,8 @@ import java.util.Date; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; /** * Nodes block for the Router Web UI. @@ -61,12 +64,15 @@ protected void render(Block html) { // Get subClusterName String subClusterName = $(NODE_SC); + String state = $(NODE_STATE); + String nodeLabel = $(NODE_LABEL); // We will try to get the subClusterName. // If the subClusterName is not empty, // it means that we need to get the Node list of a subCluster. NodesInfo nodesInfo; - if (subClusterName != null && !subClusterName.isEmpty()) { + if (subClusterName != null && !subClusterName.isEmpty() && + !ROUTER.equalsIgnoreCase(subClusterName)) { initSubClusterMetricsOverviewTable(html, subClusterName); nodesInfo = getSubClusterNodesInfo(subClusterName); } else { @@ -76,7 +82,7 @@ protected void render(Block html) { } // Initialize NodeInfo List - initYarnFederationNodesOfCluster(nodesInfo, html); + initYarnFederationNodesOfCluster(nodesInfo, html, state, nodeLabel); } private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) { @@ -100,7 +106,7 @@ private NodesInfo getSubClusterNodesInfo(String subCluster) { if (subClusterInfo != null) { // Prepare webAddress String webAddress = subClusterInfo.getRMWebServiceAddress(); - String herfWebAppAddress = ""; + String herfWebAppAddress; if (webAddress != null && !webAddress.isEmpty()) { herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAddress; @@ -124,7 +130,8 @@ private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) { return nodes; } - private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) { + private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html, + String filterState, String filterLabel) { TBODY> tbody = html.table("#nodes").thead().tr() .th(".nodelabels", "Node Labels") .th(".rack", "Rack") @@ -143,6 +150,23 @@ private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) { if (nodesInfo != null && CollectionUtils.isNotEmpty(nodesInfo.getNodes())) { for (NodeInfo info : nodesInfo.getNodes()) { + if (filterState != null && !filterState.isEmpty() && !filterState.equals(info.getState())) { + continue; + } + + // Besides state, we need to filter label as well. + if (!filterLabel.equals(RMNodeLabelsManager.ANY)) { + if (filterLabel.isEmpty()) { + // Empty label filter means only shows nodes without label + if (!info.getNodeLabels().isEmpty()) { + continue; + } + } else if (!info.getNodeLabels().contains(filterLabel)) { + // Only nodes have given label can show on web page. + continue; + } + } + int usedMemory = (int) info.getUsedMemory(); int availableMemory = (int) info.getAvailableMemory(); TR>> row = tbody.tr(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java index 0723cff792d96..386e540b7d6cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.router.webapp; +import static org.apache.hadoop.yarn.server.router.webapp.RouterWebServiceUtil.generateWebTitle; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -32,9 +34,12 @@ class NodesPage extends RouterView { protected void preHead(Page.HTML<__> html) { commonPreHead(html); String type = $(NODE_SC); + String state = $(NODE_STATE); String title = "Nodes of the cluster"; - if (type != null && !type.isEmpty()) { - title = title + " (" + type + ")"; + if (state != null && !state.isEmpty()) { + title = generateWebTitle(title, state); + } else if (type != null && !type.isEmpty()) { + title = generateWebTitle(title, type); } setTitle(title); set(DATATABLES_ID, "nodes"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java index c811c4d48c685..55bcb81f259e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java @@ -55,6 +55,8 @@ public abstract class RouterBlock extends HtmlBlock { private final FederationStateStoreFacade facade; private final Configuration conf; + public static final String ROUTER = "router"; + public RouterBlock(Router router, ViewContext ctx) { super(ctx); this.ctx = ctx; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 02913a43cbf1c..f2c385fd02220 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -764,4 +764,13 @@ public static UserGroupInformation getKerberosUserGroupInformation(Configuration // return caller UGI return callerUGI; } + + public static String generateWebTitle(String title, String msg) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(title); + stringBuilder.append(" ("); + stringBuilder.append(msg); + stringBuilder.append(")"); + return stringBuilder.toString(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java index 5bf8db04b06cd..dbcd7db21c9c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java @@ -334,7 +334,7 @@ public RESTRequestInterceptor run() throws Exception { Assert.assertNotNull(client1.interceptor); Assert.assertNotNull(client2.interceptor); - Assert.assertTrue(client1.interceptor == client2.interceptor); + Assert.assertSame(client1.interceptor, client2.interceptor); } } From ea3cb12ec80210fcad985e9cfc37d129afec888b Mon Sep 17 00:00:00 2001 From: hfutatzhanghb Date: Sun, 8 Oct 2023 10:36:09 +0800 Subject: [PATCH 076/155] HDFS-17171. CONGESTION_RATIO should be configurable (#5996) Reviewed-by: Ayush Saxena Signed-off-by: Tao Li --- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++- .../apache/hadoop/hdfs/server/datanode/DataNode.java | 12 ++++++++++-- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 8a3b9b9bda476..6b7e4fdb406db 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1573,6 +1573,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false; public static final String DFS_PIPELINE_SLOWNODE_ENABLED = "dfs.pipeline.slownode"; public static final boolean DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT = false; + public static final String DFS_PIPELINE_CONGESTION_RATIO = "dfs.pipeline.congestion.ratio"; + public static final double DFS_PIPELINE_CONGESTION_RATIO_DEFAULT = 1.5; // Key Provider Cache Expiry public static final String DFS_DATANODE_BLOCK_PINNING_ENABLED = @@ -2042,5 +2044,4 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_LEASE_HARDLIMIT_DEFAULT = HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT; - } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 4ade6ada72d04..4613d37f606ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -462,7 +462,7 @@ public static InetSocketAddress createSocketAddr(String target) { private final Tracer tracer; private static final int NUM_CORES = Runtime.getRuntime() .availableProcessors(); - private static final double CONGESTION_RATIO = 1.5; + private final double congestionRatio; private DiskBalancer diskBalancer; private DataSetLockManager dataSetLockManager; @@ -515,6 +515,10 @@ private static Tracer createTracer(Configuration conf) { volumeChecker = new DatasetVolumeChecker(conf, new Timer()); this.xferService = HadoopExecutors.newCachedThreadPool(new Daemon.DaemonFactory()); + double congestionRationTmp = conf.getDouble(DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO, + DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT); + this.congestionRatio = congestionRationTmp > 0 ? + congestionRationTmp : DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT; } /** @@ -614,6 +618,10 @@ public Map load(String key) { new DataTransferThrottler(100, ecReconstuctReadBandwidth) : null; this.ecReconstuctWriteThrottler = ecReconstuctWriteBandwidth > 0 ? new DataTransferThrottler(100, ecReconstuctWriteBandwidth) : null; + double congestionRationTmp = conf.getDouble(DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO, + DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT); + this.congestionRatio = congestionRationTmp > 0 ? + congestionRationTmp : DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT; } @Override // ReconfigurableBase @@ -1070,7 +1078,7 @@ public PipelineAck.ECN getECN() { } double load = ManagementFactory.getOperatingSystemMXBean() .getSystemLoadAverage(); - return load > NUM_CORES * CONGESTION_RATIO ? PipelineAck.ECN.CONGESTED : + return load > NUM_CORES * congestionRatio ? PipelineAck.ECN.CONGESTED : PipelineAck.ECN.SUPPORTED; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 1135b98c6bacb..678908d901ec9 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5619,6 +5619,14 @@ + + dfs.pipeline.congestion.ratio + 1.5 + + The ratio which is used to compute congestion load. + + + dfs.qjournal.accept-recovery.timeout.ms 120000 From 666af5870001d4d91a8ba3385e47f3fa2f6ca6ef Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 9 Oct 2023 21:21:12 +0530 Subject: [PATCH 077/155] HADOOP-18876. ABFS: Change default for fs.azure.data.blocks.buffer to bytebuffer (#6009) The default value for fs.azure.data.blocks.buffer is changed from "disk" to "bytebuffer" This will speed up writing to azure storage, at the risk of running out of memory -especially if there are many threads writing to abfs at the same time and the upload bandwidth is limited. If jobs do run out of memory writing to abfs, change the option back to "disk" Contributed by Anmol Asrani --- .../hadoop/fs/azurebfs/constants/ConfigurationKeys.java | 2 +- .../fs/azurebfs/constants/FileSystemConfigurations.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 872364a8e6167..461e43a9f7e75 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -85,7 +85,7 @@ public final class ConfigurationKeys { /** * What data block buffer to use. *
    - * Options include: "disk"(Default), "array", and "bytebuffer". + * Options include: "disk", "array", and "bytebuffer"(Default). *
    * Default is {@link FileSystemConfigurations#DATA_BLOCKS_BUFFER_DEFAULT}. * Value: {@value} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 32f9966e30ae9..00fc4a6a3db77 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -132,11 +132,13 @@ public final class FileSystemConfigurations { */ public static final String DATA_BLOCKS_BUFFER_DISK = "disk"; + public static final String DATA_BLOCKS_BYTEBUFFER = "bytebuffer"; + /** * Default buffer option: {@value}. */ public static final String DATA_BLOCKS_BUFFER_DEFAULT = - DATA_BLOCKS_BUFFER_DISK; + DATA_BLOCKS_BYTEBUFFER; /** * IO rate limit. Value: {@value} From 9c621fcea72a988c930ef614a7c22de00d0c7d21 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 9 Oct 2023 22:10:15 +0530 Subject: [PATCH 078/155] HADOOP-18861. ABFS: Fix failing tests for CPK (#5979) Contributed by Anmol Asrani --- .../fs/azurebfs/ITestCustomerProvidedKey.java | 46 ++++++++----------- .../fs/azurebfs/TestAccountConfiguration.java | 1 + 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java index 76b8a77fffcc2..9ae3bf2ee20f4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java @@ -109,10 +109,14 @@ public ITestCustomerProvidedKey() throws Exception { .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); } + private String getFileName() throws IOException { + return path("/" + methodName.getMethodName()).toUri().getPath(); + } + @Test public void testReadWithCPK() throws Exception { final AzureBlobFileSystem fs = getAbfs(true); - String fileName = path("/" + methodName.getMethodName()).toString(); + String fileName = getFileName(); createFileAndGetContent(fs, fileName, FILE_SIZE); AbfsClient abfsClient = fs.getAbfsClient(); @@ -162,7 +166,7 @@ public void testReadWithCPK() throws Exception { @Test public void testReadWithoutCPK() throws Exception { final AzureBlobFileSystem fs = getAbfs(false); - String fileName = path("/" + methodName.getMethodName()).toString(); + String fileName = getFileName(); createFileAndGetContent(fs, fileName, FILE_SIZE); AbfsClient abfsClient = fs.getAbfsClient(); @@ -201,7 +205,7 @@ public void testReadWithoutCPK() throws Exception { @Test public void testAppendWithCPK() throws Exception { final AzureBlobFileSystem fs = getAbfs(true); - final String fileName = path("/" + methodName.getMethodName()).toString(); + final String fileName = getFileName(); createFileAndGetContent(fs, fileName, FILE_SIZE); // Trying to append with correct CPK headers @@ -246,7 +250,7 @@ public void testAppendWithCPK() throws Exception { @Test public void testAppendWithoutCPK() throws Exception { final AzureBlobFileSystem fs = getAbfs(false); - final String fileName = path("/" + methodName.getMethodName()).toString(); + final String fileName = getFileName(); createFileAndGetContent(fs, fileName, FILE_SIZE); // Trying to append without CPK headers @@ -282,7 +286,7 @@ public void testAppendWithoutCPK() throws Exception { @Test public void testSetGetXAttr() throws Exception { final AzureBlobFileSystem fs = getAbfs(true); - final String fileName = path(methodName.getMethodName()).toString(); + final String fileName = getFileName(); createFileAndGetContent(fs, fileName, FILE_SIZE); String valSent = "testValue"; @@ -416,7 +420,7 @@ public void testListPathWithoutCPK() throws Exception { private void testListPath(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); final Path testPath = path("/" + methodName.getMethodName()); - String testDirName = testPath.toString(); + String testDirName = testPath.toUri().getPath(); fs.mkdirs(testPath); createFileAndGetContent(fs, testDirName + "/aaa", FILE_SIZE); createFileAndGetContent(fs, testDirName + "/bbb", FILE_SIZE); @@ -475,8 +479,7 @@ public void testCreatePathWithoutCPK() throws Exception { private void testCreatePath(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); createFileAndGetContent(fs, testFileName, FILE_SIZE); AbfsClient abfsClient = fs.getAbfsClient(); @@ -519,8 +522,7 @@ public void testRenamePathWithoutCPK() throws Exception { private void testRenamePath(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); createFileAndGetContent(fs, testFileName, FILE_SIZE); FileStatus fileStatusBeforeRename = fs @@ -556,8 +558,7 @@ public void testFlushWithoutCPK() throws Exception { private void testFlush(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); fs.create(new Path(testFileName)).close(); AbfsClient abfsClient = fs.getAbfsClient(); String expectedCPKSha = getCPKSha(fs); @@ -617,8 +618,7 @@ public void testSetPathPropertiesWithoutCPK() throws Exception { private void testSetPathProperties(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); createFileAndGetContent(fs, testFileName, FILE_SIZE); AbfsClient abfsClient = fs.getAbfsClient(); @@ -648,8 +648,7 @@ public void testGetPathStatusFileWithoutCPK() throws Exception { private void testGetPathStatusFile(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); createFileAndGetContent(fs, testFileName, FILE_SIZE); AbfsClient abfsClient = fs.getAbfsClient(); @@ -686,8 +685,7 @@ public void testDeletePathWithoutCPK() throws Exception { private void testDeletePath(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); createFileAndGetContent(fs, testFileName, FILE_SIZE); FileStatus[] listStatuses = fs.listStatus(new Path(testFileName)); @@ -717,8 +715,7 @@ public void testSetPermissionWithoutCPK() throws Exception { private void testSetPermission(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); Assume.assumeTrue(fs.getIsNamespaceEnabled(getTestTracingContext(fs, false))); createFileAndGetContent(fs, testFileName, FILE_SIZE); AbfsClient abfsClient = fs.getAbfsClient(); @@ -743,8 +740,7 @@ public void testSetAclWithoutCPK() throws Exception { private void testSetAcl(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); TracingContext tracingContext = getTestTracingContext(fs, false); Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext)); createFileAndGetContent(fs, testFileName, FILE_SIZE); @@ -773,8 +769,7 @@ public void testGetAclWithoutCPK() throws Exception { private void testGetAcl(final boolean isWithCPK) throws Exception { final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); TracingContext tracingContext = getTestTracingContext(fs, false); Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext)); createFileAndGetContent(fs, testFileName, FILE_SIZE); @@ -804,8 +799,7 @@ private void testCheckAccess(final boolean isWithCPK) throws Exception { getAuthType() == AuthType.OAuth); final AzureBlobFileSystem fs = getAbfs(isWithCPK); - final String testFileName = path("/" + methodName.getMethodName()) - .toString(); + final String testFileName = getFileName(); fs.create(new Path(testFileName)).close(); AbfsClient abfsClient = fs.getAbfsClient(); AbfsRestOperation abfsRestOperation = abfsClient diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java index 86bb2adbe56ed..7bbd75ca922d7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java @@ -382,6 +382,7 @@ public void testConfigPropNotFound() throws Throwable { for (String key : CONFIG_KEYS) { setAuthConfig(abfsConf, true, AuthType.OAuth); + abfsConf.unset(key); abfsConf.unset(key + "." + accountName); testMissingConfigKey(abfsConf, key); } From 6c6df40d35e69f0340ab7a9afae3f96be1f497ca Mon Sep 17 00:00:00 2001 From: Anuj Modi <128447756+anujmodi2021@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:01:56 -0700 Subject: [PATCH 079/155] HADOOP-18869: [ABFS] Fix behavior of a File System APIs on root path (#6003) Contributed by Anmol Asrani --- .../fs/azurebfs/AzureBlobFileSystem.java | 42 ++++++- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../fs/azurebfs/services/AbfsErrors.java | 2 +- .../ITestAzureBlobFileSystemAttributes.java | 104 +++++++++++++----- .../ITestAzureBlobFileSystemCreate.java | 25 ++++- .../hadoop-azure/src/test/resources/abfs.xml | 2 +- 6 files changed, 137 insertions(+), 40 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 8f7fbc570221e..b234f76d5d9dd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -110,6 +110,7 @@ import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; +import static java.net.HttpURLConnection.HTTP_CONFLICT; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS; @@ -120,6 +121,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; +import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_CREATE_ON_ROOT; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -324,6 +326,12 @@ public FSDataOutputStream create(final Path f, statIncrement(CALL_CREATE); trailingPeriodCheck(f); + if (f.isRoot()) { + throw new AbfsRestOperationException(HTTP_CONFLICT, + AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), + ERR_CREATE_ON_ROOT, + null); + } Path qualifiedPath = makeQualified(f); @@ -348,6 +356,12 @@ public FSDataOutputStream createNonRecursive(final Path f, final FsPermission pe final Progressable progress) throws IOException { statIncrement(CALL_CREATE_NON_RECURSIVE); + if (f.isRoot()) { + throw new AbfsRestOperationException(HTTP_CONFLICT, + AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), + ERR_CREATE_ON_ROOT, + null); + } final Path parent = f.getParent(); TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_NON_RECURSIVE, tracingHeaderFormat, @@ -965,15 +979,25 @@ public void setXAttr(final Path path, TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, listener); - Hashtable properties = abfsStore - .getPathStatus(qualifiedPath, tracingContext); + Hashtable properties; String xAttrName = ensureValidAttributeName(name); + + if (path.isRoot()) { + properties = abfsStore.getFilesystemProperties(tracingContext); + } else { + properties = abfsStore.getPathStatus(qualifiedPath, tracingContext); + } + boolean xAttrExists = properties.containsKey(xAttrName); XAttrSetFlag.validate(name, xAttrExists, flag); String xAttrValue = abfsStore.decodeAttribute(value); properties.put(xAttrName, xAttrValue); - abfsStore.setPathProperties(qualifiedPath, properties, tracingContext); + if (path.isRoot()) { + abfsStore.setFilesystemProperties(properties, tracingContext); + } else { + abfsStore.setPathProperties(qualifiedPath, properties, tracingContext); + } } catch (AzureBlobFileSystemException ex) { checkException(path, ex); } @@ -1005,9 +1029,15 @@ public byte[] getXAttr(final Path path, final String name) TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, listener); - Hashtable properties = abfsStore - .getPathStatus(qualifiedPath, tracingContext); + Hashtable properties; String xAttrName = ensureValidAttributeName(name); + + if (path.isRoot()) { + properties = abfsStore.getFilesystemProperties(tracingContext); + } else { + properties = abfsStore.getPathStatus(qualifiedPath, tracingContext); + } + if (properties.containsKey(xAttrName)) { String xAttrValue = properties.get(xAttrName); value = abfsStore.encodeAttribute(xAttrValue); @@ -1488,7 +1518,7 @@ public static void checkException(final Path path, case HttpURLConnection.HTTP_NOT_FOUND: throw (IOException) new FileNotFoundException(message) .initCause(exception); - case HttpURLConnection.HTTP_CONFLICT: + case HTTP_CONFLICT: throw (IOException) new FileAlreadyExistsException(message) .initCause(exception); case HttpURLConnection.HTTP_FORBIDDEN: diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index ff37ee5ee901e..4b356ceef06db 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -494,7 +494,7 @@ public void setPathProperties(final Path path, final Hashtable properties, TracingContext tracingContext) throws AzureBlobFileSystemException { try (AbfsPerfInfo perfInfo = startTracking("setPathProperties", "setPathProperties")){ - LOG.debug("setFilesystemProperties for filesystem: {} path: {} with properties: {}", + LOG.debug("setPathProperties for filesystem: {} path: {} with properties: {}", client.getFileSystem(), path, properties); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java index e15795efee68d..86285e08f2ce3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java @@ -48,6 +48,6 @@ public final class AbfsErrors { + "operation"; public static final String ERR_NO_LEASE_THREADS = "Lease desired but no lease threads " + "configured, set " + FS_AZURE_LEASE_THREADS; - + public static final String ERR_CREATE_ON_ROOT = "Cannot create file over root path"; private AbfsErrors() {} } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java index beb7d0ebaaa8e..7fe229c519fb6 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.EnumSet; -import org.junit.Assume; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.hadoop.fs.Path; @@ -46,37 +46,14 @@ public ITestAzureBlobFileSystemAttributes() throws Exception { public void testSetGetXAttr() throws Exception { AzureBlobFileSystem fs = getFileSystem(); AbfsConfiguration conf = fs.getAbfsStore().getAbfsConfiguration(); - Assume.assumeTrue(getIsNamespaceEnabled(fs)); - - byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("hi"); - byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("你好"); - String attributeName1 = "user.asciiAttribute"; - String attributeName2 = "user.unicodeAttribute"; - Path testFile = path("setGetXAttr"); - - // after creating a file, the xAttr should not be present - touch(testFile); - assertNull(fs.getXAttr(testFile, attributeName1)); - - // after setting the xAttr on the file, the value should be retrievable - fs.registerListener( - new TracingHeaderValidator(conf.getClientCorrelationId(), - fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0)); - fs.setXAttr(testFile, attributeName1, attributeValue1); - fs.setListenerOperation(FSOperationType.GET_ATTR); - assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1)); - fs.registerListener(null); - - // after setting a second xAttr on the file, the first xAttr values should not be overwritten - fs.setXAttr(testFile, attributeName2, attributeValue2); - assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1)); - assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName2)); + final Path testPath = path("setGetXAttr"); + fs.create(testPath); + testGetSetXAttrHelper(fs, testPath); } @Test public void testSetGetXAttrCreateReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); - Assume.assumeTrue(getIsNamespaceEnabled(fs)); byte[] attributeValue = fs.getAbfsStore().encodeAttribute("one"); String attributeName = "user.someAttribute"; Path testFile = path("createReplaceXAttr"); @@ -84,7 +61,9 @@ public void testSetGetXAttrCreateReplace() throws Exception { // after creating a file, it must be possible to create a new xAttr touch(testFile); fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG); - assertArrayEquals(attributeValue, fs.getXAttr(testFile, attributeName)); + Assertions.assertThat(fs.getXAttr(testFile, attributeName)) + .describedAs("Retrieved Attribute Value is Not as Expected") + .containsExactly(attributeValue); // however after the xAttr is created, creating it again must fail intercept(IOException.class, () -> fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG)); @@ -93,7 +72,6 @@ public void testSetGetXAttrCreateReplace() throws Exception { @Test public void testSetGetXAttrReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); - Assume.assumeTrue(getIsNamespaceEnabled(fs)); byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("one"); byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("two"); String attributeName = "user.someAttribute"; @@ -108,6 +86,72 @@ public void testSetGetXAttrReplace() throws Exception { // however after the xAttr is created, replacing it must succeed fs.setXAttr(testFile, attributeName, attributeValue1, CREATE_FLAG); fs.setXAttr(testFile, attributeName, attributeValue2, REPLACE_FLAG); - assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName)); + Assertions.assertThat(fs.getXAttr(testFile, attributeName)) + .describedAs("Retrieved Attribute Value is Not as Expected") + .containsExactly(attributeValue2); + } + + @Test + public void testGetSetXAttrOnRoot() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + final Path testPath = new Path("/"); + testGetSetXAttrHelper(fs, testPath); + } + + private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, + final Path testPath) throws Exception { + + String attributeName1 = "user.attribute1"; + String attributeName2 = "user.attribute2"; + String decodedAttributeValue1 = "hi"; + String decodedAttributeValue2 = "hello"; + byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1); + byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2); + + // Attribute not present initially + Assertions.assertThat(fs.getXAttr(testPath, attributeName1)) + .describedAs("Cannot get attribute before setting it") + .isNull(); + Assertions.assertThat(fs.getXAttr(testPath, attributeName2)) + .describedAs("Cannot get attribute before setting it") + .isNull(); + + // Set the Attributes + fs.registerListener( + new TracingHeaderValidator(fs.getAbfsStore().getAbfsConfiguration() + .getClientCorrelationId(), + fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0)); + fs.setXAttr(testPath, attributeName1, attributeValue1); + + // Check if the attribute is retrievable + fs.setListenerOperation(FSOperationType.GET_ATTR); + byte[] rv = fs.getXAttr(testPath, attributeName1); + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue1); + Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue1); + fs.registerListener(null); + + // Set the second Attribute + fs.setXAttr(testPath, attributeName2, attributeValue2); + + // Check all the attributes present and previous Attribute not overridden + rv = fs.getXAttr(testPath, attributeName1); + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue1); + Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue1); + + rv = fs.getXAttr(testPath, attributeName2); + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue2); + Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue2); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index d9a3cea089f63..d0b3ff2974007 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -25,6 +25,7 @@ import java.util.EnumSet; import java.util.UUID; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.hadoop.conf.Configuration; @@ -33,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.test.GenericTestUtils; @@ -146,6 +148,26 @@ public void testCreateNonRecursive2() throws Exception { assertIsFile(fs, testFile); } + @Test + public void testCreateOnRoot() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + Path testFile = path(AbfsHttpConstants.ROOT_PATH); + AbfsRestOperationException ex = intercept(AbfsRestOperationException.class, () -> + fs.create(testFile, true)); + if (ex.getStatusCode() != HTTP_CONFLICT) { + // Request should fail with 409. + throw ex; + } + + ex = intercept(AbfsRestOperationException.class, () -> + fs.createNonRecursive(testFile, FsPermission.getDefault(), + false, 1024, (short) 1, 1024, null)); + if (ex.getStatusCode() != HTTP_CONFLICT) { + // Request should fail with 409. + throw ex; + } + } + /** * Attempts to use to the ABFS stream after it is closed. */ @@ -190,7 +212,8 @@ public void testTryWithResources() throws Throwable { // the exception raised in close() must be in the caught exception's // suppressed list Throwable[] suppressed = fnfe.getSuppressed(); - assertEquals("suppressed count", 1, suppressed.length); + Assertions.assertThat(suppressed.length) + .describedAs("suppressed count should be 1").isEqualTo(1); Throwable inner = suppressed[0]; if (!(inner instanceof IOException)) { throw inner; diff --git a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml index f06e5cac9b8b2..007ca2abd6b95 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml @@ -19,7 +19,7 @@ fs.contract.test.root-tests-enabled - false + true From 882378c3e9d6bd16884655927a290586350782bb Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 9 Oct 2023 20:03:54 +0100 Subject: [PATCH 080/155] Revert "HADOOP-18869: [ABFS] Fix behavior of a File System APIs on root path (#6003)" This reverts commit 6c6df40d35e69f0340ab7a9afae3f96be1f497ca. ...so as to give the correct credit --- .../fs/azurebfs/AzureBlobFileSystem.java | 42 +------ .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../fs/azurebfs/services/AbfsErrors.java | 2 +- .../ITestAzureBlobFileSystemAttributes.java | 104 +++++------------- .../ITestAzureBlobFileSystemCreate.java | 25 +---- .../hadoop-azure/src/test/resources/abfs.xml | 2 +- 6 files changed, 40 insertions(+), 137 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index b234f76d5d9dd..8f7fbc570221e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -110,7 +110,6 @@ import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; -import static java.net.HttpURLConnection.HTTP_CONFLICT; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS; @@ -121,7 +120,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; -import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_CREATE_ON_ROOT; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -326,12 +324,6 @@ public FSDataOutputStream create(final Path f, statIncrement(CALL_CREATE); trailingPeriodCheck(f); - if (f.isRoot()) { - throw new AbfsRestOperationException(HTTP_CONFLICT, - AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), - ERR_CREATE_ON_ROOT, - null); - } Path qualifiedPath = makeQualified(f); @@ -356,12 +348,6 @@ public FSDataOutputStream createNonRecursive(final Path f, final FsPermission pe final Progressable progress) throws IOException { statIncrement(CALL_CREATE_NON_RECURSIVE); - if (f.isRoot()) { - throw new AbfsRestOperationException(HTTP_CONFLICT, - AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), - ERR_CREATE_ON_ROOT, - null); - } final Path parent = f.getParent(); TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_NON_RECURSIVE, tracingHeaderFormat, @@ -979,25 +965,15 @@ public void setXAttr(final Path path, TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, listener); - Hashtable properties; + Hashtable properties = abfsStore + .getPathStatus(qualifiedPath, tracingContext); String xAttrName = ensureValidAttributeName(name); - - if (path.isRoot()) { - properties = abfsStore.getFilesystemProperties(tracingContext); - } else { - properties = abfsStore.getPathStatus(qualifiedPath, tracingContext); - } - boolean xAttrExists = properties.containsKey(xAttrName); XAttrSetFlag.validate(name, xAttrExists, flag); String xAttrValue = abfsStore.decodeAttribute(value); properties.put(xAttrName, xAttrValue); - if (path.isRoot()) { - abfsStore.setFilesystemProperties(properties, tracingContext); - } else { - abfsStore.setPathProperties(qualifiedPath, properties, tracingContext); - } + abfsStore.setPathProperties(qualifiedPath, properties, tracingContext); } catch (AzureBlobFileSystemException ex) { checkException(path, ex); } @@ -1029,15 +1005,9 @@ public byte[] getXAttr(final Path path, final String name) TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, listener); - Hashtable properties; + Hashtable properties = abfsStore + .getPathStatus(qualifiedPath, tracingContext); String xAttrName = ensureValidAttributeName(name); - - if (path.isRoot()) { - properties = abfsStore.getFilesystemProperties(tracingContext); - } else { - properties = abfsStore.getPathStatus(qualifiedPath, tracingContext); - } - if (properties.containsKey(xAttrName)) { String xAttrValue = properties.get(xAttrName); value = abfsStore.encodeAttribute(xAttrValue); @@ -1518,7 +1488,7 @@ public static void checkException(final Path path, case HttpURLConnection.HTTP_NOT_FOUND: throw (IOException) new FileNotFoundException(message) .initCause(exception); - case HTTP_CONFLICT: + case HttpURLConnection.HTTP_CONFLICT: throw (IOException) new FileAlreadyExistsException(message) .initCause(exception); case HttpURLConnection.HTTP_FORBIDDEN: diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 4b356ceef06db..ff37ee5ee901e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -494,7 +494,7 @@ public void setPathProperties(final Path path, final Hashtable properties, TracingContext tracingContext) throws AzureBlobFileSystemException { try (AbfsPerfInfo perfInfo = startTracking("setPathProperties", "setPathProperties")){ - LOG.debug("setPathProperties for filesystem: {} path: {} with properties: {}", + LOG.debug("setFilesystemProperties for filesystem: {} path: {} with properties: {}", client.getFileSystem(), path, properties); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java index 86285e08f2ce3..e15795efee68d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java @@ -48,6 +48,6 @@ public final class AbfsErrors { + "operation"; public static final String ERR_NO_LEASE_THREADS = "Lease desired but no lease threads " + "configured, set " + FS_AZURE_LEASE_THREADS; - public static final String ERR_CREATE_ON_ROOT = "Cannot create file over root path"; + private AbfsErrors() {} } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java index 7fe229c519fb6..beb7d0ebaaa8e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.EnumSet; -import org.assertj.core.api.Assertions; +import org.junit.Assume; import org.junit.Test; import org.apache.hadoop.fs.Path; @@ -46,14 +46,37 @@ public ITestAzureBlobFileSystemAttributes() throws Exception { public void testSetGetXAttr() throws Exception { AzureBlobFileSystem fs = getFileSystem(); AbfsConfiguration conf = fs.getAbfsStore().getAbfsConfiguration(); - final Path testPath = path("setGetXAttr"); - fs.create(testPath); - testGetSetXAttrHelper(fs, testPath); + Assume.assumeTrue(getIsNamespaceEnabled(fs)); + + byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("hi"); + byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("你好"); + String attributeName1 = "user.asciiAttribute"; + String attributeName2 = "user.unicodeAttribute"; + Path testFile = path("setGetXAttr"); + + // after creating a file, the xAttr should not be present + touch(testFile); + assertNull(fs.getXAttr(testFile, attributeName1)); + + // after setting the xAttr on the file, the value should be retrievable + fs.registerListener( + new TracingHeaderValidator(conf.getClientCorrelationId(), + fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0)); + fs.setXAttr(testFile, attributeName1, attributeValue1); + fs.setListenerOperation(FSOperationType.GET_ATTR); + assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1)); + fs.registerListener(null); + + // after setting a second xAttr on the file, the first xAttr values should not be overwritten + fs.setXAttr(testFile, attributeName2, attributeValue2); + assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1)); + assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName2)); } @Test public void testSetGetXAttrCreateReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); + Assume.assumeTrue(getIsNamespaceEnabled(fs)); byte[] attributeValue = fs.getAbfsStore().encodeAttribute("one"); String attributeName = "user.someAttribute"; Path testFile = path("createReplaceXAttr"); @@ -61,9 +84,7 @@ public void testSetGetXAttrCreateReplace() throws Exception { // after creating a file, it must be possible to create a new xAttr touch(testFile); fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG); - Assertions.assertThat(fs.getXAttr(testFile, attributeName)) - .describedAs("Retrieved Attribute Value is Not as Expected") - .containsExactly(attributeValue); + assertArrayEquals(attributeValue, fs.getXAttr(testFile, attributeName)); // however after the xAttr is created, creating it again must fail intercept(IOException.class, () -> fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG)); @@ -72,6 +93,7 @@ public void testSetGetXAttrCreateReplace() throws Exception { @Test public void testSetGetXAttrReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); + Assume.assumeTrue(getIsNamespaceEnabled(fs)); byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("one"); byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("two"); String attributeName = "user.someAttribute"; @@ -86,72 +108,6 @@ public void testSetGetXAttrReplace() throws Exception { // however after the xAttr is created, replacing it must succeed fs.setXAttr(testFile, attributeName, attributeValue1, CREATE_FLAG); fs.setXAttr(testFile, attributeName, attributeValue2, REPLACE_FLAG); - Assertions.assertThat(fs.getXAttr(testFile, attributeName)) - .describedAs("Retrieved Attribute Value is Not as Expected") - .containsExactly(attributeValue2); - } - - @Test - public void testGetSetXAttrOnRoot() throws Exception { - AzureBlobFileSystem fs = getFileSystem(); - final Path testPath = new Path("/"); - testGetSetXAttrHelper(fs, testPath); - } - - private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, - final Path testPath) throws Exception { - - String attributeName1 = "user.attribute1"; - String attributeName2 = "user.attribute2"; - String decodedAttributeValue1 = "hi"; - String decodedAttributeValue2 = "hello"; - byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1); - byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2); - - // Attribute not present initially - Assertions.assertThat(fs.getXAttr(testPath, attributeName1)) - .describedAs("Cannot get attribute before setting it") - .isNull(); - Assertions.assertThat(fs.getXAttr(testPath, attributeName2)) - .describedAs("Cannot get attribute before setting it") - .isNull(); - - // Set the Attributes - fs.registerListener( - new TracingHeaderValidator(fs.getAbfsStore().getAbfsConfiguration() - .getClientCorrelationId(), - fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0)); - fs.setXAttr(testPath, attributeName1, attributeValue1); - - // Check if the attribute is retrievable - fs.setListenerOperation(FSOperationType.GET_ATTR); - byte[] rv = fs.getXAttr(testPath, attributeName1); - Assertions.assertThat(rv) - .describedAs("Retrieved Attribute Does not Matches in Encoded Form") - .containsExactly(attributeValue1); - Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) - .describedAs("Retrieved Attribute Does not Matches in Decoded Form") - .isEqualTo(decodedAttributeValue1); - fs.registerListener(null); - - // Set the second Attribute - fs.setXAttr(testPath, attributeName2, attributeValue2); - - // Check all the attributes present and previous Attribute not overridden - rv = fs.getXAttr(testPath, attributeName1); - Assertions.assertThat(rv) - .describedAs("Retrieved Attribute Does not Matches in Encoded Form") - .containsExactly(attributeValue1); - Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) - .describedAs("Retrieved Attribute Does not Matches in Decoded Form") - .isEqualTo(decodedAttributeValue1); - - rv = fs.getXAttr(testPath, attributeName2); - Assertions.assertThat(rv) - .describedAs("Retrieved Attribute Does not Matches in Encoded Form") - .containsExactly(attributeValue2); - Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) - .describedAs("Retrieved Attribute Does not Matches in Decoded Form") - .isEqualTo(decodedAttributeValue2); + assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName)); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index d0b3ff2974007..d9a3cea089f63 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -25,7 +25,6 @@ import java.util.EnumSet; import java.util.UUID; -import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.hadoop.conf.Configuration; @@ -34,7 +33,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.test.GenericTestUtils; @@ -148,26 +146,6 @@ public void testCreateNonRecursive2() throws Exception { assertIsFile(fs, testFile); } - @Test - public void testCreateOnRoot() throws Exception { - final AzureBlobFileSystem fs = getFileSystem(); - Path testFile = path(AbfsHttpConstants.ROOT_PATH); - AbfsRestOperationException ex = intercept(AbfsRestOperationException.class, () -> - fs.create(testFile, true)); - if (ex.getStatusCode() != HTTP_CONFLICT) { - // Request should fail with 409. - throw ex; - } - - ex = intercept(AbfsRestOperationException.class, () -> - fs.createNonRecursive(testFile, FsPermission.getDefault(), - false, 1024, (short) 1, 1024, null)); - if (ex.getStatusCode() != HTTP_CONFLICT) { - // Request should fail with 409. - throw ex; - } - } - /** * Attempts to use to the ABFS stream after it is closed. */ @@ -212,8 +190,7 @@ public void testTryWithResources() throws Throwable { // the exception raised in close() must be in the caught exception's // suppressed list Throwable[] suppressed = fnfe.getSuppressed(); - Assertions.assertThat(suppressed.length) - .describedAs("suppressed count should be 1").isEqualTo(1); + assertEquals("suppressed count", 1, suppressed.length); Throwable inner = suppressed[0]; if (!(inner instanceof IOException)) { throw inner; diff --git a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml index 007ca2abd6b95..f06e5cac9b8b2 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml @@ -19,7 +19,7 @@ fs.contract.test.root-tests-enabled - true + false From 594e9f29f5d89563bd9afc2e45553d624a6accc7 Mon Sep 17 00:00:00 2001 From: Anuj Modi <128447756+anujmodi2021@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:01:56 -0700 Subject: [PATCH 081/155] HADOOP-18869: [ABFS] Fix behavior of a File System APIs on root path (#6003) Contributed by Anuj Modi --- .../fs/azurebfs/AzureBlobFileSystem.java | 42 ++++++- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../fs/azurebfs/services/AbfsErrors.java | 2 +- .../ITestAzureBlobFileSystemAttributes.java | 104 +++++++++++++----- .../ITestAzureBlobFileSystemCreate.java | 25 ++++- .../hadoop-azure/src/test/resources/abfs.xml | 2 +- 6 files changed, 137 insertions(+), 40 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 8f7fbc570221e..b234f76d5d9dd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -110,6 +110,7 @@ import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; +import static java.net.HttpURLConnection.HTTP_CONFLICT; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS; @@ -120,6 +121,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; +import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_CREATE_ON_ROOT; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -324,6 +326,12 @@ public FSDataOutputStream create(final Path f, statIncrement(CALL_CREATE); trailingPeriodCheck(f); + if (f.isRoot()) { + throw new AbfsRestOperationException(HTTP_CONFLICT, + AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), + ERR_CREATE_ON_ROOT, + null); + } Path qualifiedPath = makeQualified(f); @@ -348,6 +356,12 @@ public FSDataOutputStream createNonRecursive(final Path f, final FsPermission pe final Progressable progress) throws IOException { statIncrement(CALL_CREATE_NON_RECURSIVE); + if (f.isRoot()) { + throw new AbfsRestOperationException(HTTP_CONFLICT, + AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), + ERR_CREATE_ON_ROOT, + null); + } final Path parent = f.getParent(); TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_NON_RECURSIVE, tracingHeaderFormat, @@ -965,15 +979,25 @@ public void setXAttr(final Path path, TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, listener); - Hashtable properties = abfsStore - .getPathStatus(qualifiedPath, tracingContext); + Hashtable properties; String xAttrName = ensureValidAttributeName(name); + + if (path.isRoot()) { + properties = abfsStore.getFilesystemProperties(tracingContext); + } else { + properties = abfsStore.getPathStatus(qualifiedPath, tracingContext); + } + boolean xAttrExists = properties.containsKey(xAttrName); XAttrSetFlag.validate(name, xAttrExists, flag); String xAttrValue = abfsStore.decodeAttribute(value); properties.put(xAttrName, xAttrValue); - abfsStore.setPathProperties(qualifiedPath, properties, tracingContext); + if (path.isRoot()) { + abfsStore.setFilesystemProperties(properties, tracingContext); + } else { + abfsStore.setPathProperties(qualifiedPath, properties, tracingContext); + } } catch (AzureBlobFileSystemException ex) { checkException(path, ex); } @@ -1005,9 +1029,15 @@ public byte[] getXAttr(final Path path, final String name) TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, listener); - Hashtable properties = abfsStore - .getPathStatus(qualifiedPath, tracingContext); + Hashtable properties; String xAttrName = ensureValidAttributeName(name); + + if (path.isRoot()) { + properties = abfsStore.getFilesystemProperties(tracingContext); + } else { + properties = abfsStore.getPathStatus(qualifiedPath, tracingContext); + } + if (properties.containsKey(xAttrName)) { String xAttrValue = properties.get(xAttrName); value = abfsStore.encodeAttribute(xAttrValue); @@ -1488,7 +1518,7 @@ public static void checkException(final Path path, case HttpURLConnection.HTTP_NOT_FOUND: throw (IOException) new FileNotFoundException(message) .initCause(exception); - case HttpURLConnection.HTTP_CONFLICT: + case HTTP_CONFLICT: throw (IOException) new FileAlreadyExistsException(message) .initCause(exception); case HttpURLConnection.HTTP_FORBIDDEN: diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index ff37ee5ee901e..4b356ceef06db 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -494,7 +494,7 @@ public void setPathProperties(final Path path, final Hashtable properties, TracingContext tracingContext) throws AzureBlobFileSystemException { try (AbfsPerfInfo perfInfo = startTracking("setPathProperties", "setPathProperties")){ - LOG.debug("setFilesystemProperties for filesystem: {} path: {} with properties: {}", + LOG.debug("setPathProperties for filesystem: {} path: {} with properties: {}", client.getFileSystem(), path, properties); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java index e15795efee68d..86285e08f2ce3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java @@ -48,6 +48,6 @@ public final class AbfsErrors { + "operation"; public static final String ERR_NO_LEASE_THREADS = "Lease desired but no lease threads " + "configured, set " + FS_AZURE_LEASE_THREADS; - + public static final String ERR_CREATE_ON_ROOT = "Cannot create file over root path"; private AbfsErrors() {} } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java index beb7d0ebaaa8e..7fe229c519fb6 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.util.EnumSet; -import org.junit.Assume; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.hadoop.fs.Path; @@ -46,37 +46,14 @@ public ITestAzureBlobFileSystemAttributes() throws Exception { public void testSetGetXAttr() throws Exception { AzureBlobFileSystem fs = getFileSystem(); AbfsConfiguration conf = fs.getAbfsStore().getAbfsConfiguration(); - Assume.assumeTrue(getIsNamespaceEnabled(fs)); - - byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("hi"); - byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("你好"); - String attributeName1 = "user.asciiAttribute"; - String attributeName2 = "user.unicodeAttribute"; - Path testFile = path("setGetXAttr"); - - // after creating a file, the xAttr should not be present - touch(testFile); - assertNull(fs.getXAttr(testFile, attributeName1)); - - // after setting the xAttr on the file, the value should be retrievable - fs.registerListener( - new TracingHeaderValidator(conf.getClientCorrelationId(), - fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0)); - fs.setXAttr(testFile, attributeName1, attributeValue1); - fs.setListenerOperation(FSOperationType.GET_ATTR); - assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1)); - fs.registerListener(null); - - // after setting a second xAttr on the file, the first xAttr values should not be overwritten - fs.setXAttr(testFile, attributeName2, attributeValue2); - assertArrayEquals(attributeValue1, fs.getXAttr(testFile, attributeName1)); - assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName2)); + final Path testPath = path("setGetXAttr"); + fs.create(testPath); + testGetSetXAttrHelper(fs, testPath); } @Test public void testSetGetXAttrCreateReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); - Assume.assumeTrue(getIsNamespaceEnabled(fs)); byte[] attributeValue = fs.getAbfsStore().encodeAttribute("one"); String attributeName = "user.someAttribute"; Path testFile = path("createReplaceXAttr"); @@ -84,7 +61,9 @@ public void testSetGetXAttrCreateReplace() throws Exception { // after creating a file, it must be possible to create a new xAttr touch(testFile); fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG); - assertArrayEquals(attributeValue, fs.getXAttr(testFile, attributeName)); + Assertions.assertThat(fs.getXAttr(testFile, attributeName)) + .describedAs("Retrieved Attribute Value is Not as Expected") + .containsExactly(attributeValue); // however after the xAttr is created, creating it again must fail intercept(IOException.class, () -> fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG)); @@ -93,7 +72,6 @@ public void testSetGetXAttrCreateReplace() throws Exception { @Test public void testSetGetXAttrReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); - Assume.assumeTrue(getIsNamespaceEnabled(fs)); byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("one"); byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("two"); String attributeName = "user.someAttribute"; @@ -108,6 +86,72 @@ public void testSetGetXAttrReplace() throws Exception { // however after the xAttr is created, replacing it must succeed fs.setXAttr(testFile, attributeName, attributeValue1, CREATE_FLAG); fs.setXAttr(testFile, attributeName, attributeValue2, REPLACE_FLAG); - assertArrayEquals(attributeValue2, fs.getXAttr(testFile, attributeName)); + Assertions.assertThat(fs.getXAttr(testFile, attributeName)) + .describedAs("Retrieved Attribute Value is Not as Expected") + .containsExactly(attributeValue2); + } + + @Test + public void testGetSetXAttrOnRoot() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + final Path testPath = new Path("/"); + testGetSetXAttrHelper(fs, testPath); + } + + private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, + final Path testPath) throws Exception { + + String attributeName1 = "user.attribute1"; + String attributeName2 = "user.attribute2"; + String decodedAttributeValue1 = "hi"; + String decodedAttributeValue2 = "hello"; + byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue1); + byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2); + + // Attribute not present initially + Assertions.assertThat(fs.getXAttr(testPath, attributeName1)) + .describedAs("Cannot get attribute before setting it") + .isNull(); + Assertions.assertThat(fs.getXAttr(testPath, attributeName2)) + .describedAs("Cannot get attribute before setting it") + .isNull(); + + // Set the Attributes + fs.registerListener( + new TracingHeaderValidator(fs.getAbfsStore().getAbfsConfiguration() + .getClientCorrelationId(), + fs.getFileSystemId(), FSOperationType.SET_ATTR, true, 0)); + fs.setXAttr(testPath, attributeName1, attributeValue1); + + // Check if the attribute is retrievable + fs.setListenerOperation(FSOperationType.GET_ATTR); + byte[] rv = fs.getXAttr(testPath, attributeName1); + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue1); + Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue1); + fs.registerListener(null); + + // Set the second Attribute + fs.setXAttr(testPath, attributeName2, attributeValue2); + + // Check all the attributes present and previous Attribute not overridden + rv = fs.getXAttr(testPath, attributeName1); + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue1); + Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue1); + + rv = fs.getXAttr(testPath, attributeName2); + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue2); + Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue2); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index d9a3cea089f63..d0b3ff2974007 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -25,6 +25,7 @@ import java.util.EnumSet; import java.util.UUID; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.hadoop.conf.Configuration; @@ -33,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.test.GenericTestUtils; @@ -146,6 +148,26 @@ public void testCreateNonRecursive2() throws Exception { assertIsFile(fs, testFile); } + @Test + public void testCreateOnRoot() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + Path testFile = path(AbfsHttpConstants.ROOT_PATH); + AbfsRestOperationException ex = intercept(AbfsRestOperationException.class, () -> + fs.create(testFile, true)); + if (ex.getStatusCode() != HTTP_CONFLICT) { + // Request should fail with 409. + throw ex; + } + + ex = intercept(AbfsRestOperationException.class, () -> + fs.createNonRecursive(testFile, FsPermission.getDefault(), + false, 1024, (short) 1, 1024, null)); + if (ex.getStatusCode() != HTTP_CONFLICT) { + // Request should fail with 409. + throw ex; + } + } + /** * Attempts to use to the ABFS stream after it is closed. */ @@ -190,7 +212,8 @@ public void testTryWithResources() throws Throwable { // the exception raised in close() must be in the caught exception's // suppressed list Throwable[] suppressed = fnfe.getSuppressed(); - assertEquals("suppressed count", 1, suppressed.length); + Assertions.assertThat(suppressed.length) + .describedAs("suppressed count should be 1").isEqualTo(1); Throwable inner = suppressed[0]; if (!(inner instanceof IOException)) { throw inner; diff --git a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml index f06e5cac9b8b2..007ca2abd6b95 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml @@ -19,7 +19,7 @@ fs.contract.test.root-tests-enabled - false + true From b00d605832eaf7f05f52681cd44f552b398d8825 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 10 Oct 2023 05:06:41 +0800 Subject: [PATCH 082/155] YARN-9048. Add znode hierarchy in Federation ZK State Store. (#6016) --- .../impl/ZookeeperFederationStateStore.java | 412 ++++++++++++++++-- .../src/test/resources/yarn-site.xml | 22 + 2 files changed, 395 insertions(+), 39 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/yarn-site.xml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 63a8aa62f0030..8c59b2ad8cf4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -26,6 +26,8 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.List; +import java.util.Map; +import java.util.HashMap; import java.util.TimeZone; import java.util.Comparator; import java.util.stream.Collectors; @@ -111,6 +113,7 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,8 +134,16 @@ * | |----- SC1 * | |----- SC2 * |--- APPLICATION - * | |----- APP1 - * | |----- APP2 + * | |----- HIERARCHIES + * | | |----- 1 + * | | | |----- (#ApplicationId barring last character) + * | | | | | |----- APP Data + * | | | .... + * | | | + * | | |----- 2 + * | | | |----- (#ApplicationId barring last 2 characters) + * | | | | |----- (#Last 2 characters of ApplicationId) + * | | | | | |----- APP Data * |--- POLICY * | |----- QUEUE1 * | |----- QUEUE1 @@ -194,12 +205,19 @@ public class ZookeeperFederationStateStore implements FederationStateStore { private int maxAppsInStateStore; /** Directory to store the delegation token data. **/ + private Map routerAppRootHierarchies; private String routerRMDTSecretManagerRoot; private String routerRMDTMasterKeysRootPath; private String routerRMDelegationTokensRootPath; private String routerRMSequenceNumberPath; private String routerRMMasterKeyIdPath; + private int appIdNodeSplitIndex = 0; + private final static int HIERARCHIES_LEVEL = 4; + + @VisibleForTesting + public static final String ROUTER_APP_ROOT_HIERARCHIES = "HIERARCHIES"; + private volatile Clock clock = SystemClock.getInstance(); protected static final Version CURRENT_VERSION_INFO = Version.newInstance(1, 1); @@ -208,6 +226,27 @@ public class ZookeeperFederationStateStore implements FederationStateStore { private ZKFederationStateStoreOpDurations opDurations = ZKFederationStateStoreOpDurations.getInstance(); + /* + * Indicates different app attempt state store operations. + */ + private enum AppAttemptOp { + STORE, + UPDATE, + REMOVE + }; + + /** + * Encapsulates full app node path and corresponding split index. + */ + private final static class AppNodeSplitInfo { + private final String path; + private final int splitIndex; + AppNodeSplitInfo(String path, int splitIndex) { + this.path = path; + this.splitIndex = splitIndex; + } + } + @Override public void init(Configuration conf) throws YarnException { @@ -234,6 +273,23 @@ public void init(Configuration conf) throws YarnException { reservationsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_RESERVATION); versionNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_VERSION); + String hierarchiesPath = getNodePath(appsZNode, ROUTER_APP_ROOT_HIERARCHIES); + routerAppRootHierarchies = new HashMap<>(); + routerAppRootHierarchies.put(0, appsZNode); + for (int splitIndex = 1; splitIndex <= HIERARCHIES_LEVEL; splitIndex++) { + routerAppRootHierarchies.put(splitIndex, + getNodePath(hierarchiesPath, Integer.toString(splitIndex))); + } + + appIdNodeSplitIndex = conf.getInt(YarnConfiguration.ZK_APPID_NODE_SPLIT_INDEX, + YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX); + if (appIdNodeSplitIndex < 1 || appIdNodeSplitIndex > HIERARCHIES_LEVEL) { + LOG.info("Invalid value {} for config {} specified. Resetting it to {}", + appIdNodeSplitIndex, YarnConfiguration.ZK_APPID_NODE_SPLIT_INDEX, + YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX); + appIdNodeSplitIndex = YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX; + } + // delegation token znodes routerRMDTSecretManagerRoot = getNodePath(baseZNode, ROUTER_RM_DT_SECRET_MANAGER_ROOT); routerRMDTMasterKeysRootPath = getNodePath(routerRMDTSecretManagerRoot, @@ -250,6 +306,12 @@ public void init(Configuration conf) throws YarnException { List zkAcl = ZKCuratorManager.getZKAcls(conf); zkManager.createRootDirRecursively(membershipZNode, zkAcl); zkManager.createRootDirRecursively(appsZNode, zkAcl); + zkManager.createRootDirRecursively( + getNodePath(appsZNode, ROUTER_APP_ROOT_HIERARCHIES)); + for (int splitIndex = 1; splitIndex <= HIERARCHIES_LEVEL; splitIndex++) { + zkManager.createRootDirRecursively( + routerAppRootHierarchies.get(splitIndex)); + } zkManager.createRootDirRecursively(policiesZNode, zkAcl); zkManager.createRootDirRecursively(reservationsZNode, zkAcl); zkManager.createRootDirRecursively(routerRMDTSecretManagerRoot, zkAcl); @@ -320,6 +382,21 @@ public void close() throws Exception { } } + /** + * Register the home {@code SubClusterId} of the newly submitted + * {@code ApplicationId}. Currently response is empty if the operation was + * successful, if not an exception reporting reason for a failure. If a + * mapping for the application already existed, the {@code SubClusterId} in + * this response will return the existing mapping which might be different + * from that in the {@code AddApplicationHomeSubClusterRequest}. + * + * @param request the request to register a new application with its home sub-cluster. + * @return upon successful registration of the application in the StateStore, + * {@code AddApplicationHomeSubClusterRequest} containing the home + * sub-cluster of the application. Otherwise, an exception reporting + * reason for a failure. + * @throws YarnException indicates exceptions from yarn servers. + */ @Override public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( AddApplicationHomeSubClusterRequest request) throws YarnException { @@ -367,6 +444,17 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( throw new YarnException("Cannot addApplicationHomeSubCluster by request"); } + /** + * Update the home {@code SubClusterId} of a previously submitted + * {@code ApplicationId}. Currently response is empty if the operation was + * successful, if not an exception reporting reason for a failure. + * + * @param request the request to update the home sub-cluster of an + * application. + * @return empty on successful update of the application in the StateStore, if + * not an exception reporting reason for a failure + * @throws YarnException indicates exceptions from yarn servers. + */ @Override public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster( UpdateApplicationHomeSubClusterRequest request) throws YarnException { @@ -402,6 +490,15 @@ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster( return UpdateApplicationHomeSubClusterResponse.newInstance(); } + /** + * Get information about the application identified by the input + * {@code ApplicationId}. + * + * @param request contains the application queried + * @return {@code ApplicationHomeSubCluster} containing the application's home + * subcluster + * @throws YarnException indicates exceptions from yarn servers. + */ @Override public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster( GetApplicationHomeSubClusterRequest request) throws YarnException { @@ -437,6 +534,14 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster( subClusterId, createTime); } + /** + * Get the {@code ApplicationHomeSubCluster} list representing the mapping of + * all submitted applications to it's home sub-cluster. + * + * @param request empty representing all applications + * @return the mapping of all submitted application to it's home sub-cluster + * @throws YarnException indicates exceptions from yarn servers. + */ @Override public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( GetApplicationsHomeSubClusterRequest request) throws YarnException { @@ -448,9 +553,7 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( try { long start = clock.getTime(); SubClusterId requestSC = request.getSubClusterId(); - List children = zkManager.getChildren(appsZNode); - List result = children.stream() - .map(child -> generateAppHomeSC(child)) + List result = loadRouterApplications().stream() .sorted(Comparator.comparing(ApplicationHomeSubCluster::getCreateTime).reversed()) .filter(appHomeSC -> filterHomeSubCluster(requestSC, appHomeSC.getHomeSubCluster())) .limit(maxAppsInStateStore) @@ -467,48 +570,51 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( throw new YarnException("Cannot get app by request"); } - private ApplicationHomeSubCluster generateAppHomeSC(String appId) { - try { - // Parse ApplicationHomeSubCluster - ApplicationId applicationId = ApplicationId.fromString(appId); - ApplicationHomeSubCluster zkStoreApplicationHomeSubCluster = - getApplicationHomeSubCluster(applicationId); - - // Prepare to return data - SubClusterId subClusterId = zkStoreApplicationHomeSubCluster.getHomeSubCluster(); - ApplicationHomeSubCluster resultApplicationHomeSubCluster = - ApplicationHomeSubCluster.newInstance(applicationId, subClusterId); - return resultApplicationHomeSubCluster; - } catch (Exception ex) { - LOG.error("get homeSubCluster by appId = {}.", appId, ex); - } - return null; - } - + /** + * Delete the mapping of home {@code SubClusterId} of a previously submitted + * {@code ApplicationId}. Currently response is empty if the operation was + * successful, if not an exception reporting reason for a failure. + * + * @param request the request to delete the home sub-cluster of an + * application. + * @return empty on successful update of the application in the StateStore, if + * not an exception reporting reason for a failure + * @throws YarnException if the request is invalid/fails + */ @Override - public DeleteApplicationHomeSubClusterResponse - deleteApplicationHomeSubCluster( - DeleteApplicationHomeSubClusterRequest request) - throws YarnException { + public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster( + DeleteApplicationHomeSubClusterRequest request) throws YarnException { long start = clock.getTime(); FederationApplicationHomeSubClusterStoreInputValidator.validate(request); ApplicationId appId = request.getApplicationId(); - String appZNode = getNodePath(appsZNode, appId.toString()); + String appIdRemovePath = getLeafAppIdNodePath(appId.toString(), false); + int splitIndex = appIdNodeSplitIndex; - boolean exists = false; + boolean exists = true; try { - exists = zkManager.exists(appZNode); + if (!exists(appIdRemovePath)) { + AppNodeSplitInfo alternatePathInfo = getAlternatePath(appId.toString()); + if (alternatePathInfo != null) { + appIdRemovePath = alternatePathInfo.path; + splitIndex = alternatePathInfo.splitIndex; + } else { + exists = false; + } + } } catch (Exception e) { String errMsg = "Cannot check app: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } + if (!exists) { String errMsg = "Application " + appId + " does not exist"; FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } try { - zkManager.delete(appZNode); + zkManager.delete(appIdRemovePath); + // Check if we should remove the parent app node as well. + checkRemoveParentAppNode(appIdRemovePath, splitIndex); } catch (Exception e) { String errMsg = "Cannot delete app: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); @@ -730,11 +836,12 @@ public void storeVersion() throws Exception { * * @param appId Application identifier. * @return ApplicationHomeSubCluster identifier. - * @throws Exception If it cannot contact ZooKeeper. + * @throws YarnException If it cannot contact ZooKeeper. */ private ApplicationHomeSubCluster getApplicationHomeSubCluster( - final ApplicationId appId) throws YarnException { - String appZNode = getNodePath(appsZNode, appId.toString()); + final ApplicationId appId) throws YarnException{ + + String appZNode = getLeafAppIdNodePath(appId.toString(), false); ApplicationHomeSubCluster appHomeSubCluster = null; byte[] data = get(appZNode); @@ -761,11 +868,44 @@ private ApplicationHomeSubCluster getApplicationHomeSubCluster( private void storeOrUpdateApplicationHomeSubCluster(final ApplicationId applicationId, final ApplicationHomeSubCluster applicationHomeSubCluster, boolean update) throws YarnException { - String appZNode = getNodePath(appsZNode, applicationId.toString()); - ApplicationHomeSubClusterProto proto = - ((ApplicationHomeSubClusterPBImpl) applicationHomeSubCluster).getProto(); - byte[] data = proto.toByteArray(); - put(appZNode, data, update); + try { + ApplicationHomeSubClusterProto proto = + ((ApplicationHomeSubClusterPBImpl) applicationHomeSubCluster).getProto(); + byte[] data = proto.toByteArray(); + if (update) { + updateApplicationStateInternal(applicationId, data); + } else { + storeApplicationStateInternal(applicationId, data); + } + } catch (Exception e) { + throw new YarnException(e); + } + } + + protected void storeApplicationStateInternal(final ApplicationId applicationId, byte[] data) + throws Exception { + String nodeCreatePath = getLeafAppIdNodePath(applicationId.toString(), true); + LOG.debug("Storing info for app: {} at: {}.", applicationId, nodeCreatePath); + put(nodeCreatePath, data, false); + } + + protected void updateApplicationStateInternal(final ApplicationId applicationId, byte[] data) + throws Exception { + String nodeUpdatePath = getLeafAppIdNodePath(applicationId.toString(), false); + if (!exists(nodeUpdatePath)) { + AppNodeSplitInfo alternatePathInfo = getAlternatePath(applicationId.toString()); + if (alternatePathInfo != null) { + nodeUpdatePath = alternatePathInfo.path; + } else if (appIdNodeSplitIndex != 0) { + // No alternate path exists. Create path as per configured split index. + String rootNode = getSplitAppNodeParent(nodeUpdatePath, appIdNodeSplitIndex); + if (!exists(rootNode)) { + zkManager.create(rootNode); + } + } + } + LOG.debug("Storing final state info for app: {} at: {}.", applicationId, nodeUpdatePath); + put(nodeUpdatePath, data, true); } /** @@ -1674,4 +1814,198 @@ public int incrementCurrentKeyId() { } return keyIdSeqCounter.getCount(); } + + /** + * Get parent app node path based on full path and split index supplied. + * @param appIdPath App id path for which parent needs to be returned. + * @param splitIndex split index. + * @return parent app node path. + */ + private String getSplitAppNodeParent(String appIdPath, int splitIndex) { + // Calculated as string upto index (appIdPath Length - split index - 1). We + // deduct 1 to exclude path separator. + return appIdPath.substring(0, appIdPath.length() - splitIndex - 1); + } + + /** + * Checks if parent app node has no leaf nodes and if it does not have, + * removes it. Called while removing application. + * + * @param appIdPath path of app id to be removed. + * @param splitIndex split index. + * @throws Exception if any problem occurs while performing ZK operation. + */ + private void checkRemoveParentAppNode(String appIdPath, int splitIndex) + throws Exception { + if (splitIndex == 0) { + return; + } + + String parentAppNode = getSplitAppNodeParent(appIdPath, splitIndex); + List children; + try { + children = getChildren(parentAppNode); + } catch (KeeperException.NoNodeException ke) { + // It should be fine to swallow this exception as the parent app node we + // intend to delete is already deleted. + LOG.debug("Unable to remove app parent node {} as it does not exist.", + parentAppNode); + return; + } + + // If children==null or children is not empty, we cannot delete the parent path. + if (children == null || !children.isEmpty()) { + return; + } + + // No apps stored under parent path. + try { + zkManager.delete(parentAppNode); + LOG.debug("No leaf app node exists. Removing parent node {}.", parentAppNode); + } catch (KeeperException.NotEmptyException ke) { + // It should be fine to swallow this exception as the parent app node + // has to be deleted only if it has no children. And this node has. + LOG.debug("Unable to remove app parent node {} as it has children.", + parentAppNode); + } + } + + List getChildren(final String path) throws Exception { + return zkManager.getChildren(path); + } + + /** + * Get alternate path for app id if path according to configured split index + * does not exist. We look for path based on all possible split indices. + * @param appId + * @return a {@link AppNodeSplitInfo} object containing the path and split + * index if it exists, null otherwise. + * @throws Exception if any problem occurs while performing ZK operation. + */ + private AppNodeSplitInfo getAlternatePath(String appId) throws Exception { + for (Map.Entry entry : routerAppRootHierarchies.entrySet()) { + // Look for other paths + int splitIndex = entry.getKey(); + if (splitIndex != appIdNodeSplitIndex) { + String alternatePath = + getLeafAppIdNodePath(appId, entry.getValue(), splitIndex, false); + if (exists(alternatePath)) { + return new AppNodeSplitInfo(alternatePath, splitIndex); + } + } + } + return null; + } + + /** + * Returns leaf app node path based on app id and passed split index. If the + * passed flag createParentIfNotExists is true, also creates the parent app + * node if it does not exist. + * @param appId application id. + * @param rootNode app root node based on split index. + * @param appIdNodeSplitIdx split index. + * @param createParentIfNotExists flag which determines if parent app node + * needs to be created(as per split) if it does not exist. + * @return leaf app node path. + * @throws Exception if any problem occurs while performing ZK operation. + */ + private String getLeafAppIdNodePath(String appId, String rootNode, + int appIdNodeSplitIdx, boolean createParentIfNotExists) throws Exception { + if (appIdNodeSplitIdx == 0) { + return getNodePath(rootNode, appId); + } + String nodeName = appId; + int splitIdx = nodeName.length() - appIdNodeSplitIdx; + String rootNodePath = getNodePath(rootNode, nodeName.substring(0, splitIdx)); + if (createParentIfNotExists && !exists(rootNodePath)) { + try { + zkManager.create(rootNodePath); + } catch (KeeperException.NodeExistsException e) { + LOG.debug("Unable to create app parent node {} as it already exists.", rootNodePath); + } + } + return getNodePath(rootNodePath, nodeName.substring(splitIdx)); + } + + /** + * Returns leaf app node path based on app id and configured split index. If + * the passed flag createParentIfNotExists is true, also creates the parent + * app node if it does not exist. + * @param appId application id. + * @param createParentIfNotExists flag which determines if parent app node + * needs to be created(as per split) if it does not exist. + * @return leaf app node path. + * @throws YarnException if any problem occurs while performing ZK operation. + */ + private String getLeafAppIdNodePath(String appId, + boolean createParentIfNotExists) throws YarnException { + try { + String rootNode = routerAppRootHierarchies.get(appIdNodeSplitIndex); + return getLeafAppIdNodePath(appId, rootNode, appIdNodeSplitIndex, createParentIfNotExists); + } catch (Exception e) { + throw new YarnException(e); + } + } + + private ApplicationHomeSubCluster loadRouterAppStateFromAppNode(String appNodePath) + throws Exception { + byte[] data = get(appNodePath); + LOG.debug("Loading application from znode: {}", appNodePath); + ApplicationHomeSubCluster appHomeSubCluster = null; + + if (data == null) { + return appHomeSubCluster; + } + + try { + appHomeSubCluster = new ApplicationHomeSubClusterPBImpl( + ApplicationHomeSubClusterProto.parseFrom(data)); + } catch (InvalidProtocolBufferException e) { + String errMsg = "Cannot parse application at " + appNodePath; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + + return appHomeSubCluster; + } + + private List loadRouterApplications() throws Exception { + List applicationHomeSubClusters = new ArrayList<>(); + for (int splitIndex = 0; splitIndex <= 4; splitIndex++) { + String appRoot = routerAppRootHierarchies.get(splitIndex); + if (appRoot == null) { + continue; + } + List childNodes = getChildren(appRoot); + boolean appNodeFound = false; + for (String childNodeName : childNodes) { + if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) { + appNodeFound = true; + if (splitIndex == 0) { + ApplicationHomeSubCluster applicationHomeSubCluster = + loadRouterAppStateFromAppNode(getNodePath(appRoot, childNodeName)); + applicationHomeSubClusters.add(applicationHomeSubCluster); + } else { + // If AppId Node is partitioned. + String parentNodePath = getNodePath(appRoot, childNodeName); + List leafNodes = getChildren(parentNodePath); + for (String leafNodeName : leafNodes) { + ApplicationHomeSubCluster applicationHomeSubCluster = + loadRouterAppStateFromAppNode(getNodePath(parentNodePath, leafNodeName)); + applicationHomeSubClusters.add(applicationHomeSubCluster); + } + } + } else if (!childNodeName.equals(ROUTER_APP_ROOT_HIERARCHIES)){ + LOG.debug("Unknown child node with name {} under {}.", childNodeName, appRoot); + } + } + if (splitIndex != appIdNodeSplitIndex && !appNodeFound) { + // If no loaded app exists for a particular split index and the split + // index for which apps are being loaded is not the one configured, then + // we do not need to keep track of this hierarchy for storing/updating/ + // removing app/app attempt znodes. + routerAppRootHierarchies.remove(splitIndex); + } + } + return applicationHomeSubClusters; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/yarn-site.xml new file mode 100644 index 0000000000000..1f283cf259aff --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/yarn-site.xml @@ -0,0 +1,22 @@ + + + + + + + yarn.resourcemanager.zk-appid-node.split-index + 1 + + From 85af6c3a2850ffa0d3216bb62c19c55ab6e4dba3 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Tue, 10 Oct 2023 10:20:07 +0800 Subject: [PATCH 083/155] HDFS-17217. Add lifeline RPC start up log when NameNode#startCommonServices (#6154). Contributed by Haiyang Hu. Reviewed-by: Shilun Fan Reviewed-by: Tao Li Signed-off-by: Shuyan Zhang --- .../org/apache/hadoop/hdfs/server/namenode/NameNode.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index a4618c000778a..bee7db315de5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -1010,10 +1010,12 @@ private void startCommonServices(Configuration conf) throws IOException { LOG.warn("ServicePlugin " + p + " could not be started", t); } } - LOG.info(getRole() + " RPC up at: " + getNameNodeAddress()); + LOG.info("{} RPC up at: {}.", getRole(), getNameNodeAddress()); if (rpcServer.getServiceRpcAddress() != null) { - LOG.info(getRole() + " service RPC up at: " - + rpcServer.getServiceRpcAddress()); + LOG.info("{} service RPC up at: {}.", getRole(), rpcServer.getServiceRpcAddress()); + } + if (rpcServer.getLifelineRpcAddress() != null) { + LOG.info("{} lifeline RPC up at: {}.", getRole(), rpcServer.getLifelineRpcAddress()); } } From 73eccd6d7c5751dfdb6460a6452683cdea539d2e Mon Sep 17 00:00:00 2001 From: Steve Vaughan Date: Tue, 10 Oct 2023 16:51:46 -0400 Subject: [PATCH 084/155] HDFS-16740. Mini cluster test flakiness (#4835) --- .../hadoop/hdfs/AdminStatesBaseTest.java | 12 +- .../apache/hadoop/hdfs/MiniDFSCluster.java | 5 +- .../hdfs/TestDFSStripedInputStream.java | 8 +- .../hdfs/TestDecommissionWithStriped.java | 27 +++-- .../hdfs/TestReconstructStripedFile.java | 8 +- .../hadoop/hdfs/TestRollingUpgrade.java | 80 ++++++------- .../hdfs/qjournal/MiniJournalCluster.java | 11 +- .../hdfs/qjournal/MiniQJMHACluster.java | 8 ++ .../server/datanode/TestBPOfferService.java | 21 ++-- .../datanode/TestDataNodeRollingUpgrade.java | 7 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 105 +++++++----------- .../server/namenode/TestNameNodeMXBean.java | 37 +++--- .../namenode/ha/TestRetryCacheWithHA.java | 9 +- 13 files changed, 172 insertions(+), 166 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java index 82a983004dfe7..bf25958b69dc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -29,6 +30,8 @@ import java.util.Random; import org.apache.hadoop.util.Lists; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -64,6 +67,9 @@ public class AdminStatesBaseTest { final private Random myrand = new Random(); + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + private HostsFileWriter hostsFileWriter; private Configuration conf; private MiniDFSCluster cluster = null; @@ -396,7 +402,7 @@ protected void startCluster(int numNameNodes, int numDatanodes, protected void startCluster(int numNameNodes, int numDatanodes, boolean setupHostsFile, long[] nodesCapacity, boolean checkDataNodeHostConfig, boolean federation) throws IOException { - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf) + MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDatanodes); if (federation) { builder.nnTopology( @@ -431,7 +437,7 @@ protected void startSimpleCluster(int numNameNodes, int numDatanodes) protected void startSimpleHACluster(int numDatanodes) throws IOException { - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes( numDatanodes).build(); cluster.transitionToActive(0); @@ -458,6 +464,6 @@ static public void cleanupFile(FileSystem fileSys, Path name) throws IOException { assertTrue(fileSys.exists(name)); fileSys.delete(name, true); - assertTrue(!fileSys.exists(name)); + assertFalse(fileSys.exists(name)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index f5638871d404d..ca8ae04bbf753 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -248,7 +248,10 @@ public Builder(Configuration conf, File basedir) { "MiniDFSCluster base directory cannot be null"); } String cdir = conf.get(HDFS_MINIDFS_BASEDIR); - if (cdir != null) { + // There are tests which restart server, and we want to allow them to restart with the same + // configuration. Although it is an error if the base directory is already set, we'll ignore + // cases where the base directory is the same. + if (cdir != null && !cdir.equals(basedir.getAbsolutePath())) { throw new IllegalArgumentException( "MiniDFSCluster base directory already defined (" + cdir + ")"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 2369d42482a41..9d3bdbf4e9520 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -86,6 +87,9 @@ public class TestDFSStripedInputStream { @Rule public Timeout globalTimeout = new Timeout(300000); + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + public ErasureCodingPolicy getEcPolicy() { return StripedFileTestUtil.getDefaultECPolicy(); } @@ -110,14 +114,12 @@ public void setup() throws IOException { CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, NativeRSRawErasureCoderFactory.CODER_NAME); } - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - GenericTestUtils.getRandomizedTempPath()); SimulatedFSDataset.setFactory(conf); startUp(); } private void startUp() throws IOException { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes( + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes( dataBlocks + parityBlocks).build(); cluster.waitActive(); for (DataNode dn : cluster.getDataNodes()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java index 206f75eae7035..1de8fc17ee802 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; @@ -62,11 +63,12 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.PathUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,7 +92,10 @@ public class TestDecommissionWithStriped { private Path decommissionDir; private Path hostsFile; private Path excludeFile; - private FileSystem localFileSys; + private LocalFileSystem localFileSys; + + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); private Configuration conf; private MiniDFSCluster cluster; @@ -118,9 +123,9 @@ public void setup() throws IOException { conf = createConfiguration(); // Set up the hosts/exclude files. localFileSys = FileSystem.getLocal(conf); + localFileSys.setWorkingDirectory(new Path(baseDir.getRoot().getPath())); Path workingDir = localFileSys.getWorkingDirectory(); - decommissionDir = new Path(workingDir, - PathUtils.getTestDirName(getClass()) + "/work-dir/decommission"); + decommissionDir = new Path(workingDir, "work-dir/decommission"); hostsFile = new Path(decommissionDir, "hosts"); excludeFile = new Path(decommissionDir, "exclude"); writeConfigFile(hostsFile, null); @@ -582,16 +587,14 @@ private void writeConfigFile(Path name, List nodes) localFileSys.delete(name, true); } - FSDataOutputStream stm = localFileSys.create(name); - - if (nodes != null) { - for (Iterator it = nodes.iterator(); it.hasNext();) { - String node = it.next(); - stm.writeBytes(node); - stm.writeBytes("\n"); + try (FSDataOutputStream stm = localFileSys.create(name)) { + if (nodes != null) { + for (String node: nodes) { + stm.writeBytes(node); + stm.writeBytes("\n"); + } } } - stm.close(); } private void cleanupFile(FileSystem fileSys, Path name) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index cda714b2cb3fb..2f734c62f420b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.erasurecode.ErasureCodingTestHelper; import org.apache.hadoop.io.ElasticByteBufferPool; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -96,6 +98,9 @@ enum ReconstructionType { Any } + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + private Configuration conf; private MiniDFSCluster cluster; private DistributedFileSystem fs; @@ -150,8 +155,7 @@ public void setup() throws IOException { getPendingTimeout()); conf.setBoolean(DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_VALIDATION_KEY, isValidationEnabled()); - File basedir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, basedir).numDataNodes(dnNum) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(dnNum) .build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index 495195a0c5914..cd05994b5bdcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -86,28 +86,7 @@ public static void runCmd(DFSAdmin dfsadmin, boolean success, } @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - /** - * Create a default HDFS configuration which has test-specific data directories. This is - * intended to protect against interactions between test runs that might corrupt results. Each - * test run's data is automatically cleaned-up by JUnit. - * - * @return a default configuration with test-specific data directories - */ - public Configuration getHdfsConfiguration() throws IOException { - Configuration conf = new HdfsConfiguration(); - - // Override the file system locations with test-specific temporary folders - conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, - folder.newFolder("dfs/name").toString()); - conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY, - folder.newFolder("dfs/namesecondary").toString()); - conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, - folder.newFolder("dfs/data").toString()); - - return conf; - } + public TemporaryFolder baseDir = new TemporaryFolder(); /** * Test DFSAdmin Upgrade Command. @@ -115,8 +94,10 @@ public Configuration getHdfsConfiguration() throws IOException { @Test public void testDFSAdminRollingUpgradeCommands() throws Exception { // start a cluster - final Configuration conf = getHdfsConfiguration(); - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) { + final Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) + .numDataNodes(0) + .build()) { cluster.waitActive(); final Path foo = new Path("/foo"); @@ -197,14 +178,15 @@ public void testRollingUpgradeWithQJM() throws Exception { LOG.info("nn1Dir=" + nn1Dir); LOG.info("nn2Dir=" + nn2Dir); - final Configuration conf = getHdfsConfiguration(); - try (MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build()) { + final Configuration conf = new HdfsConfiguration(); + try (MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf, baseDir.getRoot()) + .build()) { mjc.waitActive(); setConf(conf, nn1Dir, mjc); { // Start the cluster once to generate the dfs dirs - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(0) .manageNameDfsDirs(false) .checkExitOnShutdown(false) @@ -224,7 +206,7 @@ public void testRollingUpgradeWithQJM() throws Exception { new Path(nn2Dir.getAbsolutePath()), false, conf); // Start the cluster again - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(0) .format(false) .manageNameDfsDirs(false) @@ -335,8 +317,10 @@ private static void checkMxBean() throws Exception { @Test public void testRollback() throws Exception { // start a cluster - final Configuration conf = getHdfsConfiguration(); - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) { + final Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) + .numDataNodes(1) + .build()) { cluster.waitActive(); final Path foo = new Path("/foo"); @@ -429,8 +413,10 @@ private static void rollbackRollingUpgrade(Path foo, Path bar, @Test public void testDFSAdminDatanodeUpgradeControlCommands() throws Exception { // start a cluster - final Configuration conf = getHdfsConfiguration(); - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) { + final Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) + .numDataNodes(1) + .build()) { cluster.waitActive(); final DFSAdmin dfsadmin = new DFSAdmin(conf); DataNode dn = cluster.getDataNodes().get(0); @@ -480,13 +466,14 @@ private void testFinalize(int nnCount) throws Exception { private void testFinalize(int nnCount, boolean skipImageDeltaCheck) throws Exception { - final Configuration conf = getHdfsConfiguration(); + final Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); try { - cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build(); + cluster = new MiniQJMHACluster.Builder(conf, baseDir.getRoot()) + .setNumNameNodes(nnCount).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); @@ -546,8 +533,10 @@ public void testQueryWithMultipleNN() throws Exception { } private void testQuery(int nnCount) throws Exception{ - final Configuration conf = getHdfsConfiguration(); - try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build()) { + final Configuration conf = new HdfsConfiguration(); + try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf, baseDir.getRoot()) + .setNumNameNodes(nnCount) + .build()) { MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); @@ -582,8 +571,10 @@ private void testQuery(int nnCount) throws Exception{ @Test (timeout = 300000) public void testQueryAfterRestart() throws IOException, InterruptedException { - final Configuration conf = getHdfsConfiguration(); - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) { + final Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) + .numDataNodes(0) + .build()) { cluster.waitActive(); DistributedFileSystem dfs = cluster.getFileSystem(); @@ -612,14 +603,14 @@ public void testCheckpointWithMultipleNN() throws IOException, InterruptedExcept @Test(timeout = 60000) public void testRollBackImage() throws Exception { - final Configuration conf = getHdfsConfiguration(); + final Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 2); MiniQJMHACluster cluster = null; CheckpointFaultInjector old = CheckpointFaultInjector.getInstance(); try { - cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(2).build(); + cluster = new MiniQJMHACluster.Builder(conf, baseDir.getRoot()).setNumNameNodes(2).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); @@ -657,13 +648,14 @@ public void duringUploadInProgess() } public void testCheckpoint(int nnCount) throws IOException, InterruptedException { - final Configuration conf = getHdfsConfiguration(); + final Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1); final Path foo = new Path("/foo"); - try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount) + try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf, baseDir.getRoot()) + .setNumNameNodes(nnCount) .build()) { MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); @@ -767,8 +759,8 @@ public void testCheckpointWithSNN() throws Exception { SecondaryNameNode snn = null; try { - Configuration conf = getHdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).build(); + Configuration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build(); cluster.waitActive(); conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java index 1c43b39159a99..c5b8b79e3ca87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.qjournal; import static org.apache.hadoop.hdfs.qjournal.QJMTestUtil.FAKE_NSINFO; -import static org.junit.Assert.fail; import java.io.Closeable; import java.io.File; @@ -64,7 +63,12 @@ public static class Builder { public Builder(Configuration conf) { this.conf = conf; } - + + public Builder(Configuration conf, File baseDir) { + this.conf = conf; + baseDir(baseDir.toString()); + } + public Builder baseDir(String d) { this.baseDir = d; return this; @@ -289,7 +293,8 @@ public void waitActive() throws IOException { } }, 50, 3000); } catch (TimeoutException e) { - fail("Time out while waiting for journal node " + index + " to start."); + throw new AssertionError("Time out while waiting for journal node " + index + + " to start."); } catch (InterruptedException ite) { LOG.warn("Thread interrupted when waiting for node start", ite); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index 0791e0ace1c0a..dd0ccd729efba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import java.io.File; import java.io.IOException; import java.net.BindException; import java.net.URI; @@ -60,6 +61,13 @@ public Builder(Configuration conf) { this.dfsBuilder = new MiniDFSCluster.Builder(conf).numDataNodes(0); } + public Builder(Configuration conf, File baseDir) { + this.conf = conf; + // most QJMHACluster tests don't need DataNodes, so we'll make + // this the default + this.dfsBuilder = new MiniDFSCluster.Builder(conf, baseDir).numDataNodes(0); + } + public MiniDFSCluster.Builder getDfsBuilder() { return dfsBuilder; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index 9d4b0db0804f7..65855427d7253 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -89,7 +89,9 @@ import org.apache.hadoop.util.Time; import org.junit.Before; import org.junit.After; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -120,6 +122,9 @@ public class TestBPOfferService { GenericTestUtils.setLogLevel(DataNode.LOG, Level.TRACE); } + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + private DatanodeProtocolClientSideTranslatorPB mockNN1; private DatanodeProtocolClientSideTranslatorPB mockNN2; private final NNHAStatusHeartbeat[] mockHaStatuses = @@ -1254,8 +1259,7 @@ public void testSetIsSlownode() throws Exception { @Test(timeout = 15000) public void testCommandProcessingThread() throws Exception { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - try { + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build()) { List datanodes = cluster.getDataNodes(); assertEquals(datanodes.size(), 1); DataNode datanode = datanodes.get(0); @@ -1272,19 +1276,14 @@ public void testCommandProcessingThread() throws Exception { // Check new metric result about processedCommandsOp. // One command send back to DataNode here is #FinalizeCommand. assertCounter("ProcessedCommandsOpNumOps", 1L, mrb); - } finally { - if (cluster != null) { - cluster.shutdown(); - } } } @Test(timeout = 5000) public void testCommandProcessingThreadExit() throws Exception { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). - numDataNodes(1).build(); - try { + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()). + numDataNodes(1).build()) { List datanodes = cluster.getDataNodes(); DataNode dataNode = datanodes.get(0); List allBpOs = dataNode.getAllBpOs(); @@ -1294,10 +1293,6 @@ public void testCommandProcessingThreadExit() throws Exception { // Stop and wait util actor exit. actor.stopCommandProcessingThread(); GenericTestUtils.waitFor(() -> !actor.isAlive(), 100, 3000); - } finally { - if (cluster != null) { - cluster.shutdown(); - } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java index ae00dbe6f8cf0..fdde4cabe6bb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java @@ -27,6 +27,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -65,6 +67,9 @@ public class TestDataNodeRollingUpgrade { private static final long FILE_SIZE = BLOCK_SIZE; private static final long SEED = 0x1BADF00DL; + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + Configuration conf; MiniDFSCluster cluster = null; DistributedFileSystem fs = null; @@ -75,7 +80,7 @@ public class TestDataNodeRollingUpgrade { private void startCluster() throws IOException { conf = new HdfsConfiguration(); conf.setInt("dfs.blocksize", 1024*1024); - cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build(); + cluster = new Builder(conf, baseDir.getRoot()).numDataNodes(REPL_FACTOR).build(); cluster.waitActive(); fs = cluster.getFileSystem(); nn = cluster.getNameNode(0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 7d6e7980be5c1..1782a4644bc91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY; -import static org.apache.hadoop.hdfs.MiniDFSCluster.HDFS_MINIDFS_BASEDIR; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -123,7 +122,9 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.slf4j.LoggerFactory; /** @@ -176,6 +177,9 @@ public static String runFsck(Configuration conf, int expectedErrCode, return bStream.toString(); } + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + private MiniDFSCluster cluster = null; private Configuration conf = null; @@ -217,8 +221,7 @@ public void testFsck() throws Exception { conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(4).build(); fs = cluster.getFileSystem(); final String fileName = "/srcdat"; @@ -235,7 +238,7 @@ public void testFsck() throws Exception { shutdownCluster(); // restart the cluster; bring up namenode but not the data nodes - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(0).format(false).build(); outStr = runFsck(conf, 1, true, "/"); // expect the result is corrupt @@ -282,8 +285,7 @@ public void testFsckNonExistent() throws Exception { setNumFiles(20).build(); FileSystem fs = null; conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(4).build(); fs = cluster.getFileSystem(); util.createFiles(fs, "/srcdat"); @@ -302,8 +304,7 @@ public void testFsckPermission() throws Exception { conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); // Create a cluster with the current user, write some files - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(4).build(); final MiniDFSCluster c2 = cluster; final String dir = "/dfsck"; @@ -350,8 +351,7 @@ public void testFsckMove() throws Exception { DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, (5 * dfsBlockSize) + (dfsBlockSize - 1), 5 * dfsBlockSize); FileSystem fs = null; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDatanodes).build(); String topDir = "/srcdat"; fs = cluster.getFileSystem(); @@ -558,8 +558,7 @@ public void testFsckMoveAndDelete() throws Exception { FileSystem fs = null; conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(4).build(); String topDir = "/srcdat"; fs = cluster.getFileSystem(); @@ -624,8 +623,7 @@ public void testFsckOpenFiles() throws Exception { setNumFiles(4).build(); FileSystem fs = null; conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(4).build(); String topDir = "/srcdat"; String randomString = "HADOOP "; @@ -679,8 +677,7 @@ public void testFsckOpenECFiles() throws Exception { final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits(); int blockSize = 2 * cellSize; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes( + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes( numAllUnits + 1).build(); String topDir = "/myDir"; cluster.waitActive(); @@ -771,8 +768,7 @@ public void testCorruptBlock() throws Exception { String outStr = null; short factor = 1; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(1).build(); cluster.waitActive(); fs = cluster.getFileSystem(); @@ -844,8 +840,7 @@ public void testUnderMinReplicatedBlock() throws Exception { Random random = new Random(); String outStr = null; short factor = 1; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(2).build(); cluster.waitActive(); fs = cluster.getFileSystem(); @@ -918,8 +913,7 @@ public void testFsckReplicaDetails() throws Exception { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); DistributedFileSystem dfs; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn).hosts(hosts).racks(racks).build(); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); @@ -1065,8 +1059,7 @@ public Boolean get() { @Test public void testFsckError() throws Exception { // bring up a one-node cluster - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build(); String fileName = "/test.txt"; Path filePath = new Path(fileName); FileSystem fs = cluster.getFileSystem(); @@ -1098,8 +1091,7 @@ public void testFsckListCorruptFilesBlocks() throws Exception { conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); FileSystem fs = null; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build(); cluster.waitActive(); fs = cluster.getFileSystem(); DFSTestUtil util = new DFSTestUtil.Builder(). @@ -1158,8 +1150,7 @@ public void testFsckListCorruptFilesBlocks() throws Exception { @Test public void testToCheckTheFsckCommandOnIllegalArguments() throws Exception { // bring up a one-node cluster - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build(); String fileName = "/test.txt"; Path filePath = new Path(fileName); FileSystem fs = cluster.getFileSystem(); @@ -1203,8 +1194,7 @@ public void testFsckMissingReplicas() throws IOException { DistributedFileSystem dfs = null; // Startup a minicluster - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numReplicas).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); @@ -1265,8 +1255,7 @@ public void testFsckMisPlacedReplicas() throws IOException { DistributedFileSystem dfs = null; // Startup a minicluster - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); @@ -1374,8 +1363,7 @@ public void testFsckSymlink() throws Exception { conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(4).build(); fs = cluster.getFileSystem(); final String fileName = "/srcdat"; @@ -1402,8 +1390,7 @@ public void testFsckSymlink() throws Exception { */ @Test public void testFsckForSnapshotFiles() throws Exception { - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(1) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(1) .build(); String runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots", "-files"); @@ -1438,8 +1425,7 @@ public void testBlockIdCK() throws Exception { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); DistributedFileSystem dfs = null; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -1493,8 +1479,7 @@ public void testBlockIdCKDecommission() throws Exception { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); DistributedFileSystem dfs; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -1578,8 +1563,7 @@ public void testBlockIdCKMaintenance() throws Exception { replFactor); DistributedFileSystem dfs; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn) .hosts(hosts) .racks(racks) @@ -1785,8 +1769,7 @@ public void testBlockIdCKCorruption() throws Exception { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); DistributedFileSystem dfs = null; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -1899,8 +1882,7 @@ private void writeFile(final DistributedFileSystem dfs, */ @Test public void testStoragePoliciesCK() throws Exception { - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(3) .storageTypes( new StorageType[] {StorageType.DISK, StorageType.ARCHIVE}) @@ -1943,8 +1925,7 @@ public void testFsckWithDecommissionedReplicas() throws Exception { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); DistributedFileSystem dfs; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -2025,8 +2006,7 @@ public void testFsckWithMaintenanceReplicas() throws Exception { replFactor); DistributedFileSystem dfs; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(numDn) .hosts(hosts) .racks(racks) @@ -2134,8 +2114,7 @@ public void testECFsck() throws Exception { int parityBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int totalSize = dataBlocks + parityBlocks; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( @@ -2170,7 +2149,7 @@ public void testECFsck() throws Exception { shutdownCluster(); // restart the cluster; bring up namenode but not the data nodes - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(0).format(false).build(); outStr = runFsck(conf, 1, true, "/", "-files", "-blocks"); // expect the result is corrupt @@ -2202,8 +2181,7 @@ public void testFsckListCorruptSnapshotFiles() throws Exception { int numFiles = 3; int numSnapshots = 0; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build(); cluster.waitActive(); hdfs = cluster.getFileSystem(); DFSTestUtil util = new DFSTestUtil.Builder(). @@ -2297,8 +2275,7 @@ public void testFsckMoveAfterCorruption() throws Exception { conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replication); - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).build(); DistributedFileSystem dfs = cluster.getFileSystem(); cluster.waitActive(); @@ -2393,7 +2370,6 @@ private void testUpgradeDomain(boolean defineUpgradeDomain, HostsFileWriter hostsFileWriter = new HostsFileWriter(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor); - conf.set(HDFS_MINIDFS_BASEDIR, GenericTestUtils.getRandomizedTempPath()); if (defineUpgradeDomain) { conf.setClass(DFSConfigKeys.DFS_NAMENODE_HOSTS_PROVIDER_CLASSNAME_KEY, CombinedHostFileManager.class, HostConfigManager.class); @@ -2401,7 +2377,7 @@ private void testUpgradeDomain(boolean defineUpgradeDomain, } DistributedFileSystem dfs; - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDN). + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(numDN). hosts(hosts).racks(racks).build(); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); @@ -2445,8 +2421,7 @@ public void testFsckCorruptECFile() throws Exception { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( @@ -2517,8 +2492,7 @@ public void testFsckMissingECFile() throws Exception { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; - File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); - cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( @@ -2641,8 +2615,7 @@ public Boolean get() { @Test(timeout = 300000) public void testFsckCorruptWhenOneReplicaIsCorrupt() throws Exception { - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, - new File(GenericTestUtils.getRandomizedTempPath())) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2) .build()) { cluster.waitActive(); @@ -2671,7 +2644,7 @@ public void testFsckCorruptWhenOneReplicaIsCorrupt() @Test public void testFsckNonPrivilegedListCorrupt() throws Exception { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(4).build(); UserGroupInformation ugi = UserGroupInformation.createUserForTesting("systest", new String[]{""}); ugi.doAs(new PrivilegedExceptionAction() { @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index d670025bf5069..bc46c4857a155 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -57,8 +57,10 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; import org.eclipse.jetty.util.ajax.JSON; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,6 +99,9 @@ public class TestNameNodeMXBean { */ private static final double DELTA = 0.000001; + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + static { NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); } @@ -112,7 +117,7 @@ public void testNameNodeMXBeanInfo() throws Exception { MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(4).build(); cluster.waitActive(); // Set upgrade domain on the first DN. @@ -139,7 +144,7 @@ public void testNameNodeMXBeanInfo() throws Exception { String clusterId = (String) mbs.getAttribute(mxbeanName, "ClusterId"); assertEquals(fsn.getClusterId(), clusterId); // get attribute "BlockPoolId" - String blockpoolId = (String) mbs.getAttribute(mxbeanName, + String blockpoolId = (String) mbs.getAttribute(mxbeanName, "BlockPoolId"); assertEquals(fsn.getBlockPoolId(), blockpoolId); // get attribute "Version" @@ -270,7 +275,7 @@ public void testNameNodeMXBeanInfo() throws Exception { assertEquals(0, FileUtil.chmod( new File(failedNameDir, "current").getAbsolutePath(), "000")); cluster.getNameNodeRpc().rollEditLog(); - + nameDirStatuses = (String) (mbs.getAttribute(mxbeanName, "NameDirStatuses")); statusMap = (Map>) JSON.parse(nameDirStatuses); @@ -313,7 +318,7 @@ public void testLastContactTime() throws Exception { hostsFileWriter.initialize(conf, "temp/TestNameNodeMXBean"); try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(3).build(); cluster.waitActive(); FSNamesystem fsn = cluster.getNameNode().namesystem; @@ -366,7 +371,7 @@ public void testDecommissioningNodes() throws Exception { hostsFileWriter.initialize(conf, "temp/TestNameNodeMXBean"); try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(3).build(); cluster.waitActive(); FSNamesystem fsn = cluster.getNameNode().namesystem; @@ -469,7 +474,7 @@ public void testInServiceNodes() throws Exception { hostsFileWriter.initialize(conf, "temp/TestInServiceNodes"); try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(3).build(); cluster.waitActive(); final FSNamesystem fsn = cluster.getNameNode().namesystem; @@ -568,7 +573,7 @@ public void testMaintenanceNodes() throws Exception { hostsFileWriter.initialize(conf, "temp/TestNameNodeMXBean"); try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(3).build(); cluster.waitActive(); FSNamesystem fsn = cluster.getNameNode().namesystem; @@ -659,7 +664,7 @@ public void testTopUsers() throws Exception { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(0).build(); cluster.waitActive(); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); ObjectName mxbeanNameFsns = new ObjectName( @@ -675,7 +680,7 @@ public void testTopUsers() throws Exception { (String) (mbs.getAttribute(mxbeanNameFsns, "TopUserOpCounts")); ObjectMapper mapper = new ObjectMapper(); Map map = mapper.readValue(topUsers, Map.class); - assertTrue("Could not find map key timestamp", + assertTrue("Could not find map key timestamp", map.containsKey("timestamp")); assertTrue("Could not find map key windows", map.containsKey("windows")); List>>> windows = @@ -715,7 +720,7 @@ public void testTopUsersDisabled() throws Exception { conf.setBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY, false); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(0).build(); cluster.waitActive(); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); ObjectName mxbeanNameFsns = new ObjectName( @@ -744,7 +749,7 @@ public void testTopUsersNoPeriods() throws Exception { conf.set(DFSConfigKeys.NNTOP_WINDOWS_MINUTES_KEY, ""); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(0).build(); cluster.waitActive(); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); ObjectName mxbeanNameFsns = new ObjectName( @@ -771,7 +776,7 @@ public void testQueueLength() throws Exception { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()).numDataNodes(0).build(); cluster.waitActive(); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); ObjectName mxbeanNameFs = @@ -801,7 +806,7 @@ public void testNNDirectorySize() throws Exception{ .addNN( new MiniDFSNNTopology.NNConf("nn2").setIpcPort(ports[1]))); - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .nnTopology(topology).numDataNodes(0) .build(); break; @@ -863,7 +868,7 @@ public void testEnabledEcPoliciesMetric() throws Exception { int dataBlocks = defaultPolicy.getNumDataUnits(); int parityBlocks = defaultPolicy.getNumParityUnits(); int totalSize = dataBlocks + parityBlocks; - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); @@ -903,7 +908,7 @@ public void testVerifyMissingBlockGroupsMetrics() throws Exception { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( @@ -1046,7 +1051,7 @@ public void testTotalBlocksMetrics() throws Exception { int blockSize = stripesPerBlock * cellSize; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .nnTopology(MiniDFSNNTopology.simpleHAFederatedTopology(1)). numDataNodes(totalSize).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java index e3e934b5e3776..8f8dd59a1fbfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java @@ -38,6 +38,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -106,7 +108,10 @@ public class TestRetryCacheWithHA { defaultEcPolicy.getNumParityUnits() + 1); private static final int CHECKTIMES = 10; private static final int ResponseSize = 3; - + + @Rule + public TemporaryFolder baseDir = new TemporaryFolder(); + private MiniDFSCluster cluster; private DistributedFileSystem dfs; private final Configuration conf = new HdfsConfiguration(); @@ -144,7 +149,7 @@ public void setup() throws Exception { conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES, ResponseSize); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); - cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf, baseDir.getRoot()) .nnTopology(MiniDFSNNTopology.simpleHATopology()) .numDataNodes(DataNodes).build(); cluster.waitActive(); From 732d4e72a67b4315a8ac1d210e24606e46521f2a Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 11 Oct 2023 18:50:37 +0100 Subject: [PATCH 085/155] HADOOP-18929. Exclude commons-compress module-info.class (#6170) Contributed By: PJ Fanning --- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 6 ++++++ hadoop-client-modules/hadoop-client-runtime/pom.xml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index c772a6ed684dd..10c6d41dd6cce 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -759,6 +759,12 @@ META-INF/versions/9/module-info.class + + org.apache.commons:commons-compress + + META-INF/versions/9/module-info.class + + diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index d5185f0fffc41..60790d932edaf 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -250,6 +250,12 @@ META-INF/versions/9/module-info.class + + org.apache.commons:commons-compress + + META-INF/versions/9/module-info.class + + From bd28ba385a5e3163a8ca5d60f322b8c382461b87 Mon Sep 17 00:00:00 2001 From: jchanggg Date: Thu, 12 Oct 2023 04:13:44 -0700 Subject: [PATCH 086/155] YARN-11588. [Federation] Fix uncleaned threads in yarn router thread pool executor (#6159) Contributed by Jeffrey Chang. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../server/router/clientrm/FederationClientInterceptor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 63a57394174ef..9c3f9971d8c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -230,6 +230,9 @@ public void init(String userName) { this.executorService = new ThreadPoolExecutor(numMinThreads, numMaxThreads, keepAliveTime, TimeUnit.MILLISECONDS, workQueue, threadFactory); + // Adding this line so that unused user threads will exit and be cleaned up if idle for too long + this.executorService.allowCoreThreadTimeOut(true); + final Configuration conf = this.getConf(); try { From 5c22934d903829d2bb0238739369d3975b8759e4 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Thu, 12 Oct 2023 11:21:26 -0400 Subject: [PATCH 087/155] HADOOP-18922. Race condition in ZKDelegationTokenSecretManager creating znode (#6150). Contributed by Kevin Risden. Signed-off-by: He Xiaoqiao --- .../ZKDelegationTokenSecretManager.java | 8 ++- .../TestZKDelegationTokenSecretManager.java | 54 +++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index 2524929853cc7..925bc030c2cd8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -60,7 +60,6 @@ import org.apache.zookeeper.client.ZKClientConfig; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -270,10 +269,9 @@ public void startThreads() throws IOException { CuratorFramework nullNsFw = zkClient.usingNamespace(null); try { String nameSpace = "/" + zkClient.getNamespace(); - Stat stat = nullNsFw.checkExists().forPath(nameSpace); - if (stat == null) { - nullNsFw.create().creatingParentContainersIfNeeded().forPath(nameSpace); - } + nullNsFw.create().creatingParentContainersIfNeeded().forPath(nameSpace); + } catch (KeeperException.NodeExistsException ignore) { + // We don't care if the znode already exists } catch (Exception e) { throw new IOException("Could not create namespace", e); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java index 469d87ab30c6f..2312af3c79dfa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java @@ -20,8 +20,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.apache.curator.RetryPolicy; @@ -585,4 +590,53 @@ public void testCreateNameSpaceRepeatedly() throws Exception { "KeeperErrorCode = NodeExists for "+workingPath, () -> createModeStat.forPath(workingPath)); } + + @Test + public void testMultipleInit() throws Exception { + + String connectString = zkServer.getConnectString(); + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + Configuration conf = getSecretConf(connectString); + CuratorFramework curatorFramework = + CuratorFrameworkFactory.builder() + .connectString(connectString) + .retryPolicy(retryPolicy) + .build(); + curatorFramework.start(); + ZKDelegationTokenSecretManager.setCurator(curatorFramework); + + DelegationTokenManager tm1 = new DelegationTokenManager(conf, new Text("foo")); + DelegationTokenManager tm2 = new DelegationTokenManager(conf, new Text("bar")); + // When the init method is called, + // the ZKDelegationTokenSecretManager#startThread method will be called, + // and the creatingParentContainersIfNeeded will be called to create the nameSpace. + ExecutorService executorService = Executors.newFixedThreadPool(2); + + Callable tm1Callable = () -> { + tm1.init(); + return true; + }; + Callable tm2Callable = () -> { + tm2.init(); + return true; + }; + List> futures = executorService.invokeAll( + Arrays.asList(tm1Callable, tm2Callable)); + for(Future future : futures) { + Assert.assertTrue(future.get()); + } + executorService.shutdownNow(); + Assert.assertTrue(executorService.awaitTermination(1, TimeUnit.SECONDS)); + tm1.destroy(); + tm2.destroy(); + + String workingPath = "/" + conf.get(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH, + ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) + "/ZKDTSMRoot"; + + // Check if the created NameSpace exists. + Stat stat = curatorFramework.checkExists().forPath(workingPath); + Assert.assertNotNull(stat); + + curatorFramework.close(); + } } From 0ed484ac62c3d6068a26d95292a9bcadc4f7f6bd Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Thu, 12 Oct 2023 23:27:15 +0800 Subject: [PATCH 088/155] HDFS-17208. Add the metrics PendingAsyncDiskOperations in datanode (#6109). Contributed by Haiyang Hu. Reviewed-by: Tao Li Signed-off-by: He Xiaoqiao --- .../impl/FsDatasetAsyncDiskService.java | 6 +++- .../fsdataset/impl/FsDatasetImpl.java | 5 ++++ .../metrics/DataNodeMetricHelper.java | 5 +++- .../datanode/metrics/FSDatasetMBean.java | 5 ++++ .../server/datanode/SimulatedFSDataset.java | 5 ++++ .../extdataset/ExternalDatasetImpl.java | 4 +++ .../fsdataset/impl/TestFsDatasetImpl.java | 29 ++++++++++++++++++- 7 files changed, 56 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index e5eef8e2e23f3..e5b23bb60e516 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -161,7 +161,11 @@ synchronized void removeVolume(String storageId) { executors.remove(storageId); } } - + + /** + * The count of pending and running asynchronous disk operations, + * include deletion of block files and requesting sync_file_range() operations. + */ synchronized long countPendingDeletions() { long count = 0; for (ThreadPoolExecutor exec : executors.values()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index f1115efcc21dc..d3ac60d4a3d39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -3822,5 +3822,10 @@ public long getLastDirScannerFinishTime() { public void setLastDirScannerFinishTime(long time) { this.lastDirScannerFinishTime = time; } + + @Override + public long getPendingAsyncDeletions() { + return asyncDiskService.countPendingDeletions(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java index 65fd92ec78438..24b92df1e4f5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java @@ -73,7 +73,10 @@ public static void getMetrics(MetricsCollector collector, " blocks failed in cache eviction"), beanClass.getNumBlocksFailedToUncache()) .addGauge(Interns.info("LastDirectoryScannerFinishTime", - "Finish time of the last directory scan"), beanClass.getLastDirScannerFinishTime()); + "Finish time of the last directory scan"), beanClass.getLastDirScannerFinishTime()) + .addGauge(Interns.info("PendingAsyncDeletions", + "The count of pending and running asynchronous disk operations"), + beanClass.getPendingAsyncDeletions()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java index 0bfb14e232f23..e559f84bf35fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java @@ -127,4 +127,9 @@ public interface FSDatasetMBean extends MetricsSource { * Returns the last time in milliseconds when the directory scanner successfully ran. */ long getLastDirScannerFinishTime(); + + /** + * Returns the count of pending and running asynchronous disk operations. + */ + long getPendingAsyncDeletions(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 8c75ca9f75208..5421393c9e675 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -944,6 +944,11 @@ public long getLastDirScannerFinishTime() { return 0L; } + @Override + public long getPendingAsyncDeletions() { + return 0; + } + /** * Get metrics from the metrics source * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 1bd42e0bdfbeb..86d4319913301 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -483,4 +483,8 @@ public long getLastDirScannerFinishTime() { return 0L; } + @Override + public long getPendingAsyncDeletions() { + return 0; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index 585c36fa995e8..d49198b53dd56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.io.OutputStream; +import java.lang.management.ManagementFactory; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Random; @@ -131,6 +132,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.management.AttributeNotFoundException; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.ReflectionException; + public class TestFsDatasetImpl { private static final Logger LOG = LoggerFactory.getLogger( @@ -1842,7 +1851,8 @@ public void testTransferAndNativeCopyMetrics() throws IOException { */ @Test public void testAysncDiskServiceDeleteReplica() - throws IOException, InterruptedException, TimeoutException { + throws IOException, InterruptedException, TimeoutException, MalformedObjectNameException, + ReflectionException, AttributeNotFoundException, InstanceNotFoundException, MBeanException { HdfsConfiguration config = new HdfsConfiguration(); // Bump up replication interval. config.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 10); @@ -1896,6 +1906,17 @@ public void delayDeleteReplica() { // If this replica is deleted from memory, the client would got an ReplicaNotFoundException. assertNotNull(ds.getStoredBlock(bpid, extendedBlock.getBlockId())); + assertEquals(1, ds.asyncDiskService.countPendingDeletions()); + assertEquals(1, ds.getPendingAsyncDeletions()); + + // Validate PendingAsyncDeletions metrics. + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = new ObjectName( + "Hadoop:service=DataNode,name=FSDatasetState-" + dn.getDatanodeUuid()); + long pendingAsyncDeletions = (long) mbs.getAttribute(mxbeanName, + "PendingAsyncDeletions"); + assertEquals(1, pendingAsyncDeletions); + // Make it resume the removeReplicaFromMem method. semaphore.release(1); @@ -1903,6 +1924,12 @@ public void delayDeleteReplica() { GenericTestUtils.waitFor(() -> ds.asyncDiskService.countPendingDeletions() == 0, 100, 1000); + assertEquals(0, ds.getPendingAsyncDeletions()); + + pendingAsyncDeletions = (long) mbs.getAttribute(mxbeanName, + "PendingAsyncDeletions"); + assertEquals(0, pendingAsyncDeletions); + // Sleep for two heartbeat times. Thread.sleep(config.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, From 81edbebdd80dc590d2c4fa8e8fb95a0e91c8115d Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 12 Oct 2023 17:47:44 +0100 Subject: [PATCH 089/155] HADOOP-18889. S3A v2 SDK third party support (#6141) Tune AWS v2 SDK changes based on testing with third party stores including GCS. Contains HADOOP-18889. S3A v2 SDK error translations and troubleshooting docs * Changes needed to work with multiple third party stores * New third_party_stores document on how to bind to and test third party stores, including google gcs (which works!) * Troubleshooting docs mostly updated for v2 SDK Exception translation/resilience * New AWSUnsupportedFeatureException for unsupported/unavailable errors * Handle 501 method unimplemented as one of these * Error codes > 500 mapped to the AWSStatus500Exception if no explicit handler. * Precondition errors handled a bit better * GCS throttle exception also recognized. * GCS raises 404 on a delete of a file which doesn't exist: swallow it. * Error translation uses reflection to create IOE of the right type. All IOEs at the bottom of an AWS stack chain are regenerated. then a new exception of that specific type is created, with the top level ex its cause. This is done to retain the whole stack chain. * Reduce the number of retries within the AWS SDK * And those of s3a code. * S3ARetryPolicy explicitly declare SocketException as connectivity failure but subclasses BindException * SocketTimeoutException also considered connectivity * Log at debug whenever retry policies looked up * Reorder exceptions to alphabetical order, with commentary * Review use of the Invoke.retry() method The reduction in retries is because its clear when you try to create a bucket which doesn't resolve that the time for even an UnknownHostException to eventually fail over 90s, which then hit the s3a retry code. - Reducing the SDK retries means these escalate to our code better. - Cutting back on our own retries makes it a bit more responsive for most real deployments. - maybeTranslateNetworkException() and s3a retry policy means that unknown host exception is recognised and fails fast. Contributed by Steve Loughran --- .../src/main/resources/core-default.xml | 10 +- .../hadoop/fs/s3a/AWSBadRequestException.java | 4 +- .../hadoop/fs/s3a/AWSStatus500Exception.java | 16 +- .../s3a/AWSUnsupportedFeatureException.java | 46 ++ .../org/apache/hadoop/fs/s3a/Constants.java | 7 +- .../apache/hadoop/fs/s3a/MultipartUtils.java | 8 + .../apache/hadoop/fs/s3a/S3AFileSystem.java | 52 +- .../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 125 ++- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 39 +- .../apache/hadoop/fs/s3a/S3ListResult.java | 6 +- .../fs/s3a/SimpleAWSCredentialsProvider.java | 2 +- .../org/apache/hadoop/fs/s3a/Statistic.java | 2 +- .../hadoop/fs/s3a/auth/SignerFactory.java | 42 +- .../hadoop/fs/s3a/auth/SignerManager.java | 15 +- .../hadoop/fs/s3a/impl/AWSClientConfig.java | 14 +- .../hadoop/fs/s3a/impl/ErrorTranslation.java | 70 +- .../fs/s3a/impl/InstantiationIOException.java | 12 +- .../hadoop/fs/s3a/impl/InternalConstants.java | 11 +- .../fs/s3a/impl/RequestFactoryImpl.java | 3 +- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 6 +- .../hadoop/fs/s3a/tools/MarkerTool.java | 27 +- .../tools/hadoop-aws/assumed_roles.md | 195 +---- .../site/markdown/tools/hadoop-aws/index.md | 629 +++++++++++---- .../site/markdown/tools/hadoop-aws/testing.md | 78 +- .../tools/hadoop-aws/third_party_stores.md | 415 ++++++++++ .../tools/hadoop-aws/troubleshooting_s3a.md | 757 +++--------------- .../fs/s3a/ITestS3ABucketExistence.java | 16 +- .../hadoop/fs/s3a/ITestS3ACannedACLs.java | 4 + .../hadoop/fs/s3a/ITestS3AConfiguration.java | 12 +- .../fs/s3a/ITestS3AContentEncoding.java | 39 +- .../ITestS3AContractGetFileStatusV1List.java | 4 + .../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 99 ++- .../fs/s3a/ITestS3AFailureHandling.java | 21 +- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 12 +- .../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 20 +- .../hadoop/fs/s3a/ITestS3AStorageClass.java | 7 +- .../hadoop/fs/s3a/MultipartTestUtils.java | 8 + .../hadoop/fs/s3a/S3ATestConstants.java | 15 + .../apache/hadoop/fs/s3a/S3ATestUtils.java | 57 +- .../org/apache/hadoop/fs/s3a/TestInvoker.java | 3 +- .../hadoop/fs/s3a/auth/ITestCustomSigner.java | 49 +- .../hadoop/fs/s3a/auth/TestSignerManager.java | 31 +- .../magic/ITestS3AHugeMagicCommits.java | 16 + .../hadoop/fs/s3a/impl/ITestXAttrCost.java | 12 +- .../fs/s3a/impl/TestErrorTranslation.java | 115 +++ .../s3a/performance/AbstractS3ACostTest.java | 5 +- .../ITestDirectoryMarkerListing.java | 16 +- .../s3a/performance/ITestS3ADeleteCost.java | 16 +- .../ITestS3AHugeFilesSSECDiskBlocks.java | 7 +- .../scale/ITestS3AHugeFilesStorageClass.java | 7 +- .../src/test/resources/core-site.xml | 12 + .../src/test/resources/log4j.properties | 8 +- 52 files changed, 1980 insertions(+), 1222 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 9564a56e9e40d..24f6ca27847c2 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1584,8 +1584,14 @@ fs.s3a.attempts.maximum - 20 - How many times we should retry commands on transient errors. + 5 + + Number of times the AWS client library should retry errors before + escalating to the S3A code: {@value}. + The S3A connector does its own selective retries; the only time the AWS + SDK operations are not wrapped is during multipart copy via the AWS SDK + transfer manager. + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java index c5867eeab4f4d..dfa1725acd11e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java @@ -20,6 +20,8 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_400_BAD_REQUEST; + /** * A 400 "Bad Request" exception was received. * This is the general "bad parameters, headers, whatever" failure. @@ -28,7 +30,7 @@ public class AWSBadRequestException extends AWSServiceIOException { /** * HTTP status code which signals this failure mode was triggered: {@value}. */ - public static final int STATUS_CODE = 400; + public static final int STATUS_CODE = SC_400_BAD_REQUEST; /** * Instantiate. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java index ecfe5da1455cd..f7c72f8530959 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java @@ -21,13 +21,13 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException; /** - * A 500 response came back from a service. - * This is considered probably retriable, That is, we assume - *

      - *
    1. whatever error happened in the service itself to have happened - * before the infrastructure committed the operation.
      2. - *
    2. Nothing else got through either.
      3. - *
    + * A 5xx response came back from a service. + * The 500 error considered retriable by the AWS SDK, which will have already + * tried it {@code fs.s3a.attempts.maximum} times before reaching s3a + * code. + * How it handles other 5xx errors is unknown: S3A FS code will treat them + * as unrecoverable on the basis that they indicate some third-party store + * or gateway problem. */ public class AWSStatus500Exception extends AWSServiceIOException { public AWSStatus500Exception(String operation, @@ -37,6 +37,6 @@ public AWSStatus500Exception(String operation, @Override public boolean retryable() { - return true; + return false; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java new file mode 100644 index 0000000000000..dd1cebf45f9cf --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSUnsupportedFeatureException.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import software.amazon.awssdk.awscore.exception.AwsServiceException; + +/** + * A store returned an error indicating that it does not support a + * specific S3 feature such as the chosen ChangeDetectionPolicy or + * other AWS-S3 feature that the third-party store does not support. + * The workaround is to disable use of the feature. + * Unrecoverable. + */ +public class AWSUnsupportedFeatureException extends AWSServiceIOException { + + /** + * Instantiate. + * @param operation operation which triggered this + * @param cause the underlying cause + */ + public AWSUnsupportedFeatureException(String operation, + AwsServiceException cause) { + super(operation, cause); + } + + @Override + public boolean retryable() { + return false; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 02f496abde000..492508225fa3e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -228,6 +228,9 @@ private Constants() { /** * Number of times the AWS client library should retry errors before * escalating to the S3A code: {@value}. + * The S3A connector does its own selective retries; the only time the AWS + * SDK operations are not wrapped is during multipart copy via the AWS SDK + * transfer manager. */ public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum"; @@ -235,7 +238,7 @@ private Constants() { * Default number of times the AWS client library should retry errors before * escalating to the S3A code: {@value}. */ - public static final int DEFAULT_MAX_ERROR_RETRIES = 10; + public static final int DEFAULT_MAX_ERROR_RETRIES = 5; /** * Experimental/Unstable feature: should the AWS client library retry @@ -264,7 +267,7 @@ private Constants() { // milliseconds until we give up trying to establish a connection to s3 public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout"; - public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000; + public static final int DEFAULT_ESTABLISH_TIMEOUT = 5000; // milliseconds until we give up on a connection to s3 public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java index 296ec18dcf18d..efca093204c25 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java @@ -223,6 +223,14 @@ public static class UploadIterator /** Iterator over the current listing. */ private ListIterator batchIterator; + /** + * Construct an iterator to list uploads under a path. + * @param storeContext store context + * @param s3 s3 client + * @param maxKeys max # of keys to list per batch + * @param prefix prefix + * @throws IOException listing failure. + */ @Retries.RetryTranslated public UploadIterator( final StoreContext storeContext, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index fa7de69140fcf..a56f1db0b4556 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -334,6 +334,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); /** Exactly once log to warn about setting the region in config to avoid probe. */ private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG); + + /** Log to warn of storage class configuration problems. */ + private static final LogExactlyOnce STORAGE_CLASS_WARNING = new LogExactlyOnce(LOG); + private static final Logger PROGRESS = LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress"); private LocalDirAllocator directoryAllocator; @@ -1073,7 +1077,8 @@ private Region getS3Region(String region) throws IOException { if (exception.statusCode() == SC_404_NOT_FOUND) { throw new UnknownStoreException("s3a://" + bucket + "/", - " Bucket does " + "not exist"); + " Bucket does not exist: " + exception, + exception); } throw exception; @@ -1174,6 +1179,9 @@ protected RequestFactory createRequestFactory() { // Any encoding type String contentEncoding = getConf().getTrimmed(CONTENT_ENCODING, null); + if (contentEncoding != null) { + LOG.debug("Using content encoding set in {} = {}", CONTENT_ENCODING, contentEncoding); + } String storageClassConf = getConf() .getTrimmed(STORAGE_CLASS, "") @@ -1181,10 +1189,11 @@ protected RequestFactory createRequestFactory() { StorageClass storageClass = null; if (!storageClassConf.isEmpty()) { storageClass = StorageClass.fromValue(storageClassConf); - + LOG.debug("Using storage class {}", storageClass); if (storageClass.equals(StorageClass.UNKNOWN_TO_SDK_VERSION)) { - LOG.warn("Unknown storage class property {}: {}; falling back to default storage class", - STORAGE_CLASS, storageClassConf); + STORAGE_CLASS_WARNING.warn("Unknown storage class \"{}\" from option: {};" + + " falling back to default storage class", + storageClassConf, STORAGE_CLASS); storageClass = null; } @@ -1431,7 +1440,7 @@ public String getBucketLocation() throws IOException { public String getBucketLocation(String bucketName) throws IOException { final String region = trackDurationAndSpan( STORE_EXISTS_PROBE, bucketName, null, () -> - invoker.retry("getBucketLocation()", bucketName, true, () -> + once("getBucketLocation()", bucketName, () -> // If accessPoint then region is known from Arn accessPoint != null ? accessPoint.getRegion() @@ -2993,7 +3002,7 @@ protected void deleteObject(String key) "deleting %s", key)) { invoker.retryUntranslated(String.format("Delete %s:/%s", bucket, key), DELETE_CONSIDERED_IDEMPOTENT, - ()-> { + () -> { incrementStatistic(OBJECT_DELETE_OBJECTS); trackDurationOfInvocation(getDurationTrackerFactory(), OBJECT_DELETE_REQUEST.getSymbol(), @@ -3002,6 +3011,12 @@ protected void deleteObject(String key) .build())); return null; }); + } catch (AwsServiceException ase) { + // 404 errors get swallowed; this can be raised by + // third party stores (GCS). + if (!isObjectNotFound(ase)) { + throw ase; + } } } @@ -4287,13 +4302,13 @@ protected synchronized void stopAllServices() { } /** - * Verify that the input stream is open. Non blocking; this gives + * Verify that the filesystem has not been closed. Non blocking; this gives * the last state of the volatile {@link #closed} field. - * @throws IOException if the connection is closed. + * @throws PathIOException if the FS is closed. */ - private void checkNotClosed() throws IOException { + private void checkNotClosed() throws PathIOException { if (isClosed) { - throw new IOException(uri + ": " + E_FS_CLOSED); + throw new PathIOException(uri.toString(), E_FS_CLOSED); } } @@ -4443,7 +4458,6 @@ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, // This means the File was deleted since LIST enumerated it. LOG.debug("getObjectMetadata({}) failed to find an expected file", srcKey, e); - // We create an exception, but the text depends on the S3Guard state throw new RemoteFileChangedException( keyToQualifiedPath(srcKey).toString(), action, @@ -4454,6 +4468,8 @@ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, CopyObjectRequest.Builder copyObjectRequestBuilder = getRequestFactory().newCopyObjectRequestBuilder(srcKey, dstKey, srcom); changeTracker.maybeApplyConstraint(copyObjectRequestBuilder); + final CopyObjectRequest copyRequest = copyObjectRequestBuilder.build(); + LOG.debug("Copy Request: {}", copyRequest); CopyObjectResponse response; // transfer manager is skipped if disabled or the file is too small to worry about @@ -4468,7 +4484,7 @@ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, Copy copy = transferManager.copy( CopyRequest.builder() - .copyObjectRequest(copyObjectRequestBuilder.build()) + .copyObjectRequest(copyRequest) .build()); try { @@ -4477,6 +4493,8 @@ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, } catch (CompletionException e) { Throwable cause = e.getCause(); if (cause instanceof SdkException) { + // if this is a 412 precondition failure, it may + // be converted to a RemoteFileChangedException SdkException awsException = (SdkException)cause; changeTracker.processException(awsException, "copy"); throw awsException; @@ -4493,7 +4511,15 @@ private CopyObjectResponse copyFile(String srcKey, String dstKey, long size, () -> { LOG.debug("copyFile: single part copy {} -> {} of size {}", srcKey, dstKey, size); incrementStatistic(OBJECT_COPY_REQUESTS); - return s3Client.copyObject(copyObjectRequestBuilder.build()); + try { + return s3Client.copyObject(copyRequest); + } catch (SdkException awsException) { + // if this is a 412 precondition failure, it may + // be converted to a RemoteFileChangedException + changeTracker.processException(awsException, "copy"); + // otherwise, rethrow + throw awsException; + } }); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java index fdb4591476c56..cc30da4fbe04e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java @@ -22,7 +22,10 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; +import java.net.BindException; +import java.net.ConnectException; import java.net.NoRouteToHostException; +import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; import java.nio.file.AccessDeniedException; @@ -31,7 +34,6 @@ import java.util.concurrent.TimeUnit; import software.amazon.awssdk.core.exception.SdkException; -import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,13 +47,14 @@ import org.apache.hadoop.net.ConnectTimeoutException; +import static java.util.Objects.requireNonNull; import static org.apache.hadoop.io.retry.RetryPolicies.*; - import static org.apache.hadoop.fs.s3a.Constants.*; + /** * The S3A request retry policy. - * + *

    * This uses the retry options in the configuration file to determine retry * count and delays for "normal" retries and separately, for throttling; * the latter is best handled for longer with an exponential back-off. @@ -66,20 +69,25 @@ * For non-idempotent operations, only failures due to throttling or * from failures which are known to only arise prior to talking to S3 * are retried. - * + *

    * The retry policy is all built around that of the normal IO exceptions, * particularly those extracted from * {@link S3AUtils#translateException(String, Path, SdkException)}. * Because the {@link #shouldRetry(Exception, int, int, boolean)} method * does this translation if an {@code SdkException} is processed, * the policy defined for the IOEs also applies to the original exceptions. - * + *

    * Put differently: this retry policy aims to work for handlers of the * untranslated exceptions, as well as the translated ones. + *

    + * Note that because delete is considered idempotent, all s3a operations currently + * declare themselves idempotent. + * This means the retry policy here is more complex than it needs to be -but it + * does force us to consider when retrying operations would not be safe. * @see S3 Error responses - * @see Amazon S3 Error Best Practices + * @see Amazon S3 Error Best Practices */ -@SuppressWarnings("visibilitymodifier") // I want a struct of finals, for real. +@SuppressWarnings("visibilitymodifier") // we want a struct of finals, for real. public class S3ARetryPolicy implements RetryPolicy { private static final Logger LOG = LoggerFactory.getLogger( @@ -122,8 +130,7 @@ public class S3ARetryPolicy implements RetryPolicy { * @param conf configuration to read. */ public S3ARetryPolicy(Configuration conf) { - Preconditions.checkArgument(conf != null, "Null configuration"); - this.configuration = conf; + this.configuration = requireNonNull(conf, "Null configuration"); // base policy from configuration int limit = conf.getInt(RETRY_LIMIT, RETRY_LIMIT_DEFAULT); @@ -188,35 +195,57 @@ protected Map, RetryPolicy> createExceptionMap() { // inherit policies. Map, RetryPolicy> policyMap = new HashMap<>(); - // failfast exceptions which we consider unrecoverable - policyMap.put(UnknownHostException.class, fail); - policyMap.put(NoRouteToHostException.class, fail); - policyMap.put(InterruptedException.class, fail); - // note this does not pick up subclasses (like socket timeout) - policyMap.put(InterruptedIOException.class, fail); // Access denial and auth exceptions are not retried policyMap.put(AccessDeniedException.class, fail); - policyMap.put(NoAuthWithAWSException.class, fail); - policyMap.put(FileNotFoundException.class, fail); - policyMap.put(UnknownStoreException.class, fail); - policyMap.put(InvalidRequestException.class, fail); - // once the file has changed, trying again is not going to help - policyMap.put(RemoteFileChangedException.class, fail); + // Treated as an immediate failure + policyMap.put(AWSBadRequestException.class, fail); - // likely only recovered by changing the policy configuration or s3 - // implementation - policyMap.put(NoVersionAttributeException.class, fail); + // use specific retry policy for aws client exceptions + // nested IOExceptions will already have been extracted and used + // in this map. + policyMap.put(AWSClientIOException.class, retryAwsClientExceptions); + + // server didn't respond. + policyMap.put(AWSNoResponseException.class, retryIdempotentCalls); // should really be handled by resubmitting to new location; // that's beyond the scope of this retry policy policyMap.put(AWSRedirectException.class, fail); + // generic exception from the service + policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions); + // throttled requests are can be retried, always policyMap.put(AWSServiceThrottledException.class, throttlePolicy); + // Status 5xx error code is an immediate failure + // this is sign of a server-side problem, and while + // rare in AWS S3, it does happen on third party stores. + // (out of disk space, etc). + // by the time we get here, the aws sdk will have + // already retried. + // there is specific handling for some 5XX codes (501, 503); + // this is for everything else + policyMap.put(AWSStatus500Exception.class, fail); + + // subclass of AWSServiceIOException whose cause is always S3Exception + policyMap.put(AWSS3IOException.class, retryIdempotentCalls); + + // server doesn't support a feature. + // raised from a number of HTTP error codes -mostly from + // third-party stores which only support a subset of AWS S3 + // operations. + policyMap.put(AWSUnsupportedFeatureException.class, fail); + + // socket exception subclass we consider unrecoverable + // though this is normally only found when opening a port for listening. + // which is never done in S3A. + policyMap.put(BindException.class, fail); + // connectivity problems are retried without worrying about idempotency policyMap.put(ConnectTimeoutException.class, connectivityFailure); + policyMap.put(ConnectException.class, connectivityFailure); // this can be a sign of an HTTP connection breaking early. // which can be reacted to by another attempt if the request was idempotent. @@ -224,27 +253,38 @@ protected Map, RetryPolicy> createExceptionMap() { // which isn't going to be recovered from policyMap.put(EOFException.class, retryIdempotentCalls); - // policy on a 400/bad request still ambiguous. - // Treated as an immediate failure - policyMap.put(AWSBadRequestException.class, fail); + // object not found. 404 when not unknown bucket; 410 "gone" + policyMap.put(FileNotFoundException.class, fail); - // Status 500 error code is also treated as a connectivity problem - policyMap.put(AWSStatus500Exception.class, connectivityFailure); + // Interrupted, usually by other threads + policyMap.put(InterruptedException.class, fail); + // note this does not pick up subclasses (like socket timeout) + policyMap.put(InterruptedIOException.class, fail); + policyMap.put(InvalidRequestException.class, fail); - // server didn't respond. - policyMap.put(AWSNoResponseException.class, retryIdempotentCalls); + // auth failure. Possibly recoverable by reattempting with + // the credential provider, but not covered here. + policyMap.put(NoAuthWithAWSException.class, fail); - // use specific retry policy for aws client exceptions - policyMap.put(AWSClientIOException.class, retryAwsClientExceptions); - policyMap.put(AWSServiceIOException.class, retryAwsClientExceptions); + // network routing. + policyMap.put(NoRouteToHostException.class, fail); - // other operations - policyMap.put(AWSS3IOException.class, retryIdempotentCalls); - policyMap.put(SocketTimeoutException.class, retryIdempotentCalls); + // likely only recovered by changing the policy configuration or s3 + // implementation + policyMap.put(NoVersionAttributeException.class, fail); + // once the file has changed, trying again is not going to help + policyMap.put(RemoteFileChangedException.class, fail); + // general socket exceptions + policyMap.put(SocketException.class, connectivityFailure); + policyMap.put(SocketTimeoutException.class, connectivityFailure); + // assume that DNS wil not recover; SDK is likely to have retried. + policyMap.put(UnknownHostException.class, fail); + policyMap.put(UnknownStoreException.class, fail); // Unsupported requests do not work, however many times you try policyMap.put(UnsupportedRequestException.class, fail); + return policyMap; } @@ -253,14 +293,19 @@ public RetryAction shouldRetry(Exception exception, int retries, int failovers, boolean idempotent) throws Exception { - Preconditions.checkArgument(exception != null, "Null exception"); - Exception ex = exception; + Exception ex = requireNonNull(exception, "Null exception"); if (exception instanceof SdkException) { // update the sdk exception for the purpose of exception // processing. ex = S3AUtils.translateException("", "", (SdkException) exception); } - return retryPolicy.shouldRetry(ex, retries, failovers, idempotent); + LOG.debug("Retry probe for {} with {} retries and {} failovers," + + " idempotent={}, due to {}", + ex.getClass().getSimpleName(), retries, failovers, idempotent, ex, ex); + // look in the retry policy map + final RetryAction action = retryPolicy.shouldRetry(ex, retries, failovers, idempotent); + LOG.debug("Retry action is {}", action); + return action; } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 093608fb528c7..6798a99c19e1e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -80,6 +80,7 @@ import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -191,10 +192,14 @@ public static IOException translateException(@Nullable String operation, ioe = maybeTranslateCredentialException(path, exception); if (ioe != null) { return ioe; - } else { - // no custom handling. - return new AWSClientIOException(message, exception); } + // network problems covered by an IOE inside the exception chain. + ioe = maybeExtractNetworkException(path, exception); + if (ioe != null) { + return ioe; + } + // no custom handling. + return new AWSClientIOException(message, exception); } else { // "error response returned by an S3 or other service." // These contain more details and should be translated based @@ -209,6 +214,8 @@ public static IOException translateException(@Nullable String operation, if (ase.awsErrorDetails() != null) { message = message + ":" + ase.awsErrorDetails().errorCode(); } + + // big switch on the HTTP status code. switch (status) { case SC_301_MOVED_PERMANENTLY: @@ -242,7 +249,8 @@ public static IOException translateException(@Nullable String operation, // this is a missing bucket ioe = new UnknownStoreException(path, message, ase); } else { - // a normal unknown object + // a normal unknown object. + // Can also be raised by third-party stores when aborting an unknown multipart upload ioe = new FileNotFoundException(message); ioe.initCause(ase); } @@ -255,10 +263,13 @@ public static IOException translateException(@Nullable String operation, ioe.initCause(ase); break; - // method not allowed; seen on S3 Select. - // treated as a bad request + // errors which stores can return from requests which + // the store does not support. case SC_405_METHOD_NOT_ALLOWED: - ioe = new AWSBadRequestException(message, s3Exception); + case SC_412_PRECONDITION_FAILED: + case SC_415_UNSUPPORTED_MEDIA_TYPE: + case SC_501_NOT_IMPLEMENTED: + ioe = new AWSUnsupportedFeatureException(message, s3Exception); break; // out of range. This may happen if an object is overwritten with @@ -277,7 +288,8 @@ public static IOException translateException(@Nullable String operation, break; // throttling - case SC_503_SERVICE_UNAVAILABLE: + case SC_429_TOO_MANY_REQUESTS_GCS: // google cloud through this connector + case SC_503_SERVICE_UNAVAILABLE: // AWS ioe = new AWSServiceThrottledException(message, ase); break; @@ -295,8 +307,15 @@ public static IOException translateException(@Nullable String operation, // other 200: FALL THROUGH default: - // no specific exit code. Choose an IOE subclass based on the class - // of the caught exception + // no specifically handled exit code. + + // convert all unknown 500+ errors to a 500 exception + if (status > SC_500_INTERNAL_SERVER_ERROR) { + ioe = new AWSStatus500Exception(message, ase); + break; + } + + // Choose an IOE subclass based on the class of the caught exception ioe = s3Exception != null ? new AWSS3IOException(message, s3Exception) : new AWSServiceIOException(message, ase); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java index c77311211abcb..0300b78bec250 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java @@ -29,6 +29,8 @@ import org.slf4j.Logger; +import static java.util.Objects.requireNonNull; + /** * API version-independent container for S3 List responses. */ @@ -47,7 +49,7 @@ protected S3ListResult(ListObjectsResponse v1, ListObjectsV2Response v2) { * @return new list result container */ public static S3ListResult v1(ListObjectsResponse result) { - return new S3ListResult(result, null); + return new S3ListResult(requireNonNull(result), null); } /** @@ -56,7 +58,7 @@ public static S3ListResult v1(ListObjectsResponse result) { * @return new list result container */ public static S3ListResult v2(ListObjectsV2Response result) { - return new S3ListResult(null, result); + return new S3ListResult(null, requireNonNull(result)); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java index f7eaf825b9c94..c8fa97a9b9f9e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java @@ -90,7 +90,7 @@ public AwsCredentials resolveCredentials() { public String toString() { return "SimpleAWSCredentialsProvider{" + "accessKey.empty=" + accessKey.isEmpty() + - ", secretKey.empty'" + secretKey.isEmpty() + + ", secretKey.empty=" + secretKey.isEmpty() + '}'; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index b5f7fcbcd016d..f54113af6c5c1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -582,7 +582,7 @@ public enum Statistic { TYPE_COUNTER), MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED( StoreStatisticNames.MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED, - "Multipart Upload Abort Unner Path Invoked", + "Multipart Upload Abort Under Path Invoked", TYPE_COUNTER), MULTIPART_UPLOAD_COMPLETED( StoreStatisticNames.MULTIPART_UPLOAD_COMPLETED, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java index c786086947fac..5d34688cebe14 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java @@ -31,7 +31,10 @@ import software.amazon.awssdk.core.signer.Signer; import org.apache.hadoop.fs.s3a.S3AUtils; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable; +import static org.apache.hadoop.util.Preconditions.checkArgument; /** * Signer factory used to register and create signers. @@ -44,6 +47,9 @@ public final class SignerFactory { public static final String NO_OP_SIGNER = "NoOpSignerType"; private static final String S3_V4_SIGNER = "AWSS3V4SignerType"; + /** The v2 signer is no longer available: {@value}. */ + public static final String S3_V2_SIGNER = "S3SignerType"; + private static final Map> SIGNERS = new ConcurrentHashMap<>(); @@ -69,12 +75,8 @@ public static void registerSigner( final String signerType, final Class signerClass) { - if (signerType == null) { - throw new IllegalArgumentException("signerType cannot be null"); - } - if (signerClass == null) { - throw new IllegalArgumentException("signerClass cannot be null"); - } + checkArgument(signerType != null, "signerType cannot be null"); + checkArgument(signerClass != null, "signerClass cannot be null"); SIGNERS.put(signerType, signerClass); } @@ -82,14 +84,21 @@ public static void registerSigner( /** * Check if the signer has already been registered. * @param signerType signer to get + * @throws IllegalArgumentException if the signer type is unknown. */ public static void verifySignerRegistered(String signerType) { - Class signerClass = SIGNERS.get(signerType); - if (signerClass == null) { - throw new IllegalArgumentException("unknown signer type: " + signerType); - } + checkArgument(isSignerRegistered(signerType), + "unknown signer type: %s", signerType); } + /** + * Check if the signer has already been registered. + * @param signerType signer to get + * @return true if the signer is registered. + */ + public static boolean isSignerRegistered(String signerType) { + return SIGNERS.containsKey(signerType); + } /** * Create an instance of the given signer. @@ -97,13 +106,22 @@ public static void verifySignerRegistered(String signerType) { * @param signerType The signer type. * @param configKey Config key used to configure the signer. * @return The new signer instance. - * @throws IOException on any problem. + * @throws InstantiationIOException instantiation problems. + * @throws IOException on any other problem. + * */ public static Signer createSigner(String signerType, String configKey) throws IOException { + if (S3_V2_SIGNER.equals(signerType)) { + throw unavailable(null, null, configKey, S3_V2_SIGNER + " is no longer supported"); + } + if (!isSignerRegistered(signerType)) { + throw unavailable(null, null, configKey, "unknown signer type: " + signerType); + } Class signerClass = SIGNERS.get(signerType); + String className = signerClass.getName(); - LOG.debug("Signer class is {}", className); + LOG.debug("Signer class from {} and key {} is {}", signerType, configKey, className); Signer signer = S3AUtils.getInstanceFromReflection(className, null, null, Signer.class, "create", diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index 198bc66133a95..f0a71007b4cf8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -109,17 +109,20 @@ public void initCustomSigners() { } } - /* - * Make sure the signer class is registered once with the AWS SDK + /** + * Make sure the signer class is registered once with the AWS SDK. + * @param signerName signer name + * @param signerClassName classname + * @param conf source configuration + * @throws RuntimeException if the class is not found */ private static void maybeRegisterSigner(String signerName, String signerClassName, Configuration conf) { - try { - SignerFactory.verifySignerRegistered(signerName); - } catch (IllegalArgumentException e) { + + if (!SignerFactory.isSignerRegistered(signerName)) { // Signer is not registered with the AWS SDK. // Load the class and register the signer. - Class clazz = null; + Class clazz; try { clazz = (Class) conf.getClassByName(signerClassName); } catch (ClassNotFoundException cnfe) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java index 4ff2ec0b0b9f4..3400daff509b2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSClientConfig.java @@ -117,14 +117,14 @@ public static ApacheHttpClient.Builder createHttpClientBuilder(Configuration con S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0); int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0); - httpClientBuilder.connectionTimeout(Duration.ofSeconds(connectionEstablishTimeout)); - httpClientBuilder.socketTimeout(Duration.ofSeconds(socketTimeout)); + httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout)); + httpClientBuilder.socketTimeout(Duration.ofMillis(socketTimeout)); // set the connection TTL irrespective of whether the connection is in use or not. // this can balance requests over different S3 servers, and avoid failed // connections. See HADOOP-18845. long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1); - httpClientBuilder.connectionTimeToLive(Duration.ofSeconds(ttl)); + httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl)); NetworkBinding.bindSSLChannelMode(conf, httpClientBuilder); @@ -148,15 +148,15 @@ public static NettyNioAsyncHttpClient.Builder createAsyncHttpClientBuilder(Confi S3AUtils.intOption(conf, ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT, 0); int socketTimeout = S3AUtils.intOption(conf, SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT, 0); - httpClientBuilder.connectionTimeout(Duration.ofSeconds(connectionEstablishTimeout)); - httpClientBuilder.readTimeout(Duration.ofSeconds(socketTimeout)); - httpClientBuilder.writeTimeout(Duration.ofSeconds(socketTimeout)); + httpClientBuilder.connectionTimeout(Duration.ofMillis(connectionEstablishTimeout)); + httpClientBuilder.readTimeout(Duration.ofMillis(socketTimeout)); + httpClientBuilder.writeTimeout(Duration.ofMillis(socketTimeout)); // set the connection TTL irrespective of whether the connection is in use or not. // this can balance requests over different S3 servers, and avoid failed // connections. See HADOOP-18845. long ttl = longOption(conf, CONNECTION_TTL, DEFAULT_CONNECTION_TTL, -1); - httpClientBuilder.connectionTimeToLive(Duration.ofSeconds(ttl)); + httpClientBuilder.connectionTimeToLive(Duration.ofMillis(ttl)); // TODO: Don't think you can set a socket factory for the netty client. // NetworkBinding.bindSSLChannelMode(conf, awsConf); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java index 54a91323bc2e2..7b5190becc487 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java @@ -18,8 +18,13 @@ package org.apache.hadoop.fs.s3a.impl; +import java.io.IOException; +import java.lang.reflect.Constructor; + import software.amazon.awssdk.awscore.exception.AwsServiceException; +import org.apache.hadoop.fs.PathIOException; + import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND; /** @@ -35,7 +40,7 @@ * The existing code las been left in S3AUtils it is to avoid cherry-picking * problems on backports. */ -public class ErrorTranslation { +public final class ErrorTranslation { /** * Private constructor for utility class. @@ -66,6 +71,69 @@ public static boolean isObjectNotFound(AwsServiceException e) { return e.statusCode() == SC_404_NOT_FOUND && !isUnknownBucket(e); } + /** + * Translate an exception if it or its inner exception is an + * IOException. + * If this condition is not met, null is returned. + * @param path path of operation. + * @param thrown exception + * @return a translated exception or null. + */ + public static IOException maybeExtractNetworkException(String path, Throwable thrown) { + + if (thrown == null) { + return null; + } + + // look inside + Throwable cause = thrown.getCause(); + while (cause != null && cause.getCause() != null) { + cause = cause.getCause(); + } + if (!(cause instanceof IOException)) { + return null; + } + + // the cause can be extracted to an IOE. + // rather than just return it, we try to preserve the stack trace + // of the outer exception. + // as a new instance is created through reflection, the + // class of the returned instance will be that of the innermost, + // unless no suitable constructor is available. + return wrapWithInnerIOE(path, thrown, (IOException) cause); + + } + + /** + * Given an outer and an inner exception, create a new IOE + * of the inner type, with the outer exception as the cause. + * The message is derived from both. + * This only works if the inner exception has a constructor which + * takes a string; if not a PathIOException is created. + *

    + * See {@code NetUtils}. + * @param type of inner exception. + * @param path path of the failure. + * @param outer outermost exception. + * @param inner inner exception. + * @return the new exception. + */ + @SuppressWarnings("unchecked") + private static IOException wrapWithInnerIOE( + String path, + Throwable outer, + T inner) { + String msg = outer.toString() + ": " + inner.getMessage(); + Class clazz = inner.getClass(); + try { + Constructor ctor = clazz.getConstructor(String.class); + Throwable t = ctor.newInstance(msg); + return (T) (t.initCause(outer)); + } catch (Throwable e) { + return new PathIOException(path, msg, outer); + } + } + /** * AWS error codes explicitly recognized and processes specially; * kept in their own class for isolation. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java index 435db879fabf8..e09571fc338d6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InstantiationIOException.java @@ -73,12 +73,14 @@ public enum Kind { public InstantiationIOException( Kind kind, - @Nullable URI uri, String classname, + @Nullable URI uri, + @Nullable String classname, @Nullable String key, String message, - Throwable cause) { + @Nullable Throwable cause) { super(uri!= null ? uri.toString() : "", - "Class " + classname + " " + message + (classname != null ? ("Class " + classname + " ") : "") + + message + (key != null ? (" (configuration key " + key + ")") : ""), cause); this.kind = kind; @@ -137,8 +139,8 @@ public static InstantiationIOException isNotInstanceOf( */ public static InstantiationIOException unavailable( @Nullable URI uri, - String classname, - String key, + @Nullable String classname, + @Nullable String key, String text) { return new InstantiationIOException(Kind.Unavailable, uri, classname, key, text, null); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java index 7af82f70aebf6..e7fcbe0351042 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java @@ -148,9 +148,15 @@ private InternalConstants() { /** 412 status code: Precondition Failed. */ public static final int SC_412_PRECONDITION_FAILED = 412; + /** 415 status code: Content type unsupported by this store. */ + public static final int SC_415_UNSUPPORTED_MEDIA_TYPE = 415; + /** 416 status code: Range Not Satisfiable. */ public static final int SC_416_RANGE_NOT_SATISFIABLE = 416; + /** 429 status code: This is the google GCS throttle message. */ + public static final int SC_429_TOO_MANY_REQUESTS_GCS = 429; + /** 443 status code: No Response (unofficial). */ public static final int SC_443_NO_RESPONSE = 443; @@ -160,7 +166,10 @@ private InternalConstants() { /** 500 status code: Internal Server Error. */ public static final int SC_500_INTERNAL_SERVER_ERROR = 500; - /** 503 status code: Service Unavailable. */ + /** 501 status code: method not implemented. */ + public static final int SC_501_NOT_IMPLEMENTED = 501; + + /** 503 status code: Service Unavailable. on AWS S3: throttle response. */ public static final int SC_503_SERVICE_UNAVAILABLE = 503; /** Name of the log for throttling events. Value: {@value}. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index cacbee381bebc..ca36b658d70f3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -63,6 +63,7 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.util.Preconditions.checkNotNull; +import static software.amazon.awssdk.services.s3.model.StorageClass.UNKNOWN_TO_SDK_VERSION; /** * The standard implementation of the request factory. @@ -242,7 +243,7 @@ public CopyObjectRequest.Builder newCopyObjectRequestBuilder(String srcKey, .metadataDirective(MetadataDirective.REPLACE) .acl(cannedACL); - if (srcom.storageClass() != null) { + if (srcom.storageClass() != null && srcom.storageClass() != UNKNOWN_TO_SDK_VERSION) { copyObjectRequestBuilder.storageClass(srcom.storageClass()); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index ec68168bd0ffd..330b92186dde1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -24,7 +24,6 @@ import java.io.PrintStream; import java.net.URI; import java.net.URISyntaxException; -import java.nio.file.AccessDeniedException; import java.util.Arrays; import java.util.Collection; import java.util.Date; @@ -400,9 +399,10 @@ public int run(String[] args, PrintStream out) println(out, "Filesystem %s", fsUri); try { println(out, "Location: %s", fs.getBucketLocation()); - } catch (AccessDeniedException e) { + } catch (IOException e) { // Caller cannot get the location of this bucket due to permissions - // in their role or the bucket itself. + // in their role or the bucket itself, or it is not an operation + // supported by this store. // Note and continue. LOG.debug("failed to get bucket location", e); println(out, LOCATION_UNKNOWN); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java index ef8413ccf0a64..f54ab18650150 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.s3a.AWSBadRequestException; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -679,8 +680,30 @@ private boolean scanDirectoryTree( int count = 0; boolean result = true; - RemoteIterator listing = operations - .listObjects(path, storeContext.pathToKey(path)); + + // the path/key stuff loses any trailing / passed in. + // but this may actually be needed. + RemoteIterator listing = null; + String listkey = storeContext.pathToKey(path); + if (listkey.isEmpty()) { + // root. always give it a path to keep ranger happy. + listkey = "/"; + } + + try { + listing = operations.listObjects(path, listkey); + } catch (AWSBadRequestException e) { + // endpoint was unhappy. this is generally unrecoverable, but some + // third party stores do insist on a / here. + LOG.debug("Failed to list \"{}\"", listkey, e); + // now retry with a trailing / in case that works + if (listkey.endsWith("/")) { + // already has a trailing /, so fail + throw e; + } + // try again. + listing = operations.listObjects(path, listkey + "/"); + } while (listing.hasNext()) { count++; S3AFileStatus status = listing.next(); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index ea53b2e1fa9e3..7da23c9fe7cf2 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -371,16 +371,7 @@ The Assumed Role Credential Provider is enabled, but `fs.s3a.assumed.role.arn` i ``` java.io.IOException: Unset property fs.s3a.assumed.role.arn at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:76) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:423) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) + ``` ### "Not authorized to perform sts:AssumeRole" @@ -390,17 +381,9 @@ or one to which the caller has no access. ``` java.nio.file.AccessDeniedException: : Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider - on : com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: Not authorized to perform sts:AssumeRole (Service: AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied; Request ID: aad4e59a-f4b0-11e7-8c78-f36aaa9457f6):AccessDenied - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:215) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) + ``` ### "Roles may not be assumed by root accounts" @@ -411,31 +394,15 @@ the role. ``` java.nio.file.AccessDeniedException: : Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider - on : com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: + Roles may not be assumed by root accounts. (Service: AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied; Request ID: e86dfd8f-e758-11e7-88e7-ad127c04b5e2): No AWS Credentials provided by AssumedRoleCredentialProvider : - com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: + software.amazon.awssdk.services.sts.model.StsException: Roles may not be assumed by root accounts. (Service: AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied; Request ID: e86dfd8f-e758-11e7-88e7-ad127c04b5e2) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:215) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) - ... 22 more -Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: - Roles may not be assumed by root accounts. - (Service: AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied; - Request ID: e86dfd8f-e758-11e7-88e7-ad127c04b5e2) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717) + + ``` ### `Member must have value greater than or equal to 900` @@ -444,7 +411,7 @@ The value of `fs.s3a.assumed.role.session.duration` is too low. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: request role credentials: -com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: +software.amazon.awssdk.services.sts.model.StsException: 1 validation error detected: Value '20' at 'durationSeconds' failed to satisfy constraint: Member must have value greater than or equal to 900 (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; @@ -459,7 +426,7 @@ The value of `fs.s3a.assumed.role.session.duration` is too high. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: request role credentials: - com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: + software.amazon.awssdk.services.sts.model.StsException: The requested DurationSeconds exceeds the MaxSessionDuration set for this role. (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; Request ID: 17875165-d0a7-11e8-b85f-d15a599a7f6d) @@ -478,7 +445,7 @@ any role for up to 12h; attempting to use a larger number will fail. ``` -Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: +Caused by: software.amazon.awssdk.services.sts.model.StsException: 1 validation error detected: Value '345600' at 'durationSeconds' failed to satisfy constraint: Member must have value less than or equal to 43200 @@ -492,7 +459,7 @@ For full sessions, the duration limit is 129600 seconds: 36h. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: request session credentials: -com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: +software.amazon.awssdk.services.sts.model.StsException: 1 validation error detected: Value '345600' at 'durationSeconds' failed to satisfy constraint: Member must have value less than or equal to 129600 (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; @@ -510,26 +477,12 @@ AWS specification of Role Policies. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on : - com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException: + software.amazon.awssdk.services.sts.model.MalformedPolicyDocumentException: The policy is not in the valid JSON format. (Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c): MalformedPolicyDocument: The policy is not in the valid JSON format. (Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) -Caused by: com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException: - The policy is not in the valid JSON format. - (Service: AWSSecurityTokenService; Status Code: 400; - Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) ``` ### `MalformedPolicyDocumentException` "Syntax errors in policy" @@ -539,27 +492,13 @@ The policy set in `fs.s3a.assumed.role.policy` is not valid JSON. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on : - com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException: + software.amazon.awssdk.services.sts.model.MalformedPolicyDocumentException: Syntax errors in policy. (Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument; Request ID: 24a281e8-f553-11e7-aa91-a96becfb4d45): MalformedPolicyDocument: Syntax errors in policy. (Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument; Request ID: 24a281e8-f553-11e7-aa91-a96becfb4d45) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) - (Service: AWSSecurityTokenService; Status Code: 400; Error Code: MalformedPolicyDocument; - Request ID: 24a281e8-f553-11e7-aa91-a96becfb4d45) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055) - ... 19 more ``` ### `IOException`: "AssumedRoleCredentialProvider cannot be in fs.s3a.assumed.role.credentials.provider" @@ -591,7 +530,7 @@ inner authentication which is breaking signature creation. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider - on : com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: + on : software.amazon.awssdk.services.sts.model.StsException: 'valid/20180109/us-east-1/sts/aws4_request' not a valid key=value pair (missing equal-sign) in Authorization header: 'AWS4-HMAC-SHA256 Credential=not valid/20180109/us-east-1/sts/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date. @@ -601,21 +540,7 @@ inner authentication which is breaking signature creation. in Authorization header: 'AWS4-HMAC-SHA256 Credential=not valid/20180109/us-east-1/sts/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date, (Service: AWSSecurityTokenService; Status Code: 400; Error Code: IncompleteSignature; - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) -Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: - 'valid/20180109/us-east-1/sts/aws4_request' not a valid key=value pair (missing equal-sign) - in Authorization header: 'AWS4-HMAC-SHA256 Credential=not valid/20180109/us-east-1/sts/aws4_request, - SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date, - (Service: AWSSecurityTokenService; Status Code: 400; Error Code: IncompleteSignature; - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) ``` ### `AccessDeniedException/InvalidClientTokenId`: "The security token included in the request is invalid" @@ -625,27 +550,11 @@ The credentials used to authenticate with the AWS Security Token Service are inv ``` [ERROR] Failures: [ERROR] java.nio.file.AccessDeniedException: : Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on : - com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: + software.amazon.awssdk.services.sts.model.StsException: The security token included in the request is invalid. (Service: AWSSecurityTokenService; Status Code: 403; Error Code: InvalidClientTokenId; Request ID: 74aa7f8a-f557-11e7-850c-33d05b3658d7):InvalidClientTokenId - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:215) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - -Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: -The security token included in the request is invalid. - (Service: AWSSecurityTokenService; Status Code: 403; Error Code: InvalidClientTokenId; - Request ID: 74aa7f8a-f557-11e7-850c-33d05b3658d7) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055) - ... 25 more -``` + ``` ### `AWSSecurityTokenServiceExceptiond`: "Member must satisfy regular expression pattern: `[\w+=,.@-]*`" @@ -659,7 +568,7 @@ If set explicitly, it must be valid. ``` org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on - com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: + software.amazon.awssdk.services.sts.model.StsException: 1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName' failed to satisfy constraint: Member must satisfy regular expression pattern: [\w+=,.@-]* (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; @@ -667,23 +576,7 @@ org.apache.hadoop.fs.s3a.AWSBadRequestException: 1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName' failed to satisfy constraint: Member must satisfy regular expression pattern: [\w+=,.@-]* (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:616) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:520) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:52) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:252) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:474) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) - -Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: - 1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName' - failed to satisfy constraint: - Member must satisfy regular expression pattern: [\w+=,.@-]* - (Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError; - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) -``` + ``` ### `java.nio.file.AccessDeniedException` within a FileSystem API call @@ -692,23 +585,11 @@ If an operation fails with an `AccessDeniedException`, then the role does not ha the permission for the S3 Operation invoked during the call. ``` -java.nio.file.AccessDeniedException: s3a://bucket/readonlyDir: - rename(s3a://bucket/readonlyDir, s3a://bucket/renameDest) - on s3a://bucket/readonlyDir: - com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied - (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 2805F2ABF5246BB1; - S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=), - S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=:AccessDenied - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:216) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:143) - at org.apache.hadoop.fs.s3a.S3AFileSystem.rename(S3AFileSystem.java:853) - ... -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied - (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 2805F2ABF5246BB1; - S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=), - S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw= - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) +> hadoop fs -touch s3a://landsat-pds/a + +java.nio.file.AccessDeniedException: a: Writing Object on a: + software.amazon.awssdk.services.s3.model.S3Exception: Access Denied + (Service: S3, Status Code: 403, Request ID: F08EV50F85AYKF1V, Extended Request ID: 75vMz9xWNP5/lYplPSZfm/i4yQ5q0G32SIwOwfaj6a8gNCRj9tLBAqcLaaexT/aNg2DhWZQPvDU=):AccessDenied ``` This is the policy restriction behaving as intended: the caller is trying to @@ -743,28 +624,9 @@ If the client does have write access to the bucket, verify that the caller has ``` java.nio.file.AccessDeniedException: test/testDTFileSystemClient: Writing Object on test/testDTFileSystemClient: - com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403; + software.amazon.awssdk.services.s3.model.S3Exception: Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: E86544FF1D029857) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:243) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:314) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:406) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:310) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:285) - at org.apache.hadoop.fs.s3a.WriteOperationHelper.retry(WriteOperationHelper.java:150) - at org.apache.hadoop.fs.s3a.WriteOperationHelper.putObject(WriteOperationHelper.java:460) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.lambda$putObject$0(S3ABlockOutputStream.java:438) - at org.apache.hadoop.util.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:219) - at org.apache.hadoop.util.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:219) - at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) - at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57) - at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) - at java.lang.Thread.run(Thread.java:748) -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403; - Error Code: AccessDenied; Request ID: E86544FF1D029857) ``` Note: the ability to read encrypted data in the store does not guarantee that the caller can encrypt new data. @@ -779,14 +641,9 @@ This is a low-level networking error. Possible causes include: ``` org.apache.hadoop.fs.s3a.AWSClientIOException: request session credentials: - com.amazonaws.SdkClientException: Unable to execute HTTP request: null: Unable to execute HTTP request: null -at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultRoutePlanner.determineRoute(DefaultRoutePlanner.java:88) -at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.determineRoute(InternalHttpClient.java:124) -at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:183) -at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) -at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) + ``` ### Error "Credential should be scoped to a valid region" @@ -800,7 +657,7 @@ Variant 1: `Credential should be scoped to a valid region, not 'us-west-1'` (or ``` java.nio.file.AccessDeniedException: : request session credentials: -com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: +software.amazon.awssdk.services.sts.model.StsException: Credential should be scoped to a valid region, not 'us-west-1'. (Service: AWSSecurityTokenService; Status Code: 403; Error Code: SignatureDoesNotMatch; Request ID: d9065cc4-e2b9-11e8-8b7b-f35cb8d7aea4):SignatureDoesNotMatch ``` @@ -817,7 +674,7 @@ Variant 2: `Credential should be scoped to a valid region, not ''` ``` java.nio.file.AccessDeniedException: : request session credentials: -com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException: +software.amazon.awssdk.services.sts.model.StsException: Credential should be scoped to a valid region, not ''. ( Service: AWSSecurityTokenService; Status Code: 403; Error Code: SignatureDoesNotMatch; Request ID: bd3e5121-e2ac-11e8-a566-c1a4d66b6a16):SignatureDoesNotMatch diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 382ae36c1bf5b..0730f86cd1a2f 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -35,19 +35,20 @@ full details. * [Encryption](./encryption.html) * [Performance](./performance.html) -* [S3Guard](./s3guard.html) +* [The upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html) +* [Working with Third-party S3 Stores](./third_party_stores.html) * [Troubleshooting](./troubleshooting_s3a.html) +* [Prefetching](./prefetching.html) * [Controlling the S3A Directory Marker Behavior](directory_markers.html). +* [Auditing](./auditing.html). * [Committing work to S3 with the "S3A Committers"](./committers.html) * [S3A Committers Architecture](./committer_architecture.html) * [Working with IAM Assumed Roles](./assumed_roles.html) * [S3A Delegation Token Support](./delegation_tokens.html) * [S3A Delegation Token Architecture](delegation_token_architecture.html). -* [Auditing](./auditing.html). * [Auditing Architecture](./auditing_architecture.html). * [Testing](./testing.html) -* [Prefetching](./prefetching.html) -* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html) +* [S3Guard](./s3guard.html) ## Overview @@ -79,16 +80,15 @@ and compatible implementations. * Supports partitioned uploads for many-GB objects. * Offers a high-performance random IO mode for working with columnar data such as Apache ORC and Apache Parquet files. -* Uses Amazon's Java S3 SDK with support for latest S3 features and authentication +* Uses Amazon's Java V2 SDK with support for latest S3 features and authentication schemes. * Supports authentication via: environment variables, Hadoop configuration properties, the Hadoop key management store and IAM roles. * Supports per-bucket configuration. * Supports S3 "Server Side Encryption" for both reading and writing: SSE-S3, SSE-KMS and SSE-C. +* Supports S3-CSE client side encryption. * Instrumented with Hadoop metrics. -* Before S3 was consistent, provided a consistent view of inconsistent storage - through [S3Guard](./s3guard.html). * Actively maintained by the open source community. @@ -97,19 +97,17 @@ properties, the Hadoop key management store and IAM roles. There other Hadoop connectors to S3. Only S3A is actively maintained by the Hadoop project itself. -1. Apache's Hadoop's original `s3://` client. This is no longer included in Hadoop. 1. Amazon EMR's `s3://` client. This is from the Amazon EMR team, who actively maintain it. -1. Apache's Hadoop's [`s3n:` filesystem client](./s3n.html). - This connector is no longer available: users must migrate to the newer `s3a:` client. + ## Getting Started S3A depends upon two JARs, alongside `hadoop-common` and its dependencies. -* `hadoop-aws` JAR. -* `aws-java-sdk-bundle` JAR. +* `hadoop-aws` JAR. This contains the S3A connector. +* `bundle` JAR. This contains the full shaded AWS V2 SDK. The versions of `hadoop-common` and `hadoop-aws` must be identical. @@ -189,7 +187,8 @@ of work, as opposed to HDFS or other "real" filesystem. The [S3A committers](./committers.html) are the sole mechanism available to safely save the output of queries directly into S3 object stores -through the S3A filesystem. +through the S3A filesystem when the filesystem structure is +how the table is represented. ### Warning #2: Object stores have different authorization models @@ -199,10 +198,7 @@ authorization model of HDFS and traditional file systems. The S3A client simply reports stub information from APIs that would query this metadata: * File owner is reported as the current user. -* File group also is reported as the current user. Prior to Apache Hadoop -2.8.0, file group was reported as empty (no group associated), which is a -potential incompatibility problem for scripts that perform positional parsing of -shell output and other clients that expect to find a well-defined group. +* File group also is reported as the current user. * Directory permissions are reported as 777. * File permissions are reported as 666. @@ -239,11 +235,6 @@ However, with the upcoming upgrade to AWS Java SDK V2, these classes will need t updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). -*Important*: The S3A connector no longer supports username and secrets -in URLs of the form `s3a://key:secret@bucket/`. -It is near-impossible to stop those secrets being logged —which is why -a warning has been printed since Hadoop 2.8 whenever such a URL was used. - ### Authentication properties ```xml @@ -369,13 +360,13 @@ There are a number of AWS Credential Providers inside the `hadoop-aws` JAR: | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | EC2/k8s instance credentials | -There are also many in the Amazon SDKs, with the common ones being. +There are also many in the Amazon SDKs, with the common ones being as follows -| classname | description | -|-----------|-------------| -| `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables | -| `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials | -| `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider`| EC2/k8s Metadata Credentials | +| classname | description | +|----------------------------------------------------------------------------------|------------------------------| +| `software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | AWS Environment Variables | +| `software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider` | EC2 Metadata Credentials | +| `software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider` | EC2/k8s Metadata Credentials | @@ -716,16 +707,145 @@ Here are the S3A properties for use in production; some testing-related options are covered in [Testing](./testing.md). ```xml + + + fs.s3a.aws.credentials.provider + + org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider, + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, + software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider + + + Comma-separated class names of credential provider classes which implement + software.amazon.awssdk.auth.credentials.AwsCredentialsProvider. + + When S3A delegation tokens are not enabled, this list will be used + to directly authenticate with S3 and other AWS services. + When S3A Delegation tokens are enabled, depending upon the delegation + token binding it may be used + to communicate wih the STS endpoint to request session/role + credentials. + + + + + fs.s3a.security.credential.provider.path + + + Optional comma separated list of credential providers, a list + which is prepended to that set in hadoop.security.credential.provider.path + + + + + fs.s3a.assumed.role.arn + + + AWS ARN for the role to be assumed. + Required if the fs.s3a.aws.credentials.provider contains + org.apache.hadoop.fs.s3a.AssumedRoleCredentialProvider + + + + + fs.s3a.assumed.role.session.name + + + Session name for the assumed role, must be valid characters according to + the AWS APIs. + Only used if AssumedRoleCredentialProvider is the AWS credential provider. + If not set, one is generated from the current Hadoop/Kerberos username. + + + + + fs.s3a.assumed.role.policy + + + JSON policy to apply to the role. + Only used if AssumedRoleCredentialProvider is the AWS credential provider. + + + + + fs.s3a.assumed.role.session.duration + 30m + + Duration of assumed roles before a refresh is attempted. + Used when session tokens are requested. + Range: 15m to 1h + + + + + fs.s3a.assumed.role.sts.endpoint + + + AWS Security Token Service Endpoint. + If unset, uses the default endpoint. + Only used if AssumedRoleCredentialProvider is the AWS credential provider. + Used by the AssumedRoleCredentialProvider and in Session and Role delegation + tokens. + + + + + fs.s3a.assumed.role.sts.endpoint.region + + + AWS Security Token Service Endpoint's region; + Needed if fs.s3a.assumed.role.sts.endpoint points to an endpoint + other than the default one and the v4 signature is used. + Used by the AssumedRoleCredentialProvider and in Session and Role delegation + tokens. + + + + + fs.s3a.assumed.role.credentials.provider + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + + List of credential providers to authenticate with the STS endpoint and + retrieve short-lived role credentials. + Only used if AssumedRoleCredentialProvider is the AWS credential provider. + If unset, uses "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider". + + + + + fs.s3a.delegation.token.binding + + + The name of a class to provide delegation tokens support in S3A. + If unset: delegation token support is disabled. + + Note: for job submission to actually collect these tokens, + Kerberos must be enabled. + + Bindings available in hadoop-aws are: + org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding + org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding + org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding + + + fs.s3a.connection.maximum - 15 - Controls the maximum number of simultaneous connections to S3. + 96 + Controls the maximum number of simultaneous connections to S3. + This must be bigger than the value of fs.s3a.threads.max so as to stop + threads being blocked waiting for new HTTPS connections. + Why not equal? The AWS SDK transfer manager also uses these connections. + fs.s3a.connection.ssl.enabled true - Enables or disables SSL connections to S3. + Enables or disables SSL connections to AWS services. + Also sets the default port to use for the s3a proxy settings, + when not explicitly set in fs.s3a.proxy.port. @@ -736,16 +856,6 @@ options are covered in [Testing](./testing.md). - - fs.s3a.endpoint.region - AWS S3 region for a bucket, which bypasses the parsing of - fs.s3a.endpoint to know the region. Would be helpful in avoiding errors - while using privateLink URL and explicitly set the bucket region. - If set to a blank string (or 1+ space), falls back to the - (potentially brittle) SDK region resolution process. - - - fs.s3a.path.style.access false @@ -788,8 +898,14 @@ options are covered in [Testing](./testing.md). fs.s3a.attempts.maximum - 20 - How many times we should retry commands on transient errors. + 5 + + Number of times the AWS client library should retry errors before + escalating to the S3A code: {@value}. + The S3A connector does its own selective retries; the only time the AWS + SDK operations are not wrapped is during multipart copy via the AWS SDK + transfer manager. + @@ -804,20 +920,6 @@ options are covered in [Testing](./testing.md). Socket connection timeout in milliseconds. - - fs.s3a.paging.maximum - 5000 - How many keys to request from S3 when doing - directory listings at a time. - - - - fs.s3a.threads.max - 10 - Maximum number of concurrent active (part)uploads, - which each use a thread from the threadpool. - - fs.s3a.socket.send.buffer 8192 @@ -830,6 +932,20 @@ options are covered in [Testing](./testing.md). Socket receive buffer hint to amazon connector. Represented in bytes. + + fs.s3a.paging.maximum + 5000 + How many keys to request from S3 when doing + directory listings at a time. + + + + fs.s3a.threads.max + 64 + The total number of threads available in the filesystem for data + uploads *or any other queued filesystem operation*. + + fs.s3a.threads.keepalivetime 60 @@ -839,9 +955,25 @@ options are covered in [Testing](./testing.md). fs.s3a.max.total.tasks - 5 - Number of (part)uploads allowed to the queue before - blocking additional uploads. + 32 + The number of operations which can be queued for execution. + This is in addition to the number of active threads in fs.s3a.threads.max. + + + + + fs.s3a.executor.capacity + 16 + The maximum number of submitted tasks which is a single + operation (e.g. rename(), delete()) may submit simultaneously for + execution -excluding the IO-heavy block uploads, whose capacity + is set in "fs.s3a.fast.upload.active.blocks" + + All tasks are submitted to the shared thread pool whose size is + set in "fs.s3a.threads.max"; the value of capacity should be less than that + of the thread pool itself, as the goal is to stop a single operation + from overloading that thread pool. + @@ -854,7 +986,7 @@ options are covered in [Testing](./testing.md). fs.s3a.multipart.threshold - 128MB + 128M How big (in bytes) to split upload or copy operations up into. This also controls the partition size in renamed files, as rename() involves copying the source file(s). @@ -874,23 +1006,52 @@ options are covered in [Testing](./testing.md). fs.s3a.acl.default Set a canned ACL for newly created and copied objects. Value may be Private, - PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead, - or BucketOwnerFullControl. + PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead, + or BucketOwnerFullControl. If set, caller IAM role must have "s3:PutObjectAcl" permission on the bucket. - + fs.s3a.multipart.purge false True if you want to purge existing multipart uploads that may not have been - completed/aborted correctly + completed/aborted correctly. The corresponding purge age is defined in + fs.s3a.multipart.purge.age. + If set, when the filesystem is instantiated then all outstanding uploads + older than the purge age will be terminated -across the entire bucket. + This will impact multipart uploads by other applications and users. so should + be used sparingly, with an age value chosen to stop failed uploads, without + breaking ongoing operations. + fs.s3a.multipart.purge.age 86400 - Minimum age in seconds of multipart uploads to purge + Minimum age in seconds of multipart uploads to purge + on startup if "fs.s3a.multipart.purge" is true + + + + + fs.s3a.encryption.algorithm + Specify a server-side encryption or client-side + encryption algorithm for s3a: file system. Unset by default. It supports the + following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS' + + + + + fs.s3a.encryption.key + Specific encryption key to use if fs.s3a.encryption.algorithm + has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C + , the value of this property should be the Base64 encoded key. If you are + using SSE-KMS and leave this property empty, you'll be using your default's + S3 KMS key, otherwise you should set this property to the specific KMS key + id. In case of 'CSE-KMS' this value needs to be the AWS-KMS Key ID + generated from AWS console. + @@ -900,23 +1061,19 @@ options are covered in [Testing](./testing.md). - fs.s3a.encryption.algorithm - Specify a server-side encryption or client-side - encryption algorithm for s3a: file system. Unset by default. It supports the - following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS' - + fs.s3a.accesspoint.required + false + Require that all S3 access is made through Access Points and not through + buckets directly. If enabled, use per-bucket overrides to allow bucket access to a specific set + of buckets. - fs.s3a.encryption.key - Specific encryption key to use if fs.s3a.encryption.algorithm - has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C - , the value of this property should be the Base64 encoded key. If you are - using SSE-KMS and leave this property empty, you'll be using your default's - S3 KMS key, otherwise you should set this property to the specific KMS key - id. In case of 'CSE-KMS' this value needs to be the AWS-KMS Key ID - generated from AWS console. - + fs.s3a.block.size + 32M + Block size to use when reading files using s3a: file system. + A suffix from the set {K,M,G,T,P} may be used to scale the numeric value. + @@ -930,9 +1087,51 @@ options are covered in [Testing](./testing.md). - fs.s3a.block.size - 32M - Block size to use when reading files using s3a: file system. + fs.s3a.fast.upload.buffer + disk + + The buffering mechanism to for data being written. + Values: disk, array, bytebuffer. + + "disk" will use the directories listed in fs.s3a.buffer.dir as + the location(s) to save data prior to being uploaded. + + "array" uses arrays in the JVM heap + + "bytebuffer" uses off-heap memory within the JVM. + + Both "array" and "bytebuffer" will consume memory in a single stream up to the number + of blocks set by: + + fs.s3a.multipart.size * fs.s3a.fast.upload.active.blocks. + + If using either of these mechanisms, keep this value low + + The total number of threads performing work across all threads is set by + fs.s3a.threads.max, with fs.s3a.max.total.tasks values setting the number of queued + work items. + + + + + fs.s3a.fast.upload.active.blocks + 4 + + Maximum Number of blocks a single output stream can have + active (uploading, or queued to the central FileSystem + instance's pool of queued operations. + + This stops a single stream overloading the shared thread pool. + + + + + fs.s3a.readahead.range + 64K + Bytes to read ahead during a seek() before closing and + re-opening the S3 HTTP connection. This option will be overridden if + any call to setReadahead() is made to an open stream. + A suffix from the set {K,M,G,T,P} may be used to scale the numeric value. @@ -958,118 +1157,232 @@ options are covered in [Testing](./testing.md). - fs.AbstractFileSystem.s3a.impl - org.apache.hadoop.fs.s3a.S3A - The implementation class of the S3A AbstractFileSystem. + fs.s3a.retry.limit + 7 + + Number of times to retry any repeatable S3 client request on failure, + excluding throttling requests. + - fs.s3a.readahead.range - 64K - Bytes to read ahead during a seek() before closing and - re-opening the S3 HTTP connection. This option will be overridden if - any call to setReadahead() is made to an open stream. + fs.s3a.retry.interval + 500ms + + Initial retry interval when retrying operations for any reason other + than S3 throttle errors. + - fs.s3a.input.async.drain.threshold - 64K - Bytes to read ahead during a seek() before closing and - re-opening the S3 HTTP connection. This option will be overridden if - any call to setReadahead() is made to an open stream. + fs.s3a.retry.throttle.limit + 20 + + Number of times to retry any throttled request. + + + + + fs.s3a.retry.throttle.interval + 100ms + + Initial between retry attempts on throttled requests, +/- 50%. chosen at random. + i.e. for an intial value of 3000ms, the initial delay would be in the range 1500ms to 4500ms. + Backoffs are exponential; again randomness is used to avoid the thundering heard problem. + 500ms is the default value used by the AWS S3 Retry policy. + + + + + fs.s3a.committer.name + file + + Committer to create for output to S3A, one of: + "file", "directory", "partitioned", "magic". + + + + + fs.s3a.committer.magic.enabled + true + + Enable support in the S3A filesystem for the "Magic" committer. + + + + + fs.s3a.committer.threads + 8 + + Number of threads in committers for parallel operations on files + (upload, commit, abort, delete...) + + + + + fs.s3a.committer.staging.tmp.path + tmp/staging + + Path in the cluster filesystem for temporary data. + This is for HDFS, not the local filesystem. + It is only for the summary data of each file, not the actual + data being committed. + Using an unqualified path guarantees that the full path will be + generated relative to the home directory of the user creating the job, + hence private (assuming home directory permissions are secure). + + + + + fs.s3a.committer.staging.unique-filenames + true + + Option for final files to have a unique name through job attempt info, + or the value of fs.s3a.committer.staging.uuid + When writing data with the "append" conflict option, this guarantees + that new data will not overwrite any existing data. + + + + + fs.s3a.committer.staging.conflict-mode + append + + Staging committer conflict resolution policy. + Supported: "fail", "append", "replace". + + + + + fs.s3a.committer.abort.pending.uploads + true + + Should the committers abort all pending uploads to the destination + directory? + + Set to false if more than one job is writing to the same directory tree. + fs.s3a.list.version 2 - Select which version of the S3 SDK's List Objects API to use. - Currently support 2 (default) and 1 (older API). + + Select which version of the S3 SDK's List Objects API to use. Currently + support 2 (default) and 1 (older API). + fs.s3a.connection.request.timeout 0 - Time out on HTTP requests to the AWS service; 0 means no timeout. - Measured in seconds; the usual time suffixes are all supported + Time out on HTTP requests to the AWS service; 0 means no timeout. + Measured in seconds; the usual time suffixes are all supported - Important: this is the maximum duration of any AWS service call, - including upload and copy operations. If non-zero, it must be larger - than the time to upload multi-megabyte blocks to S3 from the client, - and to rename many-GB files. Use with care. + Important: this is the maximum duration of any AWS service call, + including upload and copy operations. If non-zero, it must be larger + than the time to upload multi-megabyte blocks to S3 from the client, + and to rename many-GB files. Use with care. - Values that are larger than Integer.MAX_VALUE milliseconds are - converged to Integer.MAX_VALUE milliseconds + Values that are larger than Integer.MAX_VALUE milliseconds are + converged to Integer.MAX_VALUE milliseconds - fs.s3a.bucket.probe - 0 + fs.s3a.etag.checksum.enabled + false - The value can be 0 (default), 1 or 2. - When set to 0, bucket existence checks won't be done - during initialization thus making it faster. - Though it should be noted that when the bucket is not available in S3, - or if fs.s3a.endpoint points to the wrong instance of a private S3 store - consecutive calls like listing, read, write etc. will fail with - an UnknownStoreException. - When set to 1, the bucket existence check will be done using the - V1 API of the S3 protocol which doesn't verify the client's permissions - to list or read data in the bucket. - When set to 2, the bucket existence check will be done using the - V2 API of the S3 protocol which does verify that the - client has permission to read the bucket. + Should calls to getFileChecksum() return the etag value of the remote + object. + WARNING: if enabled, distcp operations between HDFS and S3 will fail unless + -skipcrccheck is set. - fs.s3a.object.content.encoding - + fs.s3a.change.detection.source + etag - Content encoding: gzip, deflate, compress, br, etc. - This will be set in the "Content-Encoding" header of the object, - and returned in HTTP HEAD/GET requests. + Select which S3 object attribute to use for change detection. + Currently support 'etag' for S3 object eTags and 'versionid' for + S3 object version IDs. Use of version IDs requires object versioning to be + enabled for each S3 bucket utilized. Object versioning is disabled on + buckets by default. When version ID is used, the buckets utilized should + have versioning enabled before any data is written. - fs.s3a.create.storage.class - + fs.s3a.change.detection.mode + server - Storage class: standard, reduced_redundancy, intelligent_tiering, etc. - Specify the storage class for S3A PUT object requests. - If not set the storage class will be null - and mapped to default standard class on S3. + Determines how change detection is applied to alert to inconsistent S3 + objects read during or after an overwrite. Value 'server' indicates to apply + the attribute constraint directly on GetObject requests to S3. Value 'client' + means to do a client-side comparison of the attribute value returned in the + response. Value 'server' would not work with third-party S3 implementations + that do not support these constraints on GetObject. Values 'server' and + 'client' generate RemoteObjectChangedException when a mismatch is detected. + Value 'warn' works like 'client' but generates only a warning. Value 'none' + will ignore change detection completely. - fs.s3a.prefetch.enabled - false + fs.s3a.change.detection.version.required + true + + Determines if S3 object version attribute defined by + fs.s3a.change.detection.source should be treated as required. If true and the + referred attribute is unavailable in an S3 GetObject response, + NoVersionAttributeException is thrown. Setting to 'true' is encouraged to + avoid potential for inconsistent reads with third-party S3 implementations or + against S3 buckets that have object versioning disabled. + + + + + fs.s3a.ssl.channel.mode + default_jsse - Enables prefetching and caching when reading from input stream. + If secure connections to S3 are enabled, configures the SSL + implementation used to encrypt connections to S3. Supported values are: + "default_jsse", "default_jsse_with_gcm", "default", and "openssl". + "default_jsse" uses the Java Secure Socket Extension package (JSSE). + However, when running on Java 8, the GCM cipher is removed from the list + of enabled ciphers. This is due to performance issues with GCM in Java 8. + "default_jsse_with_gcm" uses the JSSE with the default list of cipher + suites. "default_jsse_with_gcm" is equivalent to the behavior prior to + this feature being introduced. "default" attempts to use OpenSSL rather + than the JSSE for SSL encryption, if OpenSSL libraries cannot be loaded, + it falls back to the "default_jsse" behavior. "openssl" attempts to use + OpenSSL as well, but fails if OpenSSL libraries cannot be loaded. - fs.s3a.prefetch.block.size - 8MB + fs.s3a.downgrade.syncable.exceptions + true - The size of a single prefetched block of data. - Decreasing this will increase the number of prefetches required, and may negatively impact performance. + Warn but continue when applications use Syncable.hsync when writing + to S3A. + - fs.s3a.prefetch.block.count - 8 + fs.s3a.audit.enabled + true - Maximum number of blocks prefetched concurrently at any given time. + Should auditing of S3A requests be enabled? -``` +``` ## Retry and Recovery The S3A client makes a best-effort attempt at recovering from network failures; @@ -1089,8 +1402,10 @@ not the failing operation is idempotent. * Interruptions: `InterruptedIOException`, `InterruptedException`. * Rejected HTTP requests: `InvalidRequestException` -These are all considered unrecoverable: S3A will make no attempt to recover -from them. +These and others are all considered unrecoverable: S3A will make no attempt to recover +from them. The AWS SDK itself may retry before the S3A connector sees the exception. +As an example, the SDK will retry on `UnknownHostException` in case it is a transient +DNS error. ### Possibly Recoverable Problems: Retry @@ -1141,17 +1456,9 @@ only succeed if the first `delete()` call has already succeeded. Because S3 is eventually consistent *and* doesn't support an atomic create-no-overwrite operation, the choice is more ambiguous. -Currently S3A considers delete to be -idempotent because it is convenient for many workflows, including the -commit protocols. Just be aware that in the presence of transient failures, -more things may be deleted than expected. (For anyone who considers this to -be the wrong decision: rebuild the `hadoop-aws` module with the constant -`S3AFileSystem.DELETE_CONSIDERED_IDEMPOTENT` set to `false`). - - - - - +S3A considers delete to be idempotent because it is convenient for many workflows, +including the commit protocols. Just be aware that in the presence of transient failures, +more things may be deleted than expected. ### Throttled requests from S3 @@ -1657,13 +1964,17 @@ the storage class you want. ``` Please note that S3A does not support reading from archive storage classes at the moment. -`AccessDeniedException` with InvalidObjectState will be thrown if you're trying to do so. +`AccessDeniedException` with `InvalidObjectState` will be thrown if you're trying to do so. + +When a file is "renamed" through the s3a connector it is copied then deleted. +Storage Classes will normally be propagated. + -## Configuring S3A for S3 on Outposts +## Configuring S3A for S3 on Outposts S3A now supports [S3 on Outposts](https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html). Accessing data through an access point is done by using its Amazon Resource Name (ARN), as opposed to just the bucket name. -The only supported storage class on Outposts is **OUTPOSTS**, and by default objects are encrypted with [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-outposts-data-encryption.html). +The only supported storage class on Outposts is `OUTPOSTS`, and by default objects are encrypted with [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-outposts-data-encryption.html). You can set the Access Point ARN property using the following per bucket configuration property: ```xml diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index bfec94b19c101..6100bc0ae5c95 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -379,6 +379,19 @@ This adds extra overhead to every operation, but helps verify that the connector not keeping markers where it needs to be deleting them -and hence backwards compatibility is maintained. +## Enabling prefetch for all tests + +The tests are run with prefetch if the `prefetch` property is set in the +maven build. This can be combined with the scale tests as well. + +```bash +mvn verify -Dprefetch + +mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8 + +mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8 +``` + ## Scale Tests There are a set of tests designed to measure the scalability and performance @@ -521,7 +534,7 @@ Otherwise, set a large timeout in `fs.s3a.scale.test.timeout` The tests are executed in an order to only clean up created files after the end of all the tests. If the tests are interrupted, the test data will remain. -## Load tests. +## Load tests. Some are designed to overload AWS services with more requests per second than an AWS account is permitted. @@ -547,6 +560,32 @@ which address issues. In particular, we encourage testing of Hadoop release candidates, as these third-party endpoints get even less testing than the S3 endpoint itself. +The core XML settings to turn off tests of features unavailable +on third party stores. + +```xml + + test.fs.s3a.encryption.enabled + false + + + test.fs.s3a.create.storage.class.enabled + false + + + fs.s3a.select.enabled + false + + + test.fs.s3a.sts.enabled + false + + + test.fs.s3a.create.create.acl.enabled + false + < /property> +``` + ### Public datasets used in tests Some tests rely on the presence of existing public datasets available on Amazon S3. @@ -587,7 +626,7 @@ S3 storage class, these tests might fail. They can be disabled. ``` -### Configuring the CSV file read tests** +### Configuring the CSV file read tests To test on alternate infrastructures supporting the same APIs, the option `fs.s3a.scale.test.csvfile` must either be @@ -620,19 +659,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with a meaningful error ("S3 Select not supported") rather than a generic Bad Request exception. -### Enabling prefetch for all tests +### Disabling V1 List API tests -The tests are run with prefetch if the `prefetch` property is set in the -maven build. This can be combined with the scale tests as well. -```bash -mvn verify -Dprefetch - -mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8 - -mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8 +If `ITestS3AContractGetFileStatusV1List` fails with any error about unsupported API. +```xml + + test.fs.s3a.list.v1.enabled + false + ``` +Note: there's no equivalent for turning off v2 listing API, which all stores are now +expected to support. + ### Testing Requester Pays @@ -699,6 +739,20 @@ The default is ""; meaning "use the amazon default endpoint" (`sts.amazonaws.com Consult the [AWS documentation](https://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region) for the full list of locations. +### Disabling Content Encoding tests + +Tests in `ITestS3AContentEncoding` may need disabling +```xml + + test.fs.s3a.content.encoding.enabled + false + +``` +### Tests which may fail (and which you can ignore) + +* `ITestS3AContractMultipartUploader` tests `testMultipartUploadAbort` and `testSingleUpload` raising `FileNotFoundException` +* `ITestS3AMiscOperations.testEmptyFileChecksums`: if the FS encrypts data always. + ## Debugging Test failures Logging at debug level is the standard way to provide more diagnostics output; @@ -945,7 +999,7 @@ sequential one afterwards. The IO heavy ones must also be subclasses of `S3AScaleTestBase` and so only run if the system/maven property `fs.s3a.scale.test.enabled` is true. -## Individual test cases can be run in an IDE +### Individual test cases can be run in an IDE This is invaluable for debugging test failures. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md new file mode 100644 index 0000000000000..a7ea7b2e59024 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md @@ -0,0 +1,415 @@ + + +# Working with Third-party S3 Stores + +The S3A connector works well with third-party S3 stores if the following requirements are met: + +* It correctly implements the core S3 REST API, including support for uploads and the V2 listing API. +* The store supports the AWS V4 signing API *or* a custom signer is switched to. + This release does not support the legacy v2 signing API. +* Errors are reported with the same HTTPS status codes as the S3 store. Error messages do not + need to be consistent. +* The store is consistent. + +There are also specific deployment requirements: +* The clock on the store and the client are close enough that signing works. +* The client is correctly configured to connect to the store *and not use unavailable features* +* If HTTPS authentication is used, the client/JVM TLS configurations allows it to authenticate the endpoint. + +The features which may be unavailable include: + +* Checksum-based server-side change detection during copy/read (`fs.s3a.change.detection.mode=server`) +* Object versioning and version-based change detection (`fs.s3a.change.detection.source=versionid` and `fs.s3a.versioned.store=true`) +* Bulk delete (`fs.s3a.multiobjectdelete.enable=true`) +* Encryption. (`fs.s3a.encryption.algorithm`) +* Storage class set in `fs.s3a.create.storage.class` +* Content encodings as set with `fs.s3a.object.content.encoding`. +* Optional Bucket Probes at startup (`fs.s3a.bucket.probe = 0`). + This is now the default -do not change it. +* List API to use (`fs.s3a.list.version = 1`) + +## Configuring s3a to connect to a third party store + + +### Connecting to a third party object store over HTTPS + +The core setting for a third party store is to change the endpoint in `fs.s3a.endpoint`. + +This can be a URL or a hostname/hostname prefix +For third-party stores without virtual hostname support, providing the URL is straightforward; +path style access must also be enabled in `fs.s3a.path.style.access`. + +The v4 signing algorithm requires a region to be set in `fs.s3a.endpoint.region`. +A non-empty value is generally sufficient, though some deployments may require +a specific value. + +Finally, assuming the credential source is the normal access/secret key +then these must be set, either in XML or (preferred) in a JCEKS file. + +```xml + + + fs.s3a.endpoint + https://storeendpoint.example.com + + + + fs.s3a.path.style.access + true + + + + fs.s3a.endpoint.region + anything + + + + fs.s3a.access.key + 13324445 + + + + fs.s3a.secret.key + 4C6B906D-233E-4E56-BCEA-304CC73A14F8 + + +``` + +If per-bucket settings are used here, then third-party stores and credentials may be used alongside an AWS store. + +# Troubleshooting + +The most common problem when talking to third-party stores are + +1. The S3A client is still configured to talk to the AWS S3 endpoint. This leads to authentication failures and/or reports that the bucket is unknown. +2. Path access has not been enabled, the client is generating a host name for the target bucket and it does not exist. +3. Invalid authentication credentials. +4. JVM HTTPS settings include the certificates needed to negotiate a TLS connection with the store. + + +## How to improve troubleshooting + +### log more network info + +There are some very low level logs. +```properties +# Log all HTTP requests made; includes S3 interaction. This may +# include sensitive information such as account IDs in HTTP headers. +log4j.logger.software.amazon.awssdk.request=DEBUG + +# Turn on low level HTTP protocol debugging +log4j.logger.org.apache.http.wire=DEBUG + +# async client +log4j.logger.io.netty.handler.logging=DEBUG +log4j.logger.io.netty.handler.codec.http2.Http2FrameLogger=DEBUG +``` + +### Cut back on retries, shorten timeouts + +By default, there's a lot of retries going on in the AWS connector (which even retries on DNS failures) +and in the S3A code which invokes it. + +Normally this helps prevent long-lived jobs from failing due to a transient network problem, however +it means that when trying to debug connectivity problems, the commands can hang for a long time +as they keep trying to reconnect to ports which are never going to be available. + +```xml + + + fs.iostatistics.logging.level + info + + + + fs.s3a.bucket.nonexistent-bucket-example.attempts.maximum + 0 + + + + fs.s3a.bucket.nonexistent-bucket-example.retry.limit + 1 + + + + fs.s3a.bucket.nonexistent-bucket-example.connection.timeout + 500 + + + + fs.s3a.bucket.nonexistent-bucket-example.connection.establish.timeout + 500 + +``` +## Cloudstore's Storediag + +There's an external utility, [cloudstore](https://github.com/steveloughran/cloudstore) whose [storediag](https://github.com/steveloughran/cloudstore#command-storediag) exists to debug the connection settings to hadoop cloud storage. + +```bash +hadoop jar cloudstore-1.0.jar storediag s3a://nonexistent-bucket-example/ +``` + +The main reason it's not an ASF release is that it allows for a rapid release cycle, sometimes hours; if anyone doesn't trust +third-party code then they can download and build it themselves. + + +# Problems + +## S3A client still pointing at AWS endpoint + +This is the most common initial problem, as it happens by default. + +To fix, set `fs.s3a.endpoint` to the URL of the internal store. + +### `org.apache.hadoop.fs.s3a.UnknownStoreException: `s3a://nonexistent-bucket-example/': Bucket does not exist` + +Either the bucket doesn't exist, or the bucket does exist but the endpoint is still set to an AWS endpoint. + +``` +stat: `s3a://nonexistent-bucket-example/': Bucket does not exist +``` +The hadoop filesystem commands don't log stack traces on failure -adding this adds too much risk +of breaking scripts, and the output is very uninformative + +``` +stat: nonexistent-bucket-example: getS3Region on nonexistent-bucket-example: +software.amazon.awssdk.services.s3.model.S3Exception: null +(Service: S3, Status Code: 403, Request ID: X26NWV0RJ1697SXF, Extended Request ID: bqq0rRm5Bdwt1oHSfmWaDXTfSOXoYvNhQxkhjjNAOpxhRaDvWArKCFAdL2hDIzgec6nJk1BVpJE=):null +``` + +It is possible to turn on debugging + +``` +log4j.logger.org.apache.hadoop.fs.shell=DEBUG +``` + +After which useful stack traces are logged. + +``` +org.apache.hadoop.fs.s3a.UnknownStoreException: `s3a://nonexistent-bucket-example/': Bucket does not exist + at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$null$3(S3AFileSystem.java:1075) + at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122) + at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376) + at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468) + at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372) + at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347) + at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getS3Region$4(S3AFileSystem.java:1039) + at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:543) + at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:524) + at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:445) + at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2631) + at org.apache.hadoop.fs.s3a.S3AFileSystem.getS3Region(S3AFileSystem.java:1038) + at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:982) + at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:622) + at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3452) +``` + +### `S3Exception: null (Service: S3, Status Code: 403...` or `AccessDeniedException` + +* Endpoint is default +* Credentials were not issued by AWS . +* `fs.s3a.endpoint.region` unset. + +If the client doesn't have any AWS credentials (from hadoop settings, environment variables or elsewhere) then +the binding will fail even before the existence of the bucket can be probed for. + +```bash +hadoop fs -stat s3a://nonexistent-bucket-example +``` + + +``` +stat: nonexistent-bucket-example: getS3Region on nonexistent-bucket-example: +software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 403, + Request ID: X26NWV0RJ1697SXF, Extended Request ID: bqq0rRm5Bdwt1oHSfmWaDXTfSOXoYvNhQxkhjjNAOpxhRaDvWArKCFAdL2hDIzgec6nJk1BVpJE=):null +``` + +Or with a more detailed stack trace: + +``` +java.nio.file.AccessDeniedException: nonexistent-bucket-example: getS3Region on nonexistent-bucket-example: software.amazon.awssdk.services.s3.model.S3Exception: null (Service: S3, Status Code: 403, Request ID: X26NWV0RJ1697SXF, Extended Request ID: bqq0rRm5Bdwt1oHSfmWaDXTfSOXoYvNhQxkhjjNAOpxhRaDvWArKCFAdL2hDIzgec6nJk1BVpJE=):null + at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:235) + at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:124) + at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376) + at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468) + at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372) + at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347) + at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getS3Region$4(S3AFileSystem.java:1039) + at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:543) + at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:524) + at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:445) + at org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2631) + at org.apache.hadoop.fs.s3a.S3AFileSystem.getS3Region(S3AFileSystem.java:1038) + at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:982) + at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:622) + at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3452) +``` + +## `Received an UnknownHostException when attempting to interact with a service` + + +### Hypothesis 1: Region set, but not endpoint + +The bucket `fs.s3a.endpoint.region` region setting is valid internally, but as the endpoint +is still AWS, this region is not recognised. +The S3A client's creation of an endpoint URL generates an unknown host. + +```xml + + fs.s3a.bucket.nonexistent-bucket-example.endpoint.region + internal + +``` + + +``` +ls: software.amazon.awssdk.core.exception.SdkClientException: + Received an UnknownHostException when attempting to interact with a service. + See cause for the exact endpoint that is failing to resolve. + If this is happening on an endpoint that previously worked, there may be + a network connectivity issue or your DNS cache could be storing endpoints for too long.: + nonexistent-bucket-example.s3.internal.amazonaws.com: nodename nor servname provided, or not known + + +``` + +### Hypothesis 2: region set, endpoint set, but `fs.s3a.path.style.access` is still set to `false` + +* The bucket `fs.s3a.endpoint.region` region setting is valid internally, +* and `fs.s3a.endpoint` is set to a hostname (not a URL). +* `fs.s3a.path.style.access` set to `false` + +``` +ls: software.amazon.awssdk.core.exception.SdkClientException: + Received an UnknownHostException when attempting to interact with a service. + See cause for the exact endpoint that is failing to resolve. + If this is happening on an endpoint that previously worked, there may be + a network connectivity issue or your DNS cache could be storing endpoints for too long.: + nonexistent-bucket-example.localhost: nodename nor servname provided, or not known +``` + +Fix: path style access + +```xml + + fs.s3a.bucket.nonexistent-bucket-example.path.style.access + true + +``` + +# Connecting to Google Cloud Storage through the S3A connector + +It *is* possible to connect to google cloud storage through the S3A connector. +However, Google provide their own [Cloud Storage connector](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage). +That is a well maintained Hadoop filesystem client which uses their XML API, +And except for some very unusual cases, that is the connector to use. + +When interacting with a GCS container through the S3A connector may make sense +* The installation doesn't have the gcs-connector JAR. +* The different credential mechanism may be convenient. +* There's a desired to use S3A Delegation Tokens to pass secrets with a job. +* There's a desire to use an external S3A extension (delegation tokens etc.) + +The S3A connector binding works through the Google Cloud [S3 Storage API](https://cloud.google.com/distributed-cloud/hosted/docs/ga/gdch/apis/storage-s3-rest-api), +which is a subset of the AWS API. + + +To get a compatible access and secret key, follow the instructions of +[Simple migration from Amazon S3 to Cloud Storage](https://cloud.google.com/storage/docs/aws-simple-migration#defaultproj). + +Here are the per-bucket setings for an example bucket "gcs-container" +in Google Cloud Storage. Note the multiobject delete option must be disabled; +this makes renaming and deleting significantly slower. + + +```xml + + + + fs.s3a.bucket.gcs-container.access.key + GOOG1EZ.... + + + + fs.s3a.bucket.gcs-container.secret.key + SECRETS + + + + fs.s3a.bucket.gcs-container.endpoint + https://storage.googleapis.com + + + + fs.s3a.bucket.gcs-container.bucket.probe + 0 + + + + fs.s3a.bucket.gcs-container.list.version + 1 + + + + fs.s3a.bucket.gcs-container.multiobjectdelete.enable + false + + + + fs.s3a.bucket.gcs-container.select.enabled + false + + + + fs.s3a.bucket.gcs-container.path.style.access + true + + + + fs.s3a.bucket.gcs-container.endpoint.region + dummy + + + +``` + +This is a very rarely used configuration -however, it can be done, possibly as a way to interact with Google Cloud Storage in a deployment +which lacks the GCS connector. + +It is also a way to regression test foundational S3A third-party store compatibility if you lack access to to any alternative. + +```xml + + + test.fs.s3a.encryption.enabled + false + + + fs.s3a.scale.test.csvfile + + + + test.fs.s3a.sts.enabled + false + + + test.fs.s3a.content.encoding.enabled + false + + +``` + +_Note_ If anyone is set up to test this reguarly, please let the hadoop developer team know if regressions do surface, +as it is not a common test configuration. \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 41351cdf1bdf0..8c57db1faf4c1 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -71,28 +71,38 @@ An exception reporting this class as missing means that this JAR is not on the classpath. -### `NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest` - -The library `aws-crt.jar` is not on the classpath. Its classes -are not in the AWS `bundle.jar` file, yet may be needed by some uses made -of the SDK. - -Fix: add. - -``` -java.lang.BootstrapMethodError: java.lang.NoClassDefFoundError: software/amazon/awssdk/crt/s3/S3MetaRequest -at software.amazon.awssdk.services.s3.internal.crt.S3MetaRequestPauseObservable.(S3MetaRequestPauseObservable.java:33) -at software.amazon.awssdk.transfer.s3.internal.DefaultS3TransferManager.uploadFile(DefaultS3TransferManager.java:205) -at org.apache.hadoop.fs.s3a.S3AFileSystem.putObject(S3AFileSystem.java:3064) -at org.apache.hadoop.fs.s3a.S3AFileSystem.executePut(S3AFileSystem.java:4054) - -``` -### `ClassNotFoundException: software.amazon.awssdk.services.s3.S3Client` - -(or other `software.amazon` class.) - -This means that the AWS V2 SDK `bundle.jar` JAR is not on the classpath: -add it. +### `java.lang.NoClassDefFoundError: software/amazon/awssdk/services/s3/model/S3Exception` + +This is one of the first stack traces which can surface when trying to instantiate +an S3A filesystem instance without having the AWS V2 SDK `bundle.jar` on the classpath + +``` +java.lang.NoClassDefFoundError: software/amazon/awssdk/services/s3/model/S3Exception + at java.lang.ClassLoader.defineClass1(Native Method) + at java.lang.ClassLoader.defineClass(ClassLoader.java:756) + at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) + at java.net.URLClassLoader.defineClass(URLClassLoader.java:473) + at java.net.URLClassLoader.access$100(URLClassLoader.java:74) + at java.net.URLClassLoader$1.run(URLClassLoader.java:369) + at java.net.URLClassLoader$1.run(URLClassLoader.java:363) + at java.security.AccessController.doPrivileged(Native Method) + at java.net.URLClassLoader.findClass(URLClassLoader.java:362) + at java.lang.ClassLoader.loadClass(ClassLoader.java:418) + at java.lang.ClassLoader.loadClass(ClassLoader.java:351) + at java.lang.Class.forName0(Native Method) + at java.lang.Class.forName(Class.java:348) + at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:2639) + at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2604) + at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2700) + at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3414) + at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3449) + at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:162) +``` + +Fix: add it to classpath. + +Maven/Ivy/SBT/Gradle builds which import `hadoop-aws` or +`hadoop-cloud-storage` artifacts should get the artifact automatically. ### `ClassNotFoundException: com.amazonaws.auth.AWSCredentials` @@ -152,7 +162,6 @@ the client retries a number of times before eventually failing. When it finally gives up, it will report a message about signature mismatch: ``` -com.amazonaws.services.s3.model.AmazonS3Exception: The request signature we calculated does not match the signature you provided. Check your key and signing method. (Service: Amazon S3; Status Code: 403; Error Code: SignatureDoesNotMatch, @@ -229,53 +238,18 @@ read requests are allowed, but operations which write to the bucket are denied. Check the system clock. -### "Bad Request" exception when working with AWS S3 Frankfurt, Seoul, or other "V4" endpoint - +### "Bad Request" exception when working with data stores in an AWS region other than us-eaast -S3 Frankfurt and Seoul *only* support -[the V4 authentication API](http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html). -Requests using the V2 API will be rejected with 400 `Bad Request` ``` $ bin/hadoop fs -ls s3a://frankfurt/ WARN s3a.S3AFileSystem: Client: Amazon S3 error 400: 400 Bad Request; Bad Request (retryable) -com.amazonaws.services.s3.model.AmazonS3Exception: Bad Request (Service: Amazon S3; - Status Code: 400; Error Code: 400 Bad Request; Request ID: 923C5D9E75E44C06), - S3 Extended Request ID: HDwje6k+ANEeDsM6aJ8+D5gUmNAMguOk2BvZ8PH3g9z0gpH+IuwT7N19oQOnIr5CIx7Vqb/uThE= - at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182) - at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770) - at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785) - at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1107) - at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1070) - at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:307) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:284) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2793) - at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:101) - at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2830) - at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2812) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:389) - at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356) - at org.apache.hadoop.fs.shell.PathData.expandAsGlob(PathData.java:325) - at org.apache.hadoop.fs.shell.Command.expandArgument(Command.java:235) - at org.apache.hadoop.fs.shell.Command.expandArguments(Command.java:218) - at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:103) - at org.apache.hadoop.fs.shell.Command.run(Command.java:165) - at org.apache.hadoop.fs.FsShell.run(FsShell.java:315) - at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) - at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:90) - at org.apache.hadoop.fs.FsShell.main(FsShell.java:373) -ls: doesBucketExist on frankfurt-new: com.amazonaws.services.s3.model.AmazonS3Exception: +ls: doesBucketExist on frankfurt-new: S3Exception: Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request; ``` -This happens when trying to work with any S3 service which only supports the -"V4" signing API —but the client is configured to use the default S3 service -endpoint. - The S3A client needs to be given the endpoint to use via the `fs.s3a.endpoint` property. @@ -291,10 +265,10 @@ As an example, the endpoint for S3 Frankfurt is `s3.eu-central-1.amazonaws.com`: When [PrivateLink](https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html) URL is used instead of standard s3a endpoint, it returns "authorization -header is malformed" exception. So, if we set fs.s3a.endpoint=bucket.vpce --.s3.ca-central-1.vpce.amazonaws.com and make s3 calls we get: +header is malformed" exception. So, if we set `fs.s3a.endpoint=bucket.vpce +-.s3.ca-central-1.vpce.amazonaws.com` and make s3 calls we get: ``` -com.amazonaws.services.s3.model.AmazonS3Exception: The authorization header is malformed; the region 'vpce' is wrong; expecting 'ca-central-1' +S3Exception: The authorization header is malformed; the region 'vpce' is wrong; expecting 'ca-central-1' (Service: Amazon S3; Status Code: 400; Error Code: AuthorizationHeaderMalformed; Request ID: req-id; S3 Extended Request ID: req-id-2), S3 Extended Request ID: req-id-2:AuthorizationHeaderMalformed: The authorization header is malformed; the region 'vpce' is wrong; expecting 'ca-central-1' (Service: Amazon S3; Status Code: 400; Error Code: AuthorizationHeaderMalformed; Request ID: req-id; ``` @@ -314,35 +288,27 @@ S3 region as `ca-central-1`. ``` -### `Class does not implement AWSCredentialsProvider` +### `Classdoes not implement software.amazon.awssdk.auth.credentials.AwsCredentialsProvider` A credential provider listed in `fs.s3a.aws.credentials.provider` does not implement -the interface `com.amazonaws.auth.AWSCredentialsProvider`. - -``` - Cause: java.lang.RuntimeException: java.io.IOException: Class class com.amazonaws.auth.EnvironmentVariableCredentialsProvider does not implement AWSCredentialsProvider - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:686) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:621) - at org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:219) - at org.apache.spark.sql.hive.client.HiveClientImpl.(HiveClientImpl.scala:126) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:423) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:306) - at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:433) - ... - Cause: java.io.IOException: Class class com.amazonaws.auth.EnvironmentVariableCredentialsProvider does not implement AWSCredentialsProvider - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:722) - at org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList(S3AUtils.java:687) - at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:620) - at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:673) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:414) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3462) - at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:171) - at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3522) - at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:3496) - at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:591) +the interface `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. + +``` +InstantiationIOException: `s3a://stevel-gcs/': Class org.apache.hadoop.fs.s3a.S3ARetryPolicy does not implement software.amazon.awssdk.auth.credentials.AwsCredentialsProvider (configuration key fs.s3a.aws.credentials.provider) + at org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf(InstantiationIOException.java:128) + at org.apache.hadoop.fs.s3a.S3AUtils.getInstanceFromReflection(S3AUtils.java:604) + at org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSV2CredentialProvider(CredentialProviderListFactory.java:299) + at org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.buildAWSProviderList(CredentialProviderListFactory.java:245) + at org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory.createAWSCredentialProviderList(CredentialProviderListFactory.java:144) + at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:971) + at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:624) + at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3601) + at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:171) + at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3702) + at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3653) + at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:555) + at org.apache.hadoop.fs.Path.getFileSystem(Path.java:366) + ``` There's two main causes @@ -357,12 +323,12 @@ There's two main causes If you see this and you are trying to use the S3A connector with Spark, then the cause can be that the isolated classloader used to load Hive classes is interfering with the S3A -connector's dynamic loading of `com.amazonaws` classes. To fix this, declare that +connector's dynamic loading of `software.amazon.awssdk` classes. To fix this, declare that the classes in the aws SDK are loaded from the same classloader which instantiated the S3A FileSystem instance: ``` -spark.sql.hive.metastore.sharedPrefixes com.amazonaws. +spark.sql.hive.metastore.sharedPrefixes software.amazon.awssdk. ``` ## "The security token included in the request is invalid" @@ -383,59 +349,8 @@ It may be mistyped, or the access key may have been deleted by one of the accoun ``` java.nio.file.AccessDeniedException: bucket: doesBucketExist on bucket: - com.amazonaws.services.s3.model.AmazonS3Exception: + S3Exception: The AWS Access Key Id you provided does not exist in our records. - (Service: Amazon S3; Status Code: 403; Error Code: InvalidAccessKeyId; - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:214) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:260) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:314) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:256) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:231) - at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:366) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:302) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3354) - at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:124) - at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3403) - at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3371) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:477) - at org.apache.hadoop.fs.contract.AbstractBondedFSContract.init(AbstractBondedFSContract.java:72) - at org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:177) - at org.apache.hadoop.fs.s3a.commit.AbstractCommitITest.setup(AbstractCommitITest.java:163) - at org.apache.hadoop.fs.s3a.commit.AbstractITCommitMRJob.setup(AbstractITCommitMRJob.java:129) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:498) - at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) - at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) - at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) - at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) - at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) - at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) - at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) - at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:74) -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: - The AWS Access Key Id you provided does not exist in our records. - (Service: Amazon S3; Status Code: 403; Error Code: InvalidAccessKeyId; - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4229) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4176) - at com.amazonaws.services.s3.AmazonS3Client.getAcl(AmazonS3Client.java:3381) - at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1160) - at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1150) - at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1266) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$verifyBucketExists$1(S3AFileSystem.java:367) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109) - ... 27 more ``` @@ -445,47 +360,12 @@ Caller has no permission to access the bucket at all. ``` doesBucketExist on fdsd: java.nio.file.AccessDeniedException: fdsd: doesBucketExist on fdsd: - com.amazonaws.services.s3.model.AmazonS3Exception: All access to this object has been disabled + S3Exception: All access to this object has been disabled (Service: Amazon S3; Status Code: 403; Error Code: AllAccessDisabled; Request ID: E6229D7F8134E64F; S3 Extended Request ID: 6SzVz2t4qa8J2Wxo/oc8yBuB13Mgrn9uMKnxVY0hsBd2kU/YdHzW1IaujpJdDXRDCQRX3f1RYn0=), S3 Extended Request ID: 6SzVz2t4qa8J2Wxo/oc8yBuB13Mgrn9uMKnxVY0hsBd2kU/YdHzW1IaujpJdDXRDCQRX3f1RYn0=:AllAccessDisabled All access to this object has been disabled (Service: Amazon S3; Status Code: 403; - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:205) - at org.apache.hadoop.fs.s3a.S3ALambda.once(S3ALambda.java:122) - at org.apache.hadoop.fs.s3a.S3ALambda.lambda$retry$2(S3ALambda.java:233) - at org.apache.hadoop.fs.s3a.S3ALambda.retryUntranslated(S3ALambda.java:288) - at org.apache.hadoop.fs.s3a.S3ALambda.retry(S3ALambda.java:228) - at org.apache.hadoop.fs.s3a.S3ALambda.retry(S3ALambda.java:203) - at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:357) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:293) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3288) - at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:123) - at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3337) - at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:3311) - at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:529) - at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo.run(S3GuardTool.java:997) - at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.run(S3GuardTool.java:309) - at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) - at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.run(S3GuardTool.java:1218) - at org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.main(S3GuardTool.java:1227) -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: All access to this object has been disabled - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4229) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4176) - at com.amazonaws.services.s3.AmazonS3Client.getAcl(AmazonS3Client.java:3381) - at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1160) - at com.amazonaws.services.s3.AmazonS3Client.getBucketAcl(AmazonS3Client.java:1150) - at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1266) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$verifyBucketExists$1(S3AFileSystem.java:360) - at org.apache.hadoop.fs.s3a.S3ALambda.once(S3ALambda.java:120) + ``` Check the name of the bucket is correct, and validate permissions for the active user/role. @@ -500,19 +380,9 @@ or the caller does not have the right to access the data. ``` java.nio.file.AccessDeniedException: test/: PUT 0-byte object on test/: - com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403; + S3Exception: Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: EDC662AD2EEEA33C; - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:210) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:110) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:259) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:313) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:255) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:230) - at org.apache.hadoop.fs.s3a.S3AFileSystem.createEmptyObject(S3AFileSystem.java:2691) - at org.apache.hadoop.fs.s3a.S3AFileSystem.createFakeDirectory(S3AFileSystem.java:2666) - at org.apache.hadoop.fs.s3a.S3AFileSystem.innerMkdirs(S3AFileSystem.java:2030) - at org.apache.hadoop.fs.s3a.S3AFileSystem.mkdirs(S3AFileSystem.java:1965) - at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:2305) + ``` In the AWS S3 management console, select the "permissions" tab for the bucket, then "bucket policy". @@ -539,7 +409,7 @@ _all_ the rights which the caller has. ``` mv: rename s3a://london/dest to s3a://london/src on s3a://london/dest: - com.amazonaws.services.s3.model.MultiObjectDeleteException: One or more objects + MultiObjectDeleteException: One or more objects could not be deleted (Service: null; Status Code: 200; Error Code: null; Request ID: 5C9018EF245F02C5; S3 Extended Request ID: 5fQ2RVCPF0rdvADRv2XY3U4yb2J0gHRID/4jm1eqCXp7RxpU0dH9DliChYsCUD1aVCFtbwfWJWY=), @@ -561,7 +431,7 @@ directory will _only_ be in the destination. And files for which the rename oper had yet to commence -they will only be in the source tree. The user has to recover from this themselves. Be assured: no data will have been deleted, it -is just that the data may now be scattered across two directories. +is just that the data may now be scattered across two directories. Note: this is one reason why any application which tries to atomically commit work via rename (classic Hadoop output committers, distcp with the `-atomic` option) are not safe to use with S3. It is not a file system. @@ -589,7 +459,7 @@ If you don't enable this acknowledgement within S3A, then you will see a message ``` java.nio.file.AccessDeniedException: s3a://my-bucket/my-object: getFileStatus on s3a://my-bucket/my-object: -com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; +S3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: myshortreqid; S3 Extended Request ID: mylongreqid):403 Forbidden ``` @@ -598,9 +468,9 @@ To enable requester pays, set `fs.s3a.requester.pays.enabled` property to `true` ### `AccessDeniedException` "InvalidObjectState" when trying to read files ``` -java.nio.file.AccessDeniedException: file1: copyFile(file1, file2) on file1: com.amazonaws.services.s3.model.AmazonS3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=:InvalidObjectState +java.nio.file.AccessDeniedException: file1: copyFile(file1, file2) on file1: S3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=:InvalidObjectState -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw= +Caused by: S3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw= ``` This happens when you're trying to read or copy files that have archive storage class such as @@ -614,7 +484,7 @@ Region must be provided when requesting session credentials, or an exception wil message: ``` -com.amazonaws.SdkClientException: Unable to find a region via the region provider + Unable to find a region via the region provider chain. Must provide an explicit region in the builder or setup environment to supply a region. ``` @@ -624,12 +494,13 @@ endpoint and region like the following: ```xml - fs.s3a.assumed.role.sts.endpoint - ${sts.endpoint} + fs.s3a.assumed.role.sts.endpoint + ${sts.endpoint} + -fs.s3a.assumed.role.sts.endpoint.region -${sts.region} + fs.s3a.assumed.role.sts.endpoint.region + ${sts.region} ``` @@ -672,8 +543,7 @@ can be used: ``` -Using the explicit endpoint for the region is recommended for speed and -to use the V4 signing API. +Using the explicit endpoint for the region is recommended for speed. ### `Unable to find a region via the region provider chain` @@ -690,15 +560,9 @@ This failure surfaces when _all_ the following conditions are met: Stack trace (Hadoop 3.3.1): ``` -Caused by: com.amazonaws.SdkClientException: Unable to find a region via the region provider chain. +Unable to find a region via the region provider chain. Must provide an explicit region in the builder or setup environment to supply a region. - at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:462) - at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424) - at com.amazonaws.client.builder.AwsSyncClientBuilder.build(AwsSyncClientBuilder.java:46) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.buildAmazonS3Client(DefaultS3ClientFactory.java:145) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:97) - at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:788) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:478) + ``` Log and stack trace on later releases, with @@ -725,14 +589,7 @@ warning that the SDK resolution chain is in use: at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3565) at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:3518) at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:592) -Caused by: com.amazonaws.SdkClientException: Unable to find a region via the region provider chain. - Must provide an explicit region in the builder or setup environment to supply a region. - at com.amazonaws.client.builder.AwsClientBuilder.setRegion(AwsClientBuilder.java:462) - at com.amazonaws.client.builder.AwsClientBuilder.configureMutableProperties(AwsClientBuilder.java:424) - at com.amazonaws.client.builder.AwsSyncClientBuilder.build(AwsSyncClientBuilder.java:46) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.buildAmazonS3Client(DefaultS3ClientFactory.java:185) - at org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.createS3Client(DefaultS3ClientFactory.java:117) - ... 21 more + ``` Due to changes in S3 client construction in Hadoop 3.3.1 this option surfaces in @@ -778,33 +635,9 @@ This happens when using the output stream thread pool runs out of capacity. ``` [s3a-transfer-shared-pool1-t20] INFO http.AmazonHttpClient (AmazonHttpClient.java:executeHelper(496)) - Unable to execute HTTP request: - Timeout waiting for connection from poolorg.apache.http.conn.ConnectionPoolTimeoutException: + org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection from pool - at org.apache.http.impl.conn.PoolingClientConnectionManager.leaseConnection(PoolingClientConnectionManager.java:230) - at org.apache.http.impl.conn.PoolingClientConnectionManager$1.getConnection(PoolingClientConnectionManager.java:199) - at sun.reflect.GeneratedMethodAccessor13.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:498) - at com.amazonaws.http.conn.ClientConnectionRequestFactory$Handler.invoke(ClientConnectionRequestFactory.java:70) - at com.amazonaws.http.conn.$Proxy10.getConnection(Unknown Source) - at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:424) - at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:884) - at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) - at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) - at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:728) - at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785) - at com.amazonaws.services.s3.AmazonS3Client.doUploadPart(AmazonS3Client.java:2921) - at com.amazonaws.services.s3.AmazonS3Client.uploadPart(AmazonS3Client.java:2906) - at org.apache.hadoop.fs.s3a.S3AFileSystem.uploadPart(S3AFileSystem.java:1025) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload$1.call(S3ABlockOutputStream.java:360) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload$1.call(S3ABlockOutputStream.java:355) - at org.apache.hadoop.fs.s3a.BlockingThreadPoolExecutorService$CallableWithPermitRelease.call(BlockingThreadPoolExecutorService.java:239) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) + ``` Make sure that `fs.s3a.connection.maximum` is at least larger @@ -813,12 +646,12 @@ than `fs.s3a.threads.max`. ```xml fs.s3a.threads.max - 20 + 64 fs.s3a.connection.maximum - 30 + 64 ``` @@ -836,39 +669,15 @@ Set `fs.s3a.connection.maximum` to a larger value (and at least as large as The HTTP Server did not respond. ``` -2017-02-07 10:01:07,950 INFO [s3a-transfer-shared-pool1-t7] com.amazonaws.http.AmazonHttpClient: - Unable to execute HTTP request: bucket.s3.amazonaws.com:443 failed to respond +2017-02-07 10:01:07,950 INFO [s3a-transfer-shared-pool1-t7] Unable to execute HTTP request: bucket.s3.amazonaws.com:443 failed to respond org.apache.http.NoHttpResponseException: bucket.s3.amazonaws.com:443 failed to respond - at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143) - at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) - at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261) - at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283) - at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:259) - at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:209) - at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272) - at com.amazonaws.http.protocol.SdkHttpRequestExecutor.doReceiveResponse(SdkHttpRequestExecutor.java:66) - at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124) - at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:686) - at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:488) - at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:884) - at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) - at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55) - at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:728) - at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785) - at com.amazonaws.services.s3.AmazonS3Client.copyPart(AmazonS3Client.java:1731) - at com.amazonaws.services.s3.transfer.internal.CopyPartCallable.call(CopyPartCallable.java:41) - at com.amazonaws.services.s3.transfer.internal.CopyPartCallable.call(CopyPartCallable.java:28) - at org.apache.hadoop.fs.s3a.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:222) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) + ``` Probably network problems, unless it really is an outage of S3. +If you are working with a third party store, check its network configuration. + ### Out of heap memory when writing with via Fast Upload @@ -922,19 +731,8 @@ for up to date advice. ``` org.apache.hadoop.fs.s3a.AWSClientIOException: getFileStatus on test/testname/streaming/: - com.amazonaws.AmazonClientException: Failed to sanitize XML document - destined for handler class com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler: - Failed to sanitize XML document destined for handler class - com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:105) - at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:1462) - at org.apache.hadoop.fs.s3a.S3AFileSystem.innerListStatus(S3AFileSystem.java:1227) - at org.apache.hadoop.fs.s3a.S3AFileSystem.listStatus(S3AFileSystem.java:1203) - at org.apache.hadoop.fs.s3a.S3AGlobber.listStatus(S3AGlobber.java:69) - at org.apache.hadoop.fs.s3a.S3AGlobber.doGlob(S3AGlobber.java:210) - at org.apache.hadoop.fs.s3a.S3AGlobber.glob(S3AGlobber.java:125) - at org.apache.hadoop.fs.s3a.S3AFileSystem.globStatus(S3AFileSystem.java:1853) - at org.apache.hadoop.fs.s3a.S3AFileSystem.globStatus(S3AFileSystem.java:1841) + Failed to sanitize XML document + ``` We believe this is caused by the connection to S3 being broken. @@ -944,7 +742,7 @@ It may go away if the operation is retried. ### JSON Parse Error from AWS SDK -Sometimes a JSON Parse error is reported with the stack trace in the `com.amazonaws`, +Sometimes a JSON Parse error is reported with the stack trace from `software.amazon.awssdk`, Again, we believe this is caused by the connection to S3 being broken. @@ -1031,18 +829,7 @@ from breaking. org.apache.hadoop.fs.s3a.RemoteFileChangedException: re-open `s3a://my-bucket/test/file.txt': Change reported by S3 while reading at position 1949. ETag f9c186d787d4de9657e99f280ba26555 was unavailable - at org.apache.hadoop.fs.s3a.impl.ChangeTracker.processResponse(ChangeTracker.java:137) - at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:200) - at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:346) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$2(Invoker.java:195) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:265) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:261) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:193) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:215) - at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:339) - at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:372) + ``` If an S3 object is updated while an S3A filesystem reader has an open @@ -1065,17 +852,7 @@ the following error. org.apache.hadoop.fs.s3a.NoVersionAttributeException: `s3a://my-bucket/test/file.txt': Change detection policy requires ETag at org.apache.hadoop.fs.s3a.impl.ChangeTracker.processResponse(ChangeTracker.java:153) - at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:200) - at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:346) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$2(Invoker.java:195) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109) - at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:265) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:261) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:193) - at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:215) - at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:339) - at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:372) + ``` If the change policy is `versionid` there are a number of possible causes @@ -1133,51 +910,11 @@ key arn is invalid. ``` org.apache.hadoop.fs.s3a.AWSS3IOException: innerMkdirs on /test: - com.amazonaws.services.s3.model.AmazonS3Exception: + S3Exception: Invalid arn (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: CA89F276B3394565), S3 Extended Request ID: ncz0LWn8zor1cUO2fQ7gc5eyqOk3YfyQLDn2OQNoe5Zj/GqDLggUYz9QY7JhdZHdBaDTh+TL5ZQ=: Invalid arn (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: CA89F276B3394565) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:194) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:117) - at org.apache.hadoop.fs.s3a.S3AFileSystem.mkdirs(S3AFileSystem.java:1541) - at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:2230) - at org.apache.hadoop.fs.contract.AbstractFSContractTestBase.mkdirs(AbstractFSContractTestBase.java:338) - at org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:193) - at org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase.setup(S3AScaleTestBase.java:90) - at org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.setup(AbstractSTestS3AHugeFiles.java:77) - at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:498) - at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) - at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) - at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) - at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) - at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) - at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) - at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:74) -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: - Invalid arn (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: CA89F276B3394565) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1588) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1258) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1030) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:742) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:716) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4221) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4168) - at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1718) - at com.amazonaws.services.s3.transfer.internal.UploadCallable.uploadInOneChunk(UploadCallable.java:133) - at com.amazonaws.services.s3.transfer.internal.UploadCallable.call(UploadCallable.java:125) - at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:143) - at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:48) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) ``` Possible causes: @@ -1186,14 +923,13 @@ Possible causes: * the KMS key referenced by the ARN is in a different region than the S3 bucket being used. - ### Using SSE-C "Bad Request" When performing file operations the user may run into an unexpected 400/403 error such as ``` org.apache.hadoop.fs.s3a.AWSS3IOException: getFileStatus on fork-4/: - com.amazonaws.services.s3.model.AmazonS3Exception: + S3Exception: Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request; Request ID: 42F9A1987CB49A99), S3 Extended Request ID: jU2kcwaXnWj5APB14Cgb1IKkc449gu2+dhIsW/+7x9J4D+VUkKvu78mBo03oh9jnOT2eoTLdECU=: @@ -1214,40 +950,7 @@ file using configured SSE-C keyB into that structure. Reading an unencrypted file would fail when read through CSE enabled client. ``` java.lang.SecurityException: Instruction file not found for S3 object with bucket name: ap-south-cse, key: unencryptedData.txt - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.decipher(S3CryptoModuleAE.java:190) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.getObjectSecurely(S3CryptoModuleAE.java:136) - at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.getObject(AmazonS3EncryptionClientV2.java:241) - at org.apache.hadoop.fs.s3a.S3AFileSystem$InputStreamCallbacksImpl.getObject(S3AFileSystem.java:1462) - at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$reopen$0(S3AInputStream.java:217) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:216) - at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:382) - at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:230) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$5(Invoker.java:354) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414) - at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:350) - at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:228) - at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:272) - at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:374) - at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:493) - at java.io.DataInputStream.read(DataInputStream.java:100) - at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:94) - at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:68) - at org.apache.hadoop.io.IOUtils.copyBytes(IOUtils.java:129) - at org.apache.hadoop.fs.shell.Display$Cat.printToStdout(Display.java:101) - at org.apache.hadoop.fs.shell.Display$Cat.processPath(Display.java:96) - at org.apache.hadoop.fs.shell.Command.processPathInternal(Command.java:370) - at org.apache.hadoop.fs.shell.Command.processPaths(Command.java:333) - at org.apache.hadoop.fs.shell.Command.processPathArgument(Command.java:306) - at org.apache.hadoop.fs.shell.Command.processArgument(Command.java:288) - at org.apache.hadoop.fs.shell.Command.processArguments(Command.java:272) - at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:121) - at org.apache.hadoop.fs.shell.Command.run(Command.java:179) - at org.apache.hadoop.fs.FsShell.run(FsShell.java:327) - at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:81) - at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:95) - at org.apache.hadoop.fs.FsShell.main(FsShell.java:390) + ``` CSE enabled client should read encrypted data only. @@ -1266,14 +969,14 @@ method requires KMS key ID. Use fs.s3a.encryption.key property to set it. set `fs.s3a.encryption.key=` generated through AWS console. -### `com.amazonaws.services.kms.model.IncorrectKeyException` The key ID in the request does not identify a CMK that can perform this operation. +### `software.amazon.awssdk.services.kms.model.IncorrectKeyException` The key ID in the request does not identify a CMK that can perform this operation. KMS key ID used to PUT(encrypt) the data, must be the one used to GET the data. ``` cat: open s3a://ap-south-cse/encryptedData.txt at 0 on s3a://ap-south-cse/encryptedData.txt: -com.amazonaws.services.kms.model.IncorrectKeyException: The key ID in the +software.amazon.awssdk.services.kms.model.IncorrectKeyException: The key ID in the request does not identify a CMK that can perform this operation. (Service: AWSKMS; Status Code: 400; ErrorCode: IncorrectKeyException; Request ID: da21aa8a-f00d-467c-94a0-32b627d32bc0; Proxy: null):IncorrectKeyException: @@ -1283,14 +986,14 @@ Request ID: da21aa8a-f00d-467c-94a0-32b627d32bc0; Proxy: null) ``` Use the same KMS key ID used to upload data to download and read it as well. -### `com.amazonaws.services.kms.model.NotFoundException` key/ does not exist +### `software.amazon.awssdk.services.kms.model.NotFoundException` key/ does not exist Using a KMS key ID from a different region than the bucket used to store data would lead to failure while uploading. ``` mkdir: PUT 0-byte object on testmkdir: -com.amazonaws.services.kms.model.NotFoundException: Key +software.amazon.awssdk.services.kms.model.NotFoundException: Key 'arn:aws:kms:ap-south-1:152813717728:key/' does not exist (Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: 279db85d-864d-4a38-9acd-d892adb504c0; Proxy: null):NotFoundException: @@ -1307,24 +1010,6 @@ If Range get is not supported for a CSE algorithm or is disabled: ``` java.lang.SecurityException: Unable to perform range get request: Range get support has been disabled. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.assertCanGetPartialObject(S3CryptoModuleAE.java:446) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.getObjectSecurely(S3CryptoModuleAE.java:117) - at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.getObject(AmazonS3EncryptionClientV2.java:241) - at org.apache.hadoop.fs.s3a.S3AFileSystem$InputStreamCallbacksImpl.getObject(S3AFileSystem.java:1462) - at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$reopen$0(S3AInputStream.java:217) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:216) - at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:382) - at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:230) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$5(Invoker.java:354) - at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:414) - at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:350) - at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:228) - at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:272) - at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:374) - at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:408) - at java.io.DataInputStream.readByte(DataInputStream.java:265) ``` Range gets must be enabled for CSE to work. @@ -1360,124 +1045,43 @@ enhance security. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryp We can ignore this, since this CryptoMode setting(CryptoMode.AuthenticatedEncryption) is required for range gets to work. -### com.amazonaws.services.kms.model.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK +### `software.amazon.awssdk.services.kms.mode.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK` If you generated an Asymmetric CMK from AWS console then CSE-KMS won't be able to generate unique data key for encryption. ``` -Caused by: com.amazonaws.services.kms.model.InvalidKeyUsageException: +Caused by: software.amazon.awssdk.services.kms.mode.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK (Service: AWSKMS; Status Code: 400; Error Code: InvalidKeyUsageException; Request ID: 93609c15-e490-4035-8390-f4396f0d90bf; Proxy: null) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) - at com.amazonaws.services.kms.AWSKMSClient.doInvoke(AWSKMSClient.java:7223) - at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7190) - at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7179) - at com.amazonaws.services.kms.AWSKMSClient.executeGenerateDataKey(AWSKMSClient.java:3482) - at com.amazonaws.services.kms.AWSKMSClient.generateDataKey(AWSKMSClient.java:3451) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.buildContentCryptoMaterial(S3CryptoModuleBase.java:533) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.newContentCryptoMaterial(S3CryptoModuleBase.java:481) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.createContentCryptoMaterial(S3CryptoModuleBase.java:447) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectUsingMetadata(S3CryptoModuleBase.java:160) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectSecurely(S3CryptoModuleBase.java:156) - at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.putObject(AmazonS3EncryptionClientV2.java:236) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$17(S3AFileSystem.java:2792) - at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:604) - at org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:2789) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$33(S3AFileSystem.java:4440) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - ... 49 more ``` Generate a Symmetric Key in the same region as your S3 storage for CSE-KMS to work. -### com.amazonaws.services.kms.model.NotFoundException: Invalid keyId +### software.amazon.awssdk.services.kms.mode.NotFoundException: Invalid keyId If the value in `fs.s3a.encryption.key` property, does not exist /valid in AWS KMS CMK(Customer managed keys), then this error would be seen. ``` -Caused by: com.amazonaws.services.kms.model.NotFoundException: Invalid keyId abc -(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: 9d53552a-3d1b-47c8-984c-9a599d5c2391; Proxy: null) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) - at com.amazonaws.services.kms.AWSKMSClient.doInvoke(AWSKMSClient.java:7223) - at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7190) - at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7179) - at com.amazonaws.services.kms.AWSKMSClient.executeGenerateDataKey(AWSKMSClient.java:3482) - at com.amazonaws.services.kms.AWSKMSClient.generateDataKey(AWSKMSClient.java:3451) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.buildContentCryptoMaterial(S3CryptoModuleBase.java:533) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.newContentCryptoMaterial(S3CryptoModuleBase.java:481) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.createContentCryptoMaterial(S3CryptoModuleBase.java:447) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectUsingMetadata(S3CryptoModuleBase.java:160) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectSecurely(S3CryptoModuleBase.java:156) - at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.putObject(AmazonS3EncryptionClientV2.java:236) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$17(S3AFileSystem.java:2792) - at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:604) - at org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:2789) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$33(S3AFileSystem.java:4440) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - ... 49 more +Caused by: software.amazon.awssdk.services.kms.model.NotFoundException: Invalid keyId abc +(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: + 9d53552a-3d1b-47c8-984c-9a599d5c2391; Proxy: null) ``` Check if `fs.s3a.encryption.key` is set correctly and matches the same on AWS console. -### com.amazonaws.services.kms.model.AWSKMSException: User: is not authorized to perform : kms :GenerateDataKey on resource: +### software.amazon.awssdk.services.kms.model.KmsException: User: is not authorized to perform : kms :GenerateDataKey on resource: User doesn't have authorization to the specific AWS KMS Key ID. ``` -Caused by: com.amazonaws.services.kms.model.AWSKMSException: -User: arn:aws:iam::152813717728:user/ is not authorized to perform: kms:GenerateDataKey on resource: -(Service: AWSKMS; Status Code: 400; Error Code: AccessDeniedException; Request ID: 4ded9f1f-b245-4213-87fc-16cba7a1c4b9; Proxy: null) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) - at com.amazonaws.services.kms.AWSKMSClient.doInvoke(AWSKMSClient.java:7223) - at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7190) - at com.amazonaws.services.kms.AWSKMSClient.invoke(AWSKMSClient.java:7179) - at com.amazonaws.services.kms.AWSKMSClient.executeGenerateDataKey(AWSKMSClient.java:3482) - at com.amazonaws.services.kms.AWSKMSClient.generateDataKey(AWSKMSClient.java:3451) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.buildContentCryptoMaterial(S3CryptoModuleBase.java:533) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.newContentCryptoMaterial(S3CryptoModuleBase.java:481) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.createContentCryptoMaterial(S3CryptoModuleBase.java:447) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectUsingMetadata(S3CryptoModuleBase.java:160) - at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleBase.putObjectSecurely(S3CryptoModuleBase.java:156) - at com.amazonaws.services.s3.AmazonS3EncryptionClientV2.putObject(AmazonS3EncryptionClientV2.java:236) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$17(S3AFileSystem.java:2792) - at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:604) - at org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:2789) - at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$33(S3AFileSystem.java:4440) - at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) - ... 49 more +Caused by: software.amazon.awssdk.services.kms.model.KmsException: +User: arn:aws:iam::152813717728:user/ is not authorized to perform: + kms:GenerateDataKey on resource: +(Service: AWSKMS; Status Code: 400; Error Code: AccessDeniedException; + Request ID: 4ded9f1f-b245-4213-87fc-16cba7a1c4b9; Proxy: null) ``` The user trying to use the KMS Key ID should have the right permissions to access @@ -1528,13 +1132,13 @@ is more than 10000 (specified by aws SDK). You can configure The bucket does not exist. ``` -org.apache.hadoop.fs.s3a.UnknownStoreException: - Bucket random-bucket-33013fb8-f7f7-4edb-9c26-16a6ed019184 does not exist - at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:537) - at org.apache.hadoop.fs.s3a.S3AFileSystem.doBucketProbing(S3AFileSystem.java:471) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:387) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3422) - at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:502) +org.apache.hadoop.fs.s3a.UnknownStoreException: `s3a://random-bucket-7d9217b0-b426-4344-82ea-25d6cbb316f1/': + Bucket does not exist: software.amazon.awssdk.services.s3.model.NoSuchBucketException: null + (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK, + Extended Request ID: 49F5CO1IKavFsz+VBecf2uwZeNVar3InHkdIrONvAK5yQ73gqZ1hFoAEMo8/x5wRNe3OXO3aebvZkev2bS81kw==) + (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK): null + (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK, Extended Request ID: 49F5CO1IKavFsz+VBecf2uwZeNVar3InHkdIrONvAK5yQ73gqZ1hFoAEMo8/x5wRNe3OXO3aebvZkev2bS81kw==) + (Service: S3, Status Code: 404, Request ID: RD254TC8EVDV98AK) ``` Check the URI is correct, and that the bucket actually exists. @@ -1547,20 +1151,6 @@ for a bucket is not an unusual occurrence. This can surface during filesystem API calls if the bucket is deleted while you are using it, -or the startup check for bucket existence has been disabled by setting `fs.s3a.bucket.probe` to 0. -``` -org.apache.hadoop.fs.s3a.UnknownStoreException: s3a://random-bucket-7d9217b0-b426-4344-82ea-25d6cbb316f1/ - - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:254) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:167) - at org.apache.hadoop.fs.s3a.S3AFileSystem.innerListFiles(S3AFileSystem.java:4149) - at org.apache.hadoop.fs.s3a.S3AFileSystem.listFiles(S3AFileSystem.java:3983) -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: -The specified bucket does not exist - (Service: Amazon S3; Status Code: 404; Error Code: NoSuchBucket - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367) -``` - ## S3Guard Errors @@ -1663,73 +1253,16 @@ Possible causes for this This is a very, very rare occurrence. -If the problem is a signing one, try changing the signature algorithm. - -```xml - - fs.s3a.signing-algorithm - S3SignerType - -``` - -We cannot make any promises that it will work, only that it has been known to -make the problem go away "once" ### `AWSS3IOException` The Content-MD5 you specified did not match what we received Reads work, but writes, even `mkdir`, fail: -``` -org.apache.hadoop.fs.s3a.AWSS3IOException: copyFromLocalFile(file:/tmp/hello.txt, s3a://bucket/hello.txt) - on file:/tmp/hello.txt: - The Content-MD5 you specified did not match what we received. - (Service: Amazon S3; Status Code: 400; Error Code: BadDigest; Request ID: 4018131225), - S3 Extended Request ID: null - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:127) - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:69) - at org.apache.hadoop.fs.s3a.S3AFileSystem.copyFromLocalFile(S3AFileSystem.java:1494) - at org.apache.hadoop.tools.cloudup.Cloudup.uploadOneFile(Cloudup.java:466) - at org.apache.hadoop.tools.cloudup.Cloudup.access$000(Cloudup.java:63) - at org.apache.hadoop.tools.cloudup.Cloudup$1.call(Cloudup.java:353) - at org.apache.hadoop.tools.cloudup.Cloudup$1.call(Cloudup.java:350) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:748) -Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: - The Content-MD5 you specified did not match what we received. - (Service: Amazon S3; Status Code: 400; Error Code: BadDigest; Request ID: 4018131225), - S3 Extended Request ID: null - at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1307) - at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:894) - at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:597) - at com.amazonaws.http.AmazonHttpClient.doExecute(AmazonHttpClient.java:363) - at com.amazonaws.http.AmazonHttpClient.executeWithTimer(AmazonHttpClient.java:329) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:308) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3659) - at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1422) - at com.amazonaws.services.s3.transfer.internal.UploadCallable.uploadInOneChunk(UploadCallable.java:131) - at com.amazonaws.services.s3.transfer.internal.UploadCallable.call(UploadCallable.java:123) - at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:139) - at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:47) - at org.apache.hadoop.fs.s3a.BlockingThreadPoolExecutorService$CallableWithPermitRelease.call(BlockingThreadPoolExecutorService.java:239) - ... 4 more -``` - -This stack trace was seen when interacting with a third-party S3 store whose -expectations of headers related to the AWS V4 signing mechanism was not -compatible with that of the specific AWS SDK Hadoop was using. +This has been seen with third party stores. -Workaround: revert to V2 signing. +If the store is configured to require content-MD5 headers with data uploaded: disable it. -```xml - - fs.s3a.signing-algorithm - S3SignerType - -``` +If the store requires the use of the v2 signing algorithm, know that it is unsupported on this release. ### When writing data: "java.io.FileNotFoundException: Completing multi-part upload" @@ -1738,20 +1271,9 @@ with that ID. ``` java.io.FileNotFoundException: Completing multi-part upload on fork-5/test/multipart/1c397ca6-9dfb-4ac1-9cf7-db666673246b: - com.amazonaws.services.s3.model.AmazonS3Exception: The specified upload does not exist. + S3Exception: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. (Service: Amazon S3; Status Code: 404; Error Code: NoSuchUpload; - at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182) - at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770) - at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785) - at com.amazonaws.services.s3.AmazonS3Client.completeMultipartUpload(AmazonS3Client.java:2705) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload.complete(S3ABlockOutputStream.java:473) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream$MultiPartUpload.access$200(S3ABlockOutputStream.java:382) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.close(S3ABlockOutputStream.java:272) - at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) - at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ``` This can happen when all outstanding uploads have been aborted, including the @@ -1839,37 +1361,16 @@ connections more frequently. ``` Cause: org.apache.hadoop.fs.s3a.AWSBadRequestException: put on : - com.amazonaws.services.s3.model.AmazonS3Exception: + S3Exception: The unspecified location constraint is incompatible for the region specific endpoint this request was sent to. (Service: Amazon S3; Status Code: 400; Error Code: IllegalLocationConstraintException; - - at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:178) - at org.apache.hadoop.fs.s3a.S3ALambda.execute(S3ALambda.java:64) - at org.apache.hadoop.fs.s3a.WriteOperationHelper.uploadObject(WriteOperationHelper.java:451) - at org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker.aboutToComplete(MagicCommitTracker.java:128) - at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.close(S3ABlockOutputStream.java:373) - at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) - at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:101) - at org.apache.hadoop.hive.ql.io.orc.WriterImpl.close(WriterImpl.java:2429) - at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat$OrcRecordWriter.close(OrcOutputFormat.java:106) - at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat$OrcRecordWriter.close(OrcOutputFormat.java:91) ... - Cause: com.amazonaws.services.s3.model.AmazonS3Exception: + Cause: S3Exception: The unspecified location constraint is incompatible for the region specific endpoint this request was sent to. (Service: Amazon S3; Status Code: 400; Error Code: IllegalLocationConstraintException; Request ID: EEBC5A08BCB3A645) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1588) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1258) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1030) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:742) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:716) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699) - at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667) - at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4221) - ... + ``` Something has been trying to write data to "/". @@ -1883,8 +1384,7 @@ more detail, as can S3A itself. ```properties log4j.logger.org.apache.hadoop.fs.s3a=DEBUG -log4j.logger.com.amazonaws.request=DEBUG -log4j.logger.com.amazonaws.thirdparty.apache.http=DEBUG +log4j.logger.software.amazon.awssdk.request=DEBUG ``` If using the "unshaded" JAR, then the Apache HttpClient can be directly configured: @@ -1980,8 +1480,9 @@ The number of retries and interval between each retry can be configured: ``` -Not all failures are retried. Specifically excluded are those considered -unrecoverable: +Not all failures are retried *in the S3A Code* -though some may be retried within the +AWS SDK. +Specifically excluded are those considered unrecoverable: * Low-level networking: `UnknownHostException`, `NoRouteToHostException`. * 302 redirects. @@ -2031,7 +1532,7 @@ If this value is configured too low, user may encounter `SdkClientException`s du timing-out. ``` -com.amazonaws.SdkClientException: Unable to execute HTTP request: +software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Request did not complete before the request timeout configuration.: Unable to execute HTTP request: Request did not complete before the request timeout configuration. at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:205) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java index 38c4685eb137e..ded2f0b885079 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a; import java.net.URI; +import java.nio.file.AccessDeniedException; import java.util.UUID; import java.util.concurrent.Callable; @@ -33,11 +34,13 @@ import org.apache.hadoop.test.LambdaTestUtils; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED; import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A; +import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS; import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -69,8 +72,14 @@ public void testNoBucketProbing() throws Exception { assertTrue("getFileStatus on root should always return a directory", fs.getFileStatus(root).isDirectory()); - expectUnknownStore( - () -> fs.listStatus(root)); + try { + expectUnknownStore( + () -> fs.listStatus(root)); + } catch (AccessDeniedException e) { + // this is a sign that there's tests with a third-party bucket and + // interacting with aws is not going to authenticate + skip("no aws credentials"); + } Path src = new Path(root, "testfile"); Path dest = new Path(root, "dst"); @@ -129,7 +138,8 @@ private Configuration createConfigurationWithProbe(final int probe) { removeBaseAndBucketOverrides(conf, S3A_BUCKET_PROBE, ENDPOINT, - AWS_REGION); + AWS_REGION, + PATH_STYLE_ACCESS); conf.setInt(S3A_BUCKET_PROBE, probe); conf.set(AWS_REGION, EU_WEST_1); return conf; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java index 1c6fb9c3ab6a3..e8dcca6df4baa 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java @@ -40,7 +40,9 @@ import org.apache.hadoop.fs.store.audit.AuditSpan; import static org.apache.hadoop.fs.s3a.Constants.CANNED_ACL; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfACLTestsDisabled; /** * Tests of ACL handling in the FS. @@ -55,6 +57,8 @@ public class ITestS3ACannedACLs extends AbstractS3ATestBase { @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); + skipIfACLTestsDisabled(conf); + disableFilesystemCaching(conf); removeBaseAndBucketOverrides(conf, CANNED_ACL); conf.set(CANNED_ACL, LOG_DELIVERY_WRITE); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 5570efe641004..60037e58cd611 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -19,8 +19,9 @@ package org.apache.hadoop.fs.s3a; import java.io.File; +import java.io.IOException; +import java.net.ConnectException; import java.net.URI; -import java.nio.file.AccessDeniedException; import java.security.PrivilegedExceptionAction; import org.assertj.core.api.Assertions; @@ -157,7 +158,7 @@ public void testProxyConnection() throws Exception { conf.setInt(Constants.PROXY_PORT, 1); String proxy = conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT); - expectFSCreateFailure(AWSClientIOException.class, + expectFSCreateFailure(ConnectException.class, conf, "when using proxy " + proxy); } @@ -211,10 +212,10 @@ public void testAutomaticProxyPortSelection() throws Exception { conf.unset(Constants.PROXY_PORT); conf.set(Constants.PROXY_HOST, "127.0.0.1"); conf.set(Constants.SECURE_CONNECTIONS, "true"); - expectFSCreateFailure(AWSClientIOException.class, + expectFSCreateFailure(ConnectException.class, conf, "Expected a connection error for proxy server"); conf.set(Constants.SECURE_CONNECTIONS, "false"); - expectFSCreateFailure(AWSClientIOException.class, + expectFSCreateFailure(ConnectException.class, conf, "Expected a connection error for proxy server"); } @@ -552,6 +553,7 @@ public void testS3SpecificSignerOverride() throws Exception { config.set(SIGNING_ALGORITHM_STS, "CustomSTSSigner"); config.set(AWS_REGION, EU_WEST_1); + disableFilesystemCaching(config); fs = S3ATestUtils.createTestFileSystem(config); S3Client s3Client = getS3Client("testS3SpecificSignerOverride"); @@ -564,7 +566,7 @@ public void testS3SpecificSignerOverride() throws Exception { intercept(StsException.class, "", () -> stsClient.getSessionToken()); - intercept(AccessDeniedException.class, "", () -> + final IOException ioe = intercept(IOException.class, "", () -> Invoker.once("head", bucket, () -> s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()))); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java index 4a96bf5da91d7..ed86143100ac3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContentEncoding.java @@ -29,12 +29,16 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import static org.apache.hadoop.fs.s3a.Constants.CONTENT_ENCODING; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.raiseAsAssumption; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_ENCODING; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.decodeBytes; /** * Tests of content encoding object meta data. + * Some stores don't support gzip; rejection of the content encoding + * is downgraded to a skipped test. */ public class ITestS3AContentEncoding extends AbstractS3ATestBase { @@ -43,6 +47,8 @@ public class ITestS3AContentEncoding extends AbstractS3ATestBase { @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); + skipIfNotEnabled(conf, KEY_CONTENT_ENCODING_ENABLED, + "Skipping storage class ACL tests"); removeBaseAndBucketOverrides(conf, CONTENT_ENCODING); conf.set(CONTENT_ENCODING, GZIP); @@ -51,20 +57,25 @@ protected Configuration createConfiguration() { @Test public void testCreatedObjectsHaveEncoding() throws Throwable { - S3AFileSystem fs = getFileSystem(); - Path dir = methodPath(); - fs.mkdirs(dir); - // even with content encoding enabled, directories do not have - // encoding. - Assertions.assertThat(getEncoding(dir)) - .describedAs("Encoding of object %s", dir) - .isNull(); - Path path = new Path(dir, "1"); - ContractTestUtils.touch(fs, path); - assertObjectHasEncoding(path); - Path path2 = new Path(dir, "2"); - fs.rename(path, path2); - assertObjectHasEncoding(path2); + try { + S3AFileSystem fs = getFileSystem(); + Path dir = methodPath(); + fs.mkdirs(dir); + // even with content encoding enabled, directories do not have + // encoding. + Assertions.assertThat(getEncoding(dir)) + .describedAs("Encoding of object %s", dir) + .isNull(); + Path path = new Path(dir, "1"); + ContractTestUtils.touch(fs, path); + assertObjectHasEncoding(path); + Path path2 = new Path(dir, "2"); + fs.rename(path, path2); + assertObjectHasEncoding(path2); + } catch (AWSUnsupportedFeatureException e) { + LOG.warn("Object store does not support {} content encoding", GZIP, e); + raiseAsAssumption(e); + } } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java index 6d950d9bfaaa4..e07324300197b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java @@ -24,7 +24,9 @@ import org.apache.hadoop.fs.contract.s3a.S3AContract; import static org.apache.hadoop.fs.s3a.Constants.LIST_VERSION; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_LIST_V1_ENABLED; import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * S3A contract tests for getFileStatus, using the v1 List Objects API. @@ -48,6 +50,8 @@ public void teardown() throws Exception { protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); disableFilesystemCaching(conf); + skipIfNotEnabled(conf, KEY_LIST_V1_ENABLED, + "Skipping V1 listing tests"); conf.setInt(Constants.MAX_PAGING_KEYS, 2); // Use v1 List Objects API diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index e5e109ad91b50..eb010674beedf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.nio.file.AccessDeniedException; import java.util.ArrayList; import java.util.List; @@ -38,7 +40,10 @@ import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; +import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.Statistic.STORE_REGION_PROBE; +import static org.apache.hadoop.io.IOUtils.closeStream; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -49,6 +54,26 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase { private static final String AWS_ENDPOINT_TEST = "test-endpoint"; + private static final String USW_2_BUCKET = "landsat-pds"; + + public static final String USW_2_STORE = "s3a://" + USW_2_BUCKET; + + /** + * If anyone were ever to create a bucket with this UUID pair it would break the tests. + */ + public static final String UNKNOWN_BUCKET = "23FA76D4-5F17-48B8-9D7D-9050269D0E40" + + "-8281BAF2-DBCF-47AA-8A27-F2FA3589656A"; + + /** + * New FS instance which will be closed in teardown. + */ + private S3AFileSystem newFS; + + @Override + public void teardown() throws Exception { + closeStream(newFS); + super.teardown(); + } /** * Test to verify that not setting the region config, will lead to the client factory making @@ -57,50 +82,80 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase { */ @Test public void testWithoutRegionConfig() throws IOException { + describe("Verify that region lookup takes place"); + Configuration conf = getConfiguration(); - String bucket = getFileSystem().getBucket(); - conf.unset(String.format("fs.s3a.bucket.%s.endpoint.region", bucket)); - conf.unset(AWS_REGION); + removeBaseAndBucketOverrides(conf, AWS_REGION, PATH_STYLE_ACCESS); + conf.setBoolean(PATH_STYLE_ACCESS, false); + + newFS = new S3AFileSystem(); - S3AFileSystem fs = new S3AFileSystem(); - fs.initialize(getFileSystem().getUri(), conf); + try { + newFS.initialize(getFileSystem().getUri(), conf); + newFS.getS3AInternals().getBucketMetadata(); + } catch (UnknownHostException | UnknownStoreException | AccessDeniedException allowed) { + // these are all valid failure modes from different test environments. + } + assertRegionProbeCount(1); + } + + @Test + public void testUnknownBucket() throws Exception { + describe("Verify unknown bucket probe failure"); + Configuration conf = getConfiguration(); + removeBaseAndBucketOverrides(UNKNOWN_BUCKET, conf, AWS_REGION, PATH_STYLE_ACCESS); - fs.getS3AInternals().getBucketMetadata(); + newFS = new S3AFileSystem(); - Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) - .describedAs("Region is not configured, region probe should have been made").isEqualTo(1); + try { + newFS.initialize(new URI("s3a://" + UNKNOWN_BUCKET), conf); + newFS.getS3AInternals().getBucketMetadata(); + // expect a failure by here + fail("Expected failure, got " + newFS); + } catch (UnknownHostException | UnknownStoreException expected) { + // this is good. + } + assertRegionProbeCount(1); } @Test public void testWithRegionConfig() throws IOException, URISyntaxException { + describe("Verify that region lookup is skipped if the region property is set"); Configuration conf = getConfiguration(); - conf.set(AWS_REGION, "us-east-2"); + removeBaseAndBucketOverrides(conf, AWS_REGION, PATH_STYLE_ACCESS); - S3AFileSystem fs = new S3AFileSystem(); - fs.initialize(new URI("s3a://landsat-pds"), conf); + conf.set(AWS_REGION, "us-east-2"); - Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) - .describedAs("Region is configured, region probe should not have been made").isEqualTo(0); + newFS = new S3AFileSystem(); + newFS.initialize(new URI(USW_2_STORE), conf); + assertRegionProbeCount(0); } @Test public void testRegionCache() throws IOException, URISyntaxException { + describe("Verify that region lookup is cached on the second attempt"); Configuration conf = getConfiguration(); - conf.unset(AWS_REGION); - conf.unset("fs.s3a.bucket.landsat-pds.endpoint.region"); - S3AFileSystem fs = new S3AFileSystem(); + removeBaseAndBucketOverrides(USW_2_BUCKET, conf, AWS_REGION, PATH_STYLE_ACCESS); + + newFS = new S3AFileSystem(); - fs.initialize(new URI("s3a://landsat-pds"), conf); + newFS.initialize(new URI(USW_2_STORE), conf); - Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) - .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(1); + assertRegionProbeCount(1); + closeStream(newFS); - fs.initialize(new URI("s3a://landsat-pds"), conf); + // create a new instance + newFS = new S3AFileSystem(); + newFS.initialize(new URI(USW_2_STORE), conf); // value should already be cached. - Assertions.assertThat(fs.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) - .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(0); + assertRegionProbeCount(0); + } + + private void assertRegionProbeCount(final int expected) { + Assertions.assertThat(newFS.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) + .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(expected); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index a741b11b0ce47..7c53f9c32a66a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -45,6 +45,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled; import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf; import static org.apache.hadoop.test.LambdaTestUtils.*; import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator; @@ -89,7 +90,22 @@ public void testMultiObjectDeleteLargeNumKeys() throws Exception { S3AFileSystem fs = getFileSystem(); Path path = path("largeDir"); mkdirs(path); - createFiles(fs, path, 1, 1005, 0); + final boolean bulkDeleteEnabled = isBulkDeleteEnabled(getFileSystem()); + + // with single object delete, only create a few files for a faster + // test run. + int filesToCreate; + int pages; + if (bulkDeleteEnabled) { + filesToCreate = 1005; + pages = 5; + } else { + filesToCreate = 250; + pages = 0; + } + + + createFiles(fs, path, 1, filesToCreate, 0); RemoteIterator locatedFileStatusRemoteIterator = fs.listFiles(path, false); List keys = toList(mappingRemoteIterator(locatedFileStatusRemoteIterator, @@ -102,9 +118,10 @@ public void testMultiObjectDeleteLargeNumKeys() throws Exception { } Long bulkDeleteReqAfter = getNumberOfBulkDeleteRequestsMadeTillNow(fs); // number of delete requests is 5 as we have default page size of 250. + Assertions.assertThat(bulkDeleteReqAfter - bulkDeleteReqBefore) .describedAs("Number of batched bulk delete requests") - .isEqualTo(5); + .isEqualTo(pages); } private Long getNumberOfBulkDeleteRequestsMadeTillNow(S3AFileSystem fs) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java index 4aa9c6a52eaa2..66a211181bdde 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -182,14 +182,8 @@ public void testEmptyFileChecksums() throws Throwable { * non-null. */ private void assumeNoDefaultEncryption() throws IOException { - try { - skipIfClientSideEncryption(); - Assume.assumeThat(getDefaultEncryption(), nullValue()); - } catch (AccessDeniedException e) { - // if the user can't check the default encryption, assume that it is - // null and keep going - LOG.warn("User does not have permission to call getBucketEncryption()"); - } + skipIfClientSideEncryption(); + Assume.assumeThat(getDefaultEncryption(), nullValue()); } /** @@ -418,7 +412,7 @@ private GetBucketEncryptionResponse getDefaultEncryption() throws IOException { () -> s3.getBucketEncryption(GetBucketEncryptionRequest.builder() .bucket(fs.getBucket()) .build())); - } catch (FileNotFoundException e) { + } catch (FileNotFoundException | AccessDeniedException | AWSBadRequestException e) { return null; } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java index 197811f39fb9b..263a857e03300 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java @@ -18,18 +18,19 @@ package org.apache.hadoop.fs.s3a; +import org.assertj.core.api.Assertions; +import org.junit.Test; import software.amazon.awssdk.services.s3.model.MultipartUpload; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.store.audit.AuditSpan; -import org.junit.Test; - - import java.io.IOException; import java.util.HashSet; import java.util.Set; +import static org.apache.hadoop.util.functional.RemoteIterators.foreach; /** * Tests for {@link MultipartUtils}. @@ -101,17 +102,18 @@ private void assertUploadsPresent(MultipartUtils.UploadIterator list, // Don't modify passed-in set, use copy. Set uploads = new HashSet<>(ourUploads); - while (list.hasNext()) { - MultipartTestUtils.IdKey listing = toIdKey(list.next()); - if (uploads.contains(listing)) { + foreach(list, (upload) -> { + MultipartTestUtils.IdKey listing = toIdKey(upload); + if (uploads.remove(listing)) { LOG.debug("Matched: {},{}", listing.getKey(), listing.getUploadId()); - uploads.remove(listing); } else { LOG.debug("Not our upload {},{}", listing.getKey(), listing.getUploadId()); } - } - assertTrue("Not all our uploads were listed", uploads.isEmpty()); + }); + Assertions.assertThat(uploads) + .describedAs("Uploads which we expected to be listed.") + .isEmpty(); } private MultipartTestUtils.IdKey toIdKey(MultipartUpload mu) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java index 6ccb7ac26040b..56cb541e8233a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java @@ -74,6 +74,7 @@ public ITestS3AStorageClass(String fastUploadBufferType) { @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); + skipIfStorageClassTestsDisabled(conf); disableFilesystemCaching(conf); removeBaseAndBucketOverrides(conf, STORAGE_CLASS, FAST_UPLOAD_BUFFER); conf.set(FAST_UPLOAD_BUFFER, fastUploadBufferType); @@ -81,12 +82,6 @@ protected Configuration createConfiguration() { return conf; } - @Override - public void setup() throws Exception { - super.setup(); - skipIfStorageClassTestsDisabled(getConfiguration()); - } - /* * This test ensures the default storage class configuration (no config or null) * works well with create and copy operations diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java index 2b7620ddbddfc..3e343a9ea85f2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java @@ -199,5 +199,13 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(key, uploadId); } + + @Override + public String toString() { + return "IdKey{" + + "key='" + key + '\'' + + ", uploadId='" + uploadId + '\'' + + '}'; + } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 246d111d14b8d..f2e0223adb1d9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -58,6 +58,21 @@ public interface S3ATestConstants { */ String KEY_STORAGE_CLASS_TESTS_ENABLED = TEST_FS_S3A + "create.storage.class.enabled"; + /** + * A property set to true if ACL tests are enabled: {@value}. + */ + String KEY_ACL_TESTS_ENABLED = TEST_FS_S3A + "create.acl.enabled"; + + /** + * A property set to true if V1 tests are enabled: {@value}. + */ + String KEY_LIST_V1_ENABLED = TEST_FS_S3A + "list.v1.enabled"; + + /** + * A property set to true if content encoding tests are enabled: {@value}. + */ + String KEY_CONTENT_ENCODING_ENABLED = TEST_FS_S3A + "content.encoding.enabled"; + /** * Tell tests that they are being executed in parallel: {@value}. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 239e52a7264a0..3382c300b9315 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -61,12 +61,13 @@ import org.apache.hadoop.util.functional.CallableRaisingIOE; import org.apache.hadoop.util.functional.FutureIO; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Assume; +import org.junit.AssumptionViolatedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import java.io.Closeable; import java.io.File; @@ -466,8 +467,20 @@ public static void disableFilesystemCaching(Configuration conf) { */ public static void skipIfEncryptionTestsDisabled( Configuration configuration) { - if (!configuration.getBoolean(KEY_ENCRYPTION_TESTS, true)) { - skip("Skipping encryption tests"); + skipIfNotEnabled(configuration, KEY_ENCRYPTION_TESTS, "Skipping encryption tests"); + } + + /** + * Skip a test suite/casee if a configuration has been explicitly disabled. + * @param configuration configuration to probe + * @param key key to resolve + * @param message assertion text + */ + public static void skipIfNotEnabled(final Configuration configuration, + final String key, + final String message) { + if (!configuration.getBoolean(key, true)) { + skip(message); } } @@ -477,9 +490,18 @@ public static void skipIfEncryptionTestsDisabled( */ public static void skipIfStorageClassTestsDisabled( Configuration configuration) { - if (!configuration.getBoolean(KEY_STORAGE_CLASS_TESTS_ENABLED, true)) { - skip("Skipping storage class tests"); - } + skipIfNotEnabled(configuration, KEY_STORAGE_CLASS_TESTS_ENABLED, + "Skipping storage class tests"); + } + + /** + * Skip a test if ACL class tests are disabled. + * @param configuration configuration to probe + */ + public static void skipIfACLTestsDisabled( + Configuration configuration) { + skipIfNotEnabled(configuration, KEY_ACL_TESTS_ENABLED, + "Skipping storage class ACL tests"); } /** @@ -635,9 +657,7 @@ public static AwsCredentialsProvider buildAwsCredentialsProvider( * @param conf configuration to examine */ public static void assumeSessionTestsEnabled(final Configuration conf) { - if (!conf.getBoolean(TEST_STS_ENABLED, true)) { - skip("STS functional tests disabled"); - } + skipIfNotEnabled(conf, TEST_STS_ENABLED, "STS functional tests disabled"); } /** @@ -1245,6 +1265,14 @@ public static void assume(String message, boolean condition) { Assume.assumeTrue(message, condition); } + /** + * Convert a throwable to an assumption failure. + * @param t thrown exception. + */ + public static void raiseAsAssumption(Throwable t) { + throw new AssumptionViolatedException(t.toString(), t); + } + /** * Get the statistics from a wrapped block output stream. * @param out output stream @@ -1515,4 +1543,15 @@ public static S3AInputStream getS3AInputStream( public static void disablePrefetching(Configuration conf) { removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY); } + + /** + * Does this FS support multi object delete? + * @param fs filesystem + * @return true if multi-delete is enabled. + */ + + public static boolean isBulkDeleteEnabled(FileSystem fs) { + return fs.getConf().getBoolean(Constants.ENABLE_MULTI_DELETE, + true); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java index 0ac49812e4cb6..f6e8f0c376d9b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java @@ -43,6 +43,7 @@ import static org.apache.hadoop.fs.s3a.Invoker.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.verifyExceptionClass; import static org.apache.hadoop.fs.s3a.S3AUtils.*; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_400_BAD_REQUEST; import static org.apache.hadoop.test.LambdaTestUtils.*; /** @@ -104,7 +105,7 @@ public class TestInvoker extends Assert { .cause(new Local.ConnectTimeoutException("timeout")) .build(); private static final AwsServiceException BAD_REQUEST = serviceException( - AWSBadRequestException.STATUS_CODE, + SC_400_BAD_REQUEST, "bad request"); @Before diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java index ad7d59a7319cf..fe48cd8911578 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java @@ -70,12 +70,15 @@ public class ITestCustomSigner extends AbstractS3ATestBase { public void setup() throws Exception { super.setup(); final S3AFileSystem fs = getFileSystem(); - regionName = determineRegion(fs.getBucket()); + final Configuration conf = fs.getConf(); + endpoint = conf.getTrimmed(Constants.ENDPOINT, Constants.CENTRAL_ENDPOINT); + LOG.info("Test endpoint is {}", endpoint); + regionName = conf.getTrimmed(Constants.AWS_REGION, ""); + if (regionName.isEmpty()) { + regionName = determineRegion(fs.getBucket()); + } LOG.info("Determined region name to be [{}] for bucket [{}]", regionName, fs.getBucket()); - endpoint = fs.getConf() - .get(Constants.ENDPOINT, Constants.CENTRAL_ENDPOINT); - LOG.info("Test endpoint is {}", endpoint); } @Test @@ -118,11 +121,14 @@ private FileSystem runMkDirAndVerify(UserGroupInformation ugi, .isGreaterThan(invocationCount); Assertions.assertThat(CustomSigner.lastStoreValue) - .as("Store value should not be null").isNotNull(); + .as("Store value should not be null in %s", CustomSigner.description()) + .isNotNull(); Assertions.assertThat(CustomSigner.lastStoreValue.conf) - .as("Configuration should not be null").isNotNull(); + .as("Configuration should not be null in %s", CustomSigner.description()) + .isNotNull(); Assertions.assertThat(CustomSigner.lastStoreValue.conf.get(TEST_ID_KEY)) - .as("Configuration TEST_KEY mismatch").isEqualTo(identifier); + .as("Configuration TEST_KEY mismatch in %s", CustomSigner.description()) + .isEqualTo(identifier); return fs; }); @@ -196,8 +202,9 @@ public SdkHttpFullRequest sign(SdkHttpFullRequest request, try { lastStoreValue = CustomSignerInitializer .getStoreValue(bucketName, UserGroupInformation.getCurrentUser()); + LOG.info("Store value for bucket {} is {}", bucketName, lastStoreValue); } catch (IOException e) { - throw new RuntimeException("Failed to get current Ugi", e); + throw new RuntimeException("Failed to get current Ugi " + e, e); } if (bucketName.equals("kms")) { Aws4Signer realKMSSigner = Aws4Signer.create(); @@ -247,6 +254,14 @@ public static int getInstantiationCount() { public static int getInvocationCount() { return INVOCATION_COUNT.get(); } + + public static String description() { + return "CustomSigner{" + + "invocations=" + INVOCATION_COUNT.get() + + ", instantiations=" + INSTANTIATION_COUNT.get() + + ", lastStoreValue=" + lastStoreValue + + "}"; + } } @Private @@ -260,6 +275,7 @@ public void registerStore(String bucketName, Configuration storeConf, DelegationTokenProvider dtProvider, UserGroupInformation storeUgi) { StoreKey storeKey = new StoreKey(bucketName, storeUgi); StoreValue storeValue = new StoreValue(storeConf, dtProvider); + LOG.info("Registering store {} with value {}", storeKey, storeValue); knownStores.put(storeKey, storeValue); } @@ -267,6 +283,7 @@ public void registerStore(String bucketName, Configuration storeConf, public void unregisterStore(String bucketName, Configuration storeConf, DelegationTokenProvider dtProvider, UserGroupInformation storeUgi) { StoreKey storeKey = new StoreKey(bucketName, storeUgi); + LOG.info("Unregistering store {}", storeKey); knownStores.remove(storeKey); } @@ -302,6 +319,14 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(bucketName, ugi); } + + @Override + public String toString() { + return "StoreKey{" + + "bucketName='" + bucketName + '\'' + + ", ugi=" + ugi + + '}'; + } } static class StoreValue { @@ -313,6 +338,14 @@ public StoreValue(Configuration conf, this.conf = conf; this.dtProvider = dtProvider; } + + @Override + public String toString() { + return "StoreValue{" + + "conf=" + conf + + ", dtProvider=" + dtProvider + + '}'; + } } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java index 595e2687276b1..b56b8c20bfe77 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestSignerManager.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.concurrent.TimeUnit; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.signer.Signer; @@ -32,26 +31,27 @@ import software.amazon.awssdk.http.SdkHttpMethod; import org.assertj.core.api.Assertions; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.Timeout; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.auth.TestSignerManager.SignerInitializerForTest.StoreValue; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; +import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.test.AbstractHadoopTestBase; import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; +import static org.apache.hadoop.fs.s3a.auth.SignerFactory.S3_V2_SIGNER; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Tests for the SignerManager. */ -public class TestSignerManager { +public class TestSignerManager extends AbstractHadoopTestBase { private static final Text TEST_TOKEN_KIND = new Text("TestTokenKind"); private static final Text TEST_TOKEN_SERVICE = new Text("TestTokenService"); @@ -61,9 +61,6 @@ public class TestSignerManager { private static final String TESTUSER1 = "testuser1"; private static final String TESTUSER2 = "testuser2"; - @Rule public Timeout testTimeout = new Timeout(10_000L, - TimeUnit.MILLISECONDS); - @Before public void beforeTest() { SignerForTest1.reset(); @@ -95,11 +92,8 @@ public void testCustomSignerFailureIfNotRegistered() throws Exception { // Make sure the config is respected. signerManager.initCustomSigners(); // Simulate a call from the AWS SDK to create the signer. - LambdaTestUtils.intercept(Exception.class, + intercept(InstantiationIOException.class, () -> SignerFactory.createSigner("testsignerUnregistered", null)); - // Expecting generic Exception.class to handle future implementation - // changes. - // For now, this is an NPE } @Test @@ -588,4 +582,17 @@ private SdkHttpFullRequest constructSignableRequest(String bucketName) { return SdkHttpFullRequest.builder().host(host).protocol("https").method(SdkHttpMethod.GET) .build(); } + + @Test + public void testV2SignerRejected() throws Throwable { + intercept(InstantiationIOException.class, "no longer supported", + () -> SignerFactory.createSigner(S3_V2_SIGNER, "key")); + } + + @Test + public void testUnknownSignerRejected() throws Throwable { + intercept(InstantiationIOException.class, "unknownSigner", + () -> SignerFactory.createSigner("unknownSigner", "key")); + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java index 613fab192874a..116d48e9de5fc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java @@ -23,6 +23,7 @@ import java.util.Map; import org.assertj.core.api.Assertions; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,8 +40,10 @@ import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles; +import org.apache.hadoop.fs.store.audit.AuditSpan; import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads; +import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.extractXAttrLongValue; @@ -118,6 +121,19 @@ protected Path getPathOfFileToCreate() { return magicOutputFile; } + @Test + public void test_000_CleanupPendingUploads() throws IOException { + describe("Cleanup any existing pending uploads"); + final S3AFileSystem fs = getFileSystem(); + final String key = fs.pathToKey(finalDirectory); + final AuditSpan span = fs.getAuditSpanSource().createSpan( + MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED.getSymbol(), + key, null); + final int count = fs.createWriteOperationHelper(span) + .abortMultipartUploadsUnderPath(key + "/"); + LOG.info("Aborted {} uploads under {}", count, key); + } + @Override public void test_030_postCreationAssertions() throws Throwable { describe("Committing file"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java index 3a390e34ecad2..f6fe68b8e6d33 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java @@ -37,7 +37,6 @@ import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_MAP; import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_NAMED; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_OCTET_STREAM; -import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_APPLICATION_XML; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_X_DIRECTORY; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_LENGTH; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_CONTENT_TYPE; @@ -74,15 +73,12 @@ public void testXAttrRoot() throws Throwable { fs.listXAttrs(root), with(INVOCATION_OP_XATTR_LIST, GET_METADATA_ON_OBJECT)); - // verify this contains all the standard markers, - // but not the magic marker header + // don't make any assertions on the headers entries + // as different S3 providers may have different headers + // and they may even change over time. Assertions.assertThat(headerList) .describedAs("Headers on root object") - .containsOnly( - XA_CONTENT_LENGTH, - XA_CONTENT_TYPE); - assertHeaderEntry(xAttrs, XA_CONTENT_TYPE) - .isEqualTo(CONTENT_TYPE_APPLICATION_XML); + .hasSize(xAttrs.size()); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java new file mode 100644 index 0000000000000..71536880dd5cb --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.UnknownHostException; + +import org.junit.Test; +import software.amazon.awssdk.core.exception.SdkClientException; + +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Unit tests related to the {@link ErrorTranslation} class. + */ +public class TestErrorTranslation extends AbstractHadoopTestBase { + + /** + * Create an sdk exception with the given cause. + * @param message error message + * @param cause cause + * @return a new exception + */ + private static SdkClientException sdkException( + String message, + Throwable cause) { + return SdkClientException.builder() + .message(message) + .cause(cause) + .build(); + } + + @Test + public void testUnknownHostExceptionExtraction() throws Throwable { + final SdkClientException thrown = sdkException("top", + sdkException("middle", + new UnknownHostException("bottom"))); + final IOException ioe = intercept(UnknownHostException.class, "top", + () -> { + throw maybeExtractNetworkException("", thrown); + }); + + // the wrapped exception is the top level one: no stack traces have + // been lost + if (ioe.getCause() != thrown) { + throw new AssertionError("Cause of " + ioe + " is not " + thrown, thrown); + } + + } + + @Test + public void testNoRouteToHostExceptionExtraction() throws Throwable { + intercept(NoRouteToHostException.class, "top", + () -> { + throw maybeExtractNetworkException("p2", + sdkException("top", + sdkException("middle", + new NoRouteToHostException("bottom")))); + }); + } + + @Test + public void testConnectExceptionExtraction() throws Throwable { + intercept(ConnectException.class, "top", + () -> { + throw maybeExtractNetworkException("p1", + sdkException("top", + sdkException("middle", + new ConnectException("bottom")))); + }); + } + @Test + public void testNoConstructorExtraction() throws Throwable { + intercept(PathIOException.class, NoConstructorIOE.MESSAGE, + () -> { + throw maybeExtractNetworkException("p1", + sdkException("top", + sdkException("middle", + new NoConstructorIOE()))); + }); + } + + + public static final class NoConstructorIOE extends IOException { + + public static final String MESSAGE = "no-arg constructor"; + + public NoConstructorIOE() { + super(MESSAGE); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java index 8bbf52b578e1a..48378ce75dc9c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java @@ -32,7 +32,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; -import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.Statistic; @@ -135,9 +134,7 @@ public void setup() throws Exception { setupCostValidator(); // determine bulk delete settings - final Configuration fsConf = getFileSystem().getConf(); - isBulkDelete = fsConf.getBoolean(Constants.ENABLE_MULTI_DELETE, - true); + isBulkDelete = isBulkDeleteEnabled(getFileSystem()); deleteMarkerStatistic = isBulkDelete() ? OBJECT_BULK_DELETE_REQUEST : OBJECT_DELETE_REQUEST; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java index 48791f557c002..cd4ee44406676 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java @@ -455,6 +455,7 @@ public void testRenameBase() throws Throwable { Path src = basePath; Path dest = new Path(methodPath(), "dest"); + getFileSystem().delete(dest, true); assertRenamed(src, dest); assertPathDoesNotExist("source", src); @@ -610,15 +611,18 @@ private void put(final String key, final String content) throws Exception { RequestBody.fromString(content))); } /** - * Delete an object. + * Delete an object; exceptions are swallowed. * @param key key - * @param content string */ private void deleteObject(final String key) throws Exception { - exec("DELETE " + key, () -> { - s3client.deleteObject(b -> b.bucket(bucket).key(key)); - return "deleted " + key; - }); + try { + exec("DELETE " + key, () -> { + s3client.deleteObject(b -> b.bucket(bucket).key(key)); + return "deleted " + key; + }); + } catch (IOException ignored) { + + } } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index be4de7942f7b9..fae2df973af7f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -99,16 +99,17 @@ public void testDeleteSingleFileInDir() throws Throwable { // still be there Path simpleFile = file(new Path(dir, "simple.txt")); - boolean rawAndKeeping = !isDeleting(); - boolean rawAndDeleting = isDeleting(); + boolean keeping = !isDeleting(); + boolean deleting = isDeleting(); + boolean bulkDelete = isBulkDelete(); verifyMetrics(() -> { fs.delete(simpleFile, false); return "after fs.delete(simpleFile) " + getMetricSummary(); }, - probe(rawAndKeeping, OBJECT_METADATA_REQUESTS, + probe(keeping, OBJECT_METADATA_REQUESTS, FILESTATUS_FILE_PROBE_H), // if deleting markers, look for the parent too - probe(rawAndDeleting, OBJECT_METADATA_REQUESTS, + probe(deleting, OBJECT_METADATA_REQUESTS, FILESTATUS_FILE_PROBE_H + FILESTATUS_DIR_PROBE_H), with(OBJECT_LIST_REQUEST, FILESTATUS_FILE_PROBE_L + FILESTATUS_DIR_PROBE_L), @@ -116,7 +117,9 @@ public void testDeleteSingleFileInDir() throws Throwable { with(FILES_DELETED, 1), // a single DELETE call is made to delete the object - with(OBJECT_DELETE_REQUEST, DELETE_OBJECT_REQUEST), + probe(bulkDelete, OBJECT_DELETE_REQUEST, DELETE_OBJECT_REQUEST), + probe(!bulkDelete, OBJECT_DELETE_REQUEST, + DELETE_OBJECT_REQUEST + DELETE_MARKER_REQUEST), // keeping: create no parent dirs or delete parents withWhenKeeping(DIRECTORIES_CREATED, 0), @@ -127,7 +130,8 @@ public void testDeleteSingleFileInDir() throws Throwable { // a bulk delete for all parents is issued. // the number of objects in it depends on the depth of the tree; // don't worry about that - withWhenDeleting(OBJECT_BULK_DELETE_REQUEST, DELETE_MARKER_REQUEST) + probe(deleting && bulkDelete, OBJECT_BULK_DELETE_REQUEST, + DELETE_MARKER_REQUEST) ); // there is an empty dir for a parent diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java index 3dc64ea3b45fd..6f19ba15c1c9a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java @@ -19,9 +19,9 @@ package org.apache.hadoop.fs.s3a.scale; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSUnsupportedFeatureException; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; -import org.apache.hadoop.fs.s3a.S3ATestUtils; import java.nio.file.AccessDeniedException; @@ -54,8 +54,7 @@ public class ITestS3AHugeFilesSSECDiskBlocks public void setup() throws Exception { try { super.setup(); - skipIfEncryptionTestsDisabled(getConfiguration()); - } catch (AccessDeniedException e) { + } catch (AccessDeniedException | AWSUnsupportedFeatureException e) { skip("Bucket does not allow " + S3AEncryptionMethods.SSE_C + " encryption method"); } } @@ -67,7 +66,7 @@ protected Configuration createScaleConfiguration() { removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_KEY, S3_ENCRYPTION_ALGORITHM, SERVER_SIDE_ENCRYPTION_ALGORITHM, SERVER_SIDE_ENCRYPTION_KEY); - S3ATestUtils.disableFilesystemCaching(conf); + skipIfEncryptionTestsDisabled(conf); conf.set(Constants.S3_ENCRYPTION_ALGORITHM, getSSEAlgorithm().getMethod()); conf.set(Constants.S3_ENCRYPTION_KEY, KEY_1); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java index ccc71c58644f2..58988c9c41bf8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java @@ -46,15 +46,10 @@ public class ITestS3AHugeFilesStorageClass extends AbstractSTestS3AHugeFiles { private static final Logger LOG = LoggerFactory.getLogger(ITestS3AHugeFilesStorageClass.class); - @Override - public void setup() throws Exception { - super.setup(); - skipIfStorageClassTestsDisabled(getConfiguration()); - } - @Override protected Configuration createScaleConfiguration() { Configuration conf = super.createScaleConfiguration(); + skipIfStorageClassTestsDisabled(conf); disableFilesystemCaching(conf); removeBaseAndBucketOverrides(conf, STORAGE_CLASS); diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 1533477229642..12379fe6c8a6b 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -189,6 +189,18 @@ 10s + + fs.s3a.attempts.maximum + 1 + How many times should the SDK retry commands on (probably) transient errors. + + + + fs.s3a.retry.limit + 3 + Fail fairly fast + + com.google.protobuf protobuf-java - compile + ${common.protobuf2.scope} com.google.code.gson @@ -504,11 +505,11 @@ **/ProtobufHelper.java - **/RpcWritable.java **/ProtobufRpcEngineCallback.java **/ProtobufRpcEngine.java **/ProtobufRpcEngine2.java **/ProtobufRpcEngineProtos.java + **/ProtobufWrapperLegacy.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java index 2cbfd0d0ec030..f035c1ccf982f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java @@ -37,14 +37,13 @@ import org.apache.hadoop.ha.proto.HAServiceProtocolProtos.TransitionToActiveRequestProto; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos.TransitionToStandbyRequestProto; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos.TransitionToObserverRequestProto; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolTranslator; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -84,60 +83,39 @@ public HAServiceProtocolClientSideTranslatorPB( @Override public void monitorHealth() throws IOException { - try { - rpcProxy.monitorHealth(NULL_CONTROLLER, MONITOR_HEALTH_REQ); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.monitorHealth(NULL_CONTROLLER, MONITOR_HEALTH_REQ)); } @Override public void transitionToActive(StateChangeRequestInfo reqInfo) throws IOException { - try { - TransitionToActiveRequestProto req = - TransitionToActiveRequestProto.newBuilder() + TransitionToActiveRequestProto req = + TransitionToActiveRequestProto.newBuilder() .setReqInfo(convert(reqInfo)).build(); - - rpcProxy.transitionToActive(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.transitionToActive(NULL_CONTROLLER, req)); } @Override public void transitionToStandby(StateChangeRequestInfo reqInfo) throws IOException { - try { - TransitionToStandbyRequestProto req = + TransitionToStandbyRequestProto req = TransitionToStandbyRequestProto.newBuilder() - .setReqInfo(convert(reqInfo)).build(); - rpcProxy.transitionToStandby(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + .setReqInfo(convert(reqInfo)).build(); + ipc(() -> rpcProxy.transitionToStandby(NULL_CONTROLLER, req)); } @Override public void transitionToObserver(StateChangeRequestInfo reqInfo) throws IOException { - try { - TransitionToObserverRequestProto req = - TransitionToObserverRequestProto.newBuilder() - .setReqInfo(convert(reqInfo)).build(); - rpcProxy.transitionToObserver(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + TransitionToObserverRequestProto req = + TransitionToObserverRequestProto.newBuilder() + .setReqInfo(convert(reqInfo)).build(); + ipc(() -> rpcProxy.transitionToObserver(NULL_CONTROLLER, req)); } @Override public HAServiceStatus getServiceStatus() throws IOException { GetServiceStatusResponseProto status; - try { - status = rpcProxy.getServiceStatus(NULL_CONTROLLER, - GET_SERVICE_STATUS_REQ); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + status = ipc(() -> rpcProxy.getServiceStatus(NULL_CONTROLLER, + GET_SERVICE_STATUS_REQ)); HAServiceStatus ret = new HAServiceStatus( convert(status.getState())); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java index 3777207c7e45c..307629fdfa22b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java @@ -27,15 +27,14 @@ import org.apache.hadoop.ha.ZKFCProtocol; import org.apache.hadoop.ha.proto.ZKFCProtocolProtos.CedeActiveRequestProto; import org.apache.hadoop.ha.proto.ZKFCProtocolProtos.GracefulFailoverRequestProto; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolTranslator; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; public class ZKFCProtocolClientSideTranslatorPB implements @@ -57,24 +56,16 @@ public ZKFCProtocolClientSideTranslatorPB( @Override public void cedeActive(int millisToCede) throws IOException, AccessControlException { - try { - CedeActiveRequestProto req = CedeActiveRequestProto.newBuilder() - .setMillisToCede(millisToCede) - .build(); - rpcProxy.cedeActive(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + CedeActiveRequestProto req = CedeActiveRequestProto.newBuilder() + .setMillisToCede(millisToCede) + .build(); + ipc(() -> rpcProxy.cedeActive(NULL_CONTROLLER, req)); } @Override public void gracefulFailover() throws IOException, AccessControlException { - try { - rpcProxy.gracefulFailover(NULL_CONTROLLER, - GracefulFailoverRequestProto.getDefaultInstance()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.gracefulFailover(NULL_CONTROLLER, + GracefulFailoverRequestProto.getDefaultInstance())); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java index 9ed0640c8dcfa..da4dfe640549b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java @@ -18,10 +18,10 @@ package org.apache.hadoop.ipc; import java.io.IOException; -import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.internal.ShadedProtobufHelper; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -30,31 +30,37 @@ import org.apache.hadoop.thirdparty.protobuf.ServiceException; /** - * Helper methods for protobuf related RPC implementation + * Helper methods for protobuf related RPC implementation. + * This is deprecated because it references protobuf 2.5 classes + * as well as the shaded ones -and so needs an unshaded protobuf-2.5 + * JAR on the classpath during execution. + * It MUST NOT be used internally; it is retained in case existing, + * external applications already use it. + * @deprecated hadoop code MUST use {@link ShadedProtobufHelper}. */ @InterfaceAudience.Private +@Deprecated public class ProtobufHelper { + private ProtobufHelper() { // Hidden constructor for class with only static helper methods } /** - * Return the IOException thrown by the remote server wrapped in + * Return the IOException thrown by the remote server wrapped in * ServiceException as cause. * @param se ServiceException that wraps IO exception thrown by the server * @return Exception wrapped in ServiceException or * a new IOException that wraps the unexpected ServiceException. */ public static IOException getRemoteException(ServiceException se) { - Throwable e = se.getCause(); - if (e == null) { - return new IOException(se); - } - return e instanceof IOException ? (IOException) e : new IOException(se); + return ShadedProtobufHelper.getRemoteException(se); } /** - * Kept for backward compatible. + * Extract the remote exception from an unshaded version of the protobuf + * libraries. + * Kept for backward compatibility. * Return the IOException thrown by the remote server wrapped in * ServiceException as cause. * @param se ServiceException that wraps IO exception thrown by the server @@ -71,29 +77,13 @@ public static IOException getRemoteException( return e instanceof IOException ? (IOException) e : new IOException(se); } - /** - * Map used to cache fixed strings to ByteStrings. Since there is no - * automatic expiration policy, only use this for strings from a fixed, small - * set. - *

    - * This map should not be accessed directly. Used the getFixedByteString - * methods instead. - */ - private final static ConcurrentHashMap - FIXED_BYTESTRING_CACHE = new ConcurrentHashMap<>(); - /** * Get the ByteString for frequently used fixed and small set strings. * @param key string * @return the ByteString for frequently used fixed and small set strings. */ public static ByteString getFixedByteString(Text key) { - ByteString value = FIXED_BYTESTRING_CACHE.get(key); - if (value == null) { - value = ByteString.copyFromUtf8(key.toString()); - FIXED_BYTESTRING_CACHE.put(new Text(key.copyBytes()), value); - } - return value; + return ShadedProtobufHelper.getFixedByteString(key); } /** @@ -102,34 +92,40 @@ public static ByteString getFixedByteString(Text key) { * @return ByteString for frequently used fixed and small set strings. */ public static ByteString getFixedByteString(String key) { - ByteString value = FIXED_BYTESTRING_CACHE.get(key); - if (value == null) { - value = ByteString.copyFromUtf8(key); - FIXED_BYTESTRING_CACHE.put(key, value); - } - return value; + return ShadedProtobufHelper.getFixedByteString(key); } + /** + * Get the byte string of a non-null byte array. + * If the array is 0 bytes long, return a singleton to reduce object allocation. + * @param bytes bytes to convert. + * @return a value + */ public static ByteString getByteString(byte[] bytes) { // return singleton to reduce object allocation - return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes); + return ShadedProtobufHelper.getByteString(bytes); } + /** + * Get a token from a TokenProto payload. + * @param tokenProto marshalled token + * @return the token. + */ public static Token tokenFromProto( TokenProto tokenProto) { - Token token = new Token<>( - tokenProto.getIdentifier().toByteArray(), - tokenProto.getPassword().toByteArray(), new Text(tokenProto.getKind()), - new Text(tokenProto.getService())); - return token; + return ShadedProtobufHelper.tokenFromProto(tokenProto); } + /** + * Create a {@code TokenProto} instance + * from a hadoop token. + * This builds and caches the fields + * (identifier, password, kind, service) but not + * renewer or any payload. + * @param tok token + * @return a marshallable protobuf class. + */ public static TokenProto protoFromToken(Token tok) { - TokenProto.Builder builder = TokenProto.newBuilder(). - setIdentifier(getByteString(tok.getIdentifier())). - setPassword(getByteString(tok.getPassword())). - setKindBytes(getFixedByteString(tok.getKind())). - setServiceBytes(getFixedByteString(tok.getService())); - return builder.build(); + return ShadedProtobufHelper.protoFromToken(tok); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufWrapperLegacy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufWrapperLegacy.java new file mode 100644 index 0000000000000..0f264e0dccc85 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufWrapperLegacy.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.util.Preconditions; + +/** + * A RpcWritable wrapper for unshaded protobuf messages. + * This class isolates unshaded protobuf classes from + * the rest of the RPC codebase, so it can operate without + * needing that on the classpath at runtime. + * The classes are needed at compile time; and if + * unshaded protobuf messages are to be marshalled, they + * will need to be on the classpath then. + * That is implicit: it is impossible to pass in a class + * which is a protobuf message unless that condition is met. + */ +@InterfaceAudience.Private +public class ProtobufWrapperLegacy extends RpcWritable { + + private com.google.protobuf.Message message; + + /** + * Construct. + * The type of the parameter is Object so as to keep the casting internal + * to this class. + * @param message message to wrap. + * @throws IllegalArgumentException if the class is not a protobuf message. + */ + public ProtobufWrapperLegacy(Object message) { + Preconditions.checkArgument(isUnshadedProtobufMessage(message), + "message class is not an unshaded protobuf message %s", + message.getClass()); + this.message = (com.google.protobuf.Message) message; + } + + public com.google.protobuf.Message getMessage() { + return message; + } + + + @Override + public void writeTo(ResponseBuffer out) throws IOException { + int length = message.getSerializedSize(); + length += com.google.protobuf.CodedOutputStream. + computeUInt32SizeNoTag(length); + out.ensureCapacity(length); + message.writeDelimitedTo(out); + } + + @SuppressWarnings("unchecked") + @Override + protected T readFrom(ByteBuffer bb) throws IOException { + // using the parser with a byte[]-backed coded input stream is the + // most efficient way to deserialize a protobuf. it has a direct + // path to the PB ctor that doesn't create multi-layered streams + // that internally buffer. + com.google.protobuf.CodedInputStream cis = + com.google.protobuf.CodedInputStream.newInstance( + bb.array(), bb.position() + bb.arrayOffset(), bb.remaining()); + try { + cis.pushLimit(cis.readRawVarint32()); + message = message.getParserForType().parseFrom(cis); + cis.checkLastTagWas(0); + } finally { + // advance over the bytes read. + bb.position(bb.position() + cis.getTotalBytesRead()); + } + return (T) message; + } + + /** + * Has protobuf been looked for and is known as absent? + * Saves a check on every message. + */ + private static final AtomicBoolean PROTOBUF_KNOWN_NOT_FOUND = + new AtomicBoolean(false); + + /** + * Is a message an unshaded protobuf message? + * @param payload payload + * @return true if protobuf.jar is on the classpath and the payload is a Message + */ + public static boolean isUnshadedProtobufMessage(Object payload) { + if (PROTOBUF_KNOWN_NOT_FOUND.get()) { + // protobuf is known to be absent. fail fast without examining + // jars or generating exceptions. + return false; + } + // load the protobuf message class. + // if it does not load, then the payload is guaranteed not to be + // an unshaded protobuf message + // this relies on classloader caching for performance + try { + Class protobufMessageClazz = + Class.forName("com.google.protobuf.Message"); + return protobufMessageClazz.isAssignableFrom(payload.getClass()); + } catch (ClassNotFoundException e) { + PROTOBUF_KNOWN_NOT_FOUND.set(true); + return false; + } + + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java index 4af35ad9270f1..3f0b5e46d796a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java @@ -31,9 +31,9 @@ import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto; import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto; import org.apache.hadoop.net.NetUtils; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class maintains a cache of protocol versions and corresponding protocol @@ -122,12 +122,8 @@ public static boolean isMethodSupported(Object rpcProxy, Class protocol, builder.setProtocol(protocol.getName()); builder.setRpcKind(rpcKind.toString()); GetProtocolSignatureResponseProto resp; - try { - resp = protocolInfoProxy.getProtocolSignature(NULL_CONTROLLER, - builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + resp = ipc(() -> protocolInfoProxy.getProtocolSignature(NULL_CONTROLLER, + builder.build())); versionMap = convertProtocolSignatureProtos(resp .getProtocolSignatureList()); putVersionSignatureMap(serverAddress, protocol.getName(), diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java index f5f0d071f39ed..612c9b0e043cc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java @@ -41,9 +41,11 @@ static RpcWritable wrap(Object o) { if (o instanceof RpcWritable) { return (RpcWritable)o; } else if (o instanceof Message) { + // hadoop shaded protobuf return new ProtobufWrapper((Message)o); - } else if (o instanceof com.google.protobuf.Message) { - return new ProtobufWrapperLegacy((com.google.protobuf.Message) o); + } else if (ProtobufWrapperLegacy.isUnshadedProtobufMessage(o)) { + // unshaded protobuf + return new ProtobufWrapperLegacy(o); } else if (o instanceof Writable) { return new WritableWrapper((Writable)o); } @@ -134,49 +136,6 @@ T readFrom(ByteBuffer bb) throws IOException { } } - // adapter for Protobufs. - static class ProtobufWrapperLegacy extends RpcWritable { - private com.google.protobuf.Message message; - - ProtobufWrapperLegacy(com.google.protobuf.Message message) { - this.message = message; - } - - com.google.protobuf.Message getMessage() { - return message; - } - - @Override - void writeTo(ResponseBuffer out) throws IOException { - int length = message.getSerializedSize(); - length += com.google.protobuf.CodedOutputStream. - computeUInt32SizeNoTag(length); - out.ensureCapacity(length); - message.writeDelimitedTo(out); - } - - @SuppressWarnings("unchecked") - @Override - T readFrom(ByteBuffer bb) throws IOException { - // using the parser with a byte[]-backed coded input stream is the - // most efficient way to deserialize a protobuf. it has a direct - // path to the PB ctor that doesn't create multi-layered streams - // that internally buffer. - com.google.protobuf.CodedInputStream cis = - com.google.protobuf.CodedInputStream.newInstance( - bb.array(), bb.position() + bb.arrayOffset(), bb.remaining()); - try { - cis.pushLimit(cis.readRawVarint32()); - message = message.getParserForType().parseFrom(cis); - cis.checkLastTagWas(0); - } finally { - // advance over the bytes read. - bb.position(bb.position() + cis.getTotalBytesRead()); - } - return (T)message; - } - } - /** * adapter to allow decoding of writables and protobufs from a byte buffer. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/internal/ShadedProtobufHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/internal/ShadedProtobufHelper.java new file mode 100644 index 0000000000000..c0dcdddedb6a0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/internal/ShadedProtobufHelper.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ipc.internal; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +/** + * Helper methods for protobuf related RPC implementation using the + * hadoop {@code org.apache.hadoop.thirdparty.protobuf} shaded version. + * This is absolutely private to hadoop-* modules. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class ShadedProtobufHelper { + + private ShadedProtobufHelper() { + // Hidden constructor for class with only static helper methods + } + + /** + * Return the IOException thrown by the remote server wrapped in + * ServiceException as cause. + * The signature of this method changes with updates to the hadoop-thirdparty + * shaded protobuf library. + * @param se ServiceException that wraps IO exception thrown by the server + * @return Exception wrapped in ServiceException or + * a new IOException that wraps the unexpected ServiceException. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static IOException getRemoteException(ServiceException se) { + Throwable e = se.getCause(); + if (e == null) { + return new IOException(se); + } + return e instanceof IOException + ? (IOException) e + : new IOException(se); + } + + /** + * Map used to cache fixed strings to ByteStrings. Since there is no + * automatic expiration policy, only use this for strings from a fixed, small + * set. + *

    + * This map should not be accessed directly. Used the getFixedByteString + * methods instead. + */ + private static final ConcurrentHashMap + FIXED_BYTESTRING_CACHE = new ConcurrentHashMap<>(); + + /** + * Get the ByteString for frequently used fixed and small set strings. + * @param key Hadoop Writable Text string + * @return the ByteString for frequently used fixed and small set strings. + */ + public static ByteString getFixedByteString(Text key) { + ByteString value = FIXED_BYTESTRING_CACHE.get(key); + if (value == null) { + value = ByteString.copyFromUtf8(key.toString()); + FIXED_BYTESTRING_CACHE.put(new Text(key.copyBytes()), value); + } + return value; + } + + /** + * Get the ByteString for frequently used fixed and small set strings. + * @param key string + * @return ByteString for frequently used fixed and small set strings. + */ + public static ByteString getFixedByteString(String key) { + ByteString value = FIXED_BYTESTRING_CACHE.get(key); + if (value == null) { + value = ByteString.copyFromUtf8(key); + FIXED_BYTESTRING_CACHE.put(key, value); + } + return value; + } + + /** + * Get the byte string of a non-null byte array. + * If the array is 0 bytes long, return a singleton to reduce object allocation. + * @param bytes bytes to convert. + * @return the protobuf byte string representation of the array. + */ + public static ByteString getByteString(byte[] bytes) { + // return singleton to reduce object allocation + return (bytes.length == 0) + ? ByteString.EMPTY + : ByteString.copyFrom(bytes); + } + + /** + * Create a hadoop token from a protobuf token. + * @param tokenProto token + * @return a new token + */ + public static Token tokenFromProto( + TokenProto tokenProto) { + Token token = new Token<>( + tokenProto.getIdentifier().toByteArray(), + tokenProto.getPassword().toByteArray(), + new Text(tokenProto.getKind()), + new Text(tokenProto.getService())); + return token; + } + + /** + * Create a {@code TokenProto} instance + * from a hadoop token. + * This builds and caches the fields + * (identifier, password, kind, service) but not + * renewer or any payload. + * @param tok token + * @return a marshallable protobuf class. + */ + public static TokenProto protoFromToken(Token tok) { + TokenProto.Builder builder = TokenProto.newBuilder(). + setIdentifier(getByteString(tok.getIdentifier())). + setPassword(getByteString(tok.getPassword())). + setKindBytes(getFixedByteString(tok.getKind())). + setServiceBytes(getFixedByteString(tok.getService())); + return builder.build(); + } + + /** + * Evaluate a protobuf call, converting any ServiceException to an IOException. + * @param call invocation to make + * @return the result of the call + * @param type of the result + * @throws IOException any translated protobuf exception + */ + public static T ipc(IpcCall call) throws IOException { + try { + return call.call(); + } catch (ServiceException e) { + throw getRemoteException(e); + } + } + + @FunctionalInterface + public interface IpcCall { + T call() throws ServiceException; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/internal/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/internal/package-info.java new file mode 100644 index 0000000000000..0ff2ba9ab7ca5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/internal/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * IPC internal classes not for any use by libraries outside + * the apache hadoop source tree. + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "YARN"}) +@InterfaceStability.Unstable +package org.apache.hadoop.ipc.internal; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java index cb35e938483b0..f91fa23194c45 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java @@ -18,8 +18,12 @@ /** * Tools to help define network clients and servers. + * Other ASF projects use this package, often with their own shaded/unshaded + * versions of protobuf messages. + * Changes to the API signatures will break things, especially changes to + * {@link org.apache.hadoop.ipc.RPC} and {@link org.apache.hadoop.ipc.RpcEngine}. */ -@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "MapReduce"}) +@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "MapReduce", "YARN", "Hive", "Ozone"}) @InterfaceStability.Evolving package org.apache.hadoop.ipc; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java index f8a3e25867f91..78413be68d1db 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java @@ -25,7 +25,6 @@ import java.util.Collection; import java.util.List; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RefreshResponse; @@ -34,9 +33,9 @@ import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshRequestProto; import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshResponseProto; import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshResponseCollectionProto; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; public class GenericRefreshProtocolClientSideTranslatorPB implements ProtocolMetaInterface, GenericRefreshProtocol, Closeable { @@ -59,17 +58,13 @@ public void close() throws IOException { public Collection refresh(String identifier, String[] args) throws IOException { List argList = Arrays.asList(args); - try { - GenericRefreshRequestProto request = GenericRefreshRequestProto.newBuilder() - .setIdentifier(identifier) - .addAllArgs(argList) - .build(); + GenericRefreshRequestProto request = GenericRefreshRequestProto.newBuilder() + .setIdentifier(identifier).addAllArgs(argList).build(); + + GenericRefreshResponseCollectionProto resp = ipc(() -> + rpcProxy.refresh(NULL_CONTROLLER, request)); + return unpack(resp); - GenericRefreshResponseCollectionProto resp = rpcProxy.refresh(NULL_CONTROLLER, request); - return unpack(resp); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } } private Collection unpack(GenericRefreshResponseCollectionProto collection) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java index e378a93f87906..46182ca16527b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java @@ -21,16 +21,14 @@ import java.io.Closeable; import java.io.IOException; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; import org.apache.hadoop.ipc.RefreshCallQueueProtocol; import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueRequestProto; -import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; public class RefreshCallQueueProtocolClientSideTranslatorPB implements ProtocolMetaInterface, RefreshCallQueueProtocol, Closeable { @@ -55,12 +53,8 @@ public void close() throws IOException { @Override public void refreshCallQueue() throws IOException { - try { - rpcProxy.refreshCallQueue(NULL_CONTROLLER, - VOID_REFRESH_CALL_QUEUE_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshCallQueue(NULL_CONTROLLER, + VOID_REFRESH_CALL_QUEUE_REQUEST)); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java index ef309cb2247fd..bbbcc95288888 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java @@ -18,6 +18,7 @@ package org.apache.hadoop.security; +import org.apache.hadoop.ipc.internal.ShadedProtobufHelper; import org.apache.hadoop.thirdparty.protobuf.ByteString; import java.io.BufferedInputStream; @@ -46,7 +47,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.proto.SecurityProtos.CredentialsKVProto; @@ -382,7 +382,7 @@ void writeProto(DataOutput out) throws IOException { CredentialsKVProto.Builder kv = CredentialsKVProto.newBuilder(). setAliasBytes(ByteString.copyFrom( e.getKey().getBytes(), 0, e.getKey().getLength())). - setToken(ProtobufHelper.protoFromToken(e.getValue())); + setToken(ShadedProtobufHelper.protoFromToken(e.getValue())); storage.addTokens(kv.build()); } @@ -404,7 +404,7 @@ void readProto(DataInput in) throws IOException { CredentialsProto storage = CredentialsProto.parseDelimitedFrom((DataInputStream)in); for (CredentialsKVProto kv : storage.getTokensList()) { addToken(new Text(kv.getAliasBytes().toByteArray()), - ProtobufHelper.tokenFromProto(kv.getToken())); + ShadedProtobufHelper.tokenFromProto(kv.getToken())); } for (CredentialsKVProto kv : storage.getSecretsList()) { addSecretKey(new Text(kv.getAliasBytes().toByteArray()), diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java index 6e7a856aecc39..49887bdd2908a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java @@ -21,16 +21,14 @@ import java.io.Closeable; import java.io.IOException; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.security.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshServiceAclRequestProto; -import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; public class RefreshAuthorizationPolicyProtocolClientSideTranslatorPB implements ProtocolMetaInterface, RefreshAuthorizationPolicyProtocol, Closeable { @@ -55,12 +53,8 @@ public void close() throws IOException { @Override public void refreshServiceAcl() throws IOException { - try { - rpcProxy.refreshServiceAcl(NULL_CONTROLLER, - VOID_REFRESH_SERVICE_ACL_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshServiceAcl(NULL_CONTROLLER, + VOID_REFRESH_SERVICE_ACL_REQUEST)); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java index ac4003881ba5c..cb80d067bb8bc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java @@ -21,16 +21,15 @@ import java.io.Closeable; import java.io.IOException; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshSuperUserGroupsConfigurationRequestProto; import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserToGroupsMappingsRequestProto; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; public class RefreshUserMappingsProtocolClientSideTranslatorPB implements ProtocolMetaInterface, RefreshUserMappingsProtocol, Closeable { @@ -59,22 +58,14 @@ public void close() throws IOException { @Override public void refreshUserToGroupsMappings() throws IOException { - try { - rpcProxy.refreshUserToGroupsMappings(NULL_CONTROLLER, - VOID_REFRESH_USER_TO_GROUPS_MAPPING_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshUserToGroupsMappings(NULL_CONTROLLER, + VOID_REFRESH_USER_TO_GROUPS_MAPPING_REQUEST)); } @Override public void refreshSuperUserGroupsConfiguration() throws IOException { - try { - rpcProxy.refreshSuperUserGroupsConfiguration(NULL_CONTROLLER, - VOID_REFRESH_SUPERUSER_GROUPS_CONFIGURATION_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshSuperUserGroupsConfiguration(NULL_CONTROLLER, + VOID_REFRESH_SUPERUSER_GROUPS_CONFIGURATION_REQUEST)); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java index b0c0fda542448..8d8885adcba9a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java @@ -20,7 +20,7 @@ import java.io.Closeable; import java.io.IOException; -import org.apache.hadoop.ipc.ProtobufHelper; + import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; @@ -29,7 +29,8 @@ import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetGroupsForUserResponseProto; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; public class GetUserMappingsProtocolClientSideTranslatorPB implements ProtocolMetaInterface, GetUserMappingsProtocol, Closeable { @@ -53,11 +54,7 @@ public String[] getGroupsForUser(String user) throws IOException { GetGroupsForUserRequestProto request = GetGroupsForUserRequestProto .newBuilder().setUser(user).build(); GetGroupsForUserResponseProto resp; - try { - resp = rpcProxy.getGroupsForUser(NULL_CONTROLLER, request); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + resp = ipc(() -> rpcProxy.getGroupsForUser(NULL_CONTROLLER, request)); return resp.getGroupsList().toArray(new String[resp.getGroupsCount()]); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestShadedProtobufHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestShadedProtobufHelper.java new file mode 100644 index 0000000000000..fb4e83168566b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestShadedProtobufHelper.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import java.io.IOException; + +import org.junit.Test; + +import org.apache.hadoop.ipc.internal.ShadedProtobufHelper; +import org.apache.hadoop.test.AbstractHadoopTestBase; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.apache.hadoop.test.LambdaTestUtils.verifyCause; + +/** + * Test methods in {@link ShadedProtobufHelper}. + */ +public class TestShadedProtobufHelper extends AbstractHadoopTestBase { + + @Test + public void testExtractRemoteExceptionNoCause() throws Throwable { + ServiceException source = new ServiceException("empty"); + + IOException ex = ShadedProtobufHelper.getRemoteException(source); + verifyCause(ServiceException.class, ex); + } + + @Test + public void testExtractRemoteExceptionIOECause() throws Throwable { + IOException source = new IOException("ioe"); + + IOException ex = ShadedProtobufHelper.getRemoteException( + new ServiceException(source)); + // if not the same, throw + if (!(ex == source)) { + throw ex; + } + } + + @Test + public void testExtractRemoteExceptionOtherCause() throws Throwable { + NullPointerException source = new NullPointerException("npe"); + + IOException ex = ShadedProtobufHelper.getRemoteException( + new ServiceException(source)); + // if not the same, throw + ServiceException c1 = verifyCause(ServiceException.class, ex); + verifyCause(NullPointerException.class, c1); + } + + @Test + public void testIPCWrapperServiceException() throws Throwable { + intercept(IOException.class, "expected", () -> { + ipc(() -> { + throw new ServiceException("expected"); + }); + }); + } + + @Test + public void testIPCWrapperNPE() throws Throwable { + final IOException ex = intercept(IOException.class, "npe", () -> { + ipc(() -> { + throw new ServiceException(new NullPointerException("npe")); + }); + }); + ServiceException c1 = verifyCause(ServiceException.class, ex); + verifyCause(NullPointerException.class, c1); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java index feafeb8f25dbb..b968fecf4805c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java @@ -819,7 +819,7 @@ public static E verifyCause( if (cause == null || !clazz.isAssignableFrom(cause.getClass())) { throw caught; } else { - return (E) caught; + return (E) cause; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java index 47234e8b65d78..9aa34135caa17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java @@ -67,7 +67,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus; import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus.Result; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.ProtocolTranslator; @@ -76,13 +75,13 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; + /** * This class is the client side translator to translate the requests made on * {@link ClientDatanodeProtocol} interfaces to the RPC server implementing @@ -197,31 +196,19 @@ public long getReplicaVisibleLength(ExtendedBlock b) throws IOException { GetReplicaVisibleLengthRequestProto req = GetReplicaVisibleLengthRequestProto.newBuilder() .setBlock(PBHelperClient.convert(b)).build(); - try { - return rpcProxy.getReplicaVisibleLength(NULL_CONTROLLER, req).getLength(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.getReplicaVisibleLength(NULL_CONTROLLER, req).getLength()); } @Override public void refreshNamenodes() throws IOException { - try { - rpcProxy.refreshNamenodes(NULL_CONTROLLER, VOID_REFRESH_NAMENODES); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.refreshNamenodes(NULL_CONTROLLER, VOID_REFRESH_NAMENODES)); } @Override public void deleteBlockPool(String bpid, boolean force) throws IOException { DeleteBlockPoolRequestProto req = DeleteBlockPoolRequestProto.newBuilder() .setBlockPool(bpid).setForce(force).build(); - try { - rpcProxy.deleteBlockPool(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.deleteBlockPool(NULL_CONTROLLER, req)); } @Override @@ -232,11 +219,7 @@ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block, .setBlock(PBHelperClient.convert(block)) .setToken(PBHelperClient.convert(token)).build(); GetBlockLocalPathInfoResponseProto resp; - try { - resp = rpcProxy.getBlockLocalPathInfo(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + resp = ipc(() -> rpcProxy.getBlockLocalPathInfo(NULL_CONTROLLER, req)); return new BlockLocalPathInfo(PBHelperClient.convert(resp.getBlock()), resp.getLocalPath(), resp.getLocalMetaPath()); } @@ -257,94 +240,61 @@ public Object getUnderlyingProxyObject() { public void shutdownDatanode(boolean forUpgrade) throws IOException { ShutdownDatanodeRequestProto request = ShutdownDatanodeRequestProto .newBuilder().setForUpgrade(forUpgrade).build(); - try { - rpcProxy.shutdownDatanode(NULL_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.shutdownDatanode(NULL_CONTROLLER, request)); } @Override public void evictWriters() throws IOException { - try { - rpcProxy.evictWriters(NULL_CONTROLLER, VOID_EVICT_WRITERS); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.evictWriters(NULL_CONTROLLER, VOID_EVICT_WRITERS)); } @Override public DatanodeLocalInfo getDatanodeInfo() throws IOException { GetDatanodeInfoResponseProto response; - try { - response = rpcProxy.getDatanodeInfo(NULL_CONTROLLER, - VOID_GET_DATANODE_INFO); - return PBHelperClient.convert(response.getLocalInfo()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + response = ipc(() -> rpcProxy.getDatanodeInfo(NULL_CONTROLLER, + VOID_GET_DATANODE_INFO)); + return PBHelperClient.convert(response.getLocalInfo()); } @Override public void startReconfiguration() throws IOException { - try { - rpcProxy.startReconfiguration(NULL_CONTROLLER, VOID_START_RECONFIG); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.startReconfiguration(NULL_CONTROLLER, VOID_START_RECONFIG)); } @Override public ReconfigurationTaskStatus getReconfigurationStatus() throws IOException { - try { - return ReconfigurationProtocolUtils.getReconfigurationStatus( - rpcProxy - .getReconfigurationStatus( - NULL_CONTROLLER, - VOID_GET_RECONFIG_STATUS)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ReconfigurationProtocolUtils.getReconfigurationStatus( + ipc(() -> rpcProxy.getReconfigurationStatus( + NULL_CONTROLLER, + VOID_GET_RECONFIG_STATUS))); } @Override public List listReconfigurableProperties() throws IOException { ListReconfigurablePropertiesResponseProto response; - try { - response = rpcProxy.listReconfigurableProperties(NULL_CONTROLLER, - VOID_LIST_RECONFIGURABLE_PROPERTIES); - return response.getNameList(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + response = ipc(() -> rpcProxy.listReconfigurableProperties(NULL_CONTROLLER, + VOID_LIST_RECONFIGURABLE_PROPERTIES)); + return response.getNameList(); } @Override public void triggerBlockReport(BlockReportOptions options) throws IOException { - try { - TriggerBlockReportRequestProto.Builder builder = TriggerBlockReportRequestProto.newBuilder(). - setIncremental(options.isIncremental()); - if (options.getNamenodeAddr() != null) { - builder.setNnAddress(NetUtils.getHostPortString(options.getNamenodeAddr())); - } - rpcProxy.triggerBlockReport(NULL_CONTROLLER, builder.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + TriggerBlockReportRequestProto.Builder builder = TriggerBlockReportRequestProto.newBuilder(). + setIncremental(options.isIncremental()); + if (options.getNamenodeAddr() != null) { + builder.setNnAddress(NetUtils.getHostPortString(options.getNamenodeAddr())); } + ipc(() -> rpcProxy.triggerBlockReport(NULL_CONTROLLER, builder.build())); } @Override public long getBalancerBandwidth() throws IOException { GetBalancerBandwidthResponseProto response; - try { - response = rpcProxy.getBalancerBandwidth(NULL_CONTROLLER, - VOID_GET_BALANCER_BANDWIDTH); - return response.getBandwidth(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + response = ipc(() -> rpcProxy.getBalancerBandwidth(NULL_CONTROLLER, + VOID_GET_BALANCER_BANDWIDTH)); + return response.getBandwidth(); } /** @@ -363,19 +313,15 @@ public long getBalancerBandwidth() throws IOException { public void submitDiskBalancerPlan(String planID, long planVersion, String planFile, String planData, boolean skipDateCheck) throws IOException { - try { - SubmitDiskBalancerPlanRequestProto request = - SubmitDiskBalancerPlanRequestProto.newBuilder() - .setPlanID(planID) - .setPlanVersion(planVersion) - .setPlanFile(planFile) - .setPlan(planData) - .setIgnoreDateCheck(skipDateCheck) - .build(); - rpcProxy.submitDiskBalancerPlan(NULL_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + SubmitDiskBalancerPlanRequestProto request = + SubmitDiskBalancerPlanRequestProto.newBuilder() + .setPlanID(planID) + .setPlanVersion(planVersion) + .setPlanFile(planFile) + .setPlan(planData) + .setIgnoreDateCheck(skipDateCheck) + .build(); + ipc(() -> rpcProxy.submitDiskBalancerPlan(NULL_CONTROLLER, request)); } /** @@ -387,13 +333,9 @@ public void submitDiskBalancerPlan(String planID, long planVersion, @Override public void cancelDiskBalancePlan(String planID) throws IOException { - try { - CancelPlanRequestProto request = CancelPlanRequestProto.newBuilder() - .setPlanID(planID).build(); - rpcProxy.cancelDiskBalancerPlan(NULL_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + CancelPlanRequestProto request = CancelPlanRequestProto.newBuilder() + .setPlanID(planID).build(); + ipc(() -> rpcProxy.cancelDiskBalancerPlan(NULL_CONTROLLER, request)); } /** @@ -401,56 +343,44 @@ public void cancelDiskBalancePlan(String planID) */ @Override public DiskBalancerWorkStatus queryDiskBalancerPlan() throws IOException { - try { - QueryPlanStatusRequestProto request = - QueryPlanStatusRequestProto.newBuilder().build(); - QueryPlanStatusResponseProto response = - rpcProxy.queryDiskBalancerPlan(NULL_CONTROLLER, request); - DiskBalancerWorkStatus.Result result = Result.NO_PLAN; - if(response.hasResult()) { - result = DiskBalancerWorkStatus.Result.values()[ - response.getResult()]; - } - - return new DiskBalancerWorkStatus(result, - response.hasPlanID() ? response.getPlanID() : null, - response.hasPlanFile() ? response.getPlanFile() : null, - response.hasCurrentStatus() ? response.getCurrentStatus() : null); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + QueryPlanStatusRequestProto request = + QueryPlanStatusRequestProto.newBuilder().build(); + QueryPlanStatusResponseProto response = + ipc(() -> rpcProxy.queryDiskBalancerPlan(NULL_CONTROLLER, request)); + DiskBalancerWorkStatus.Result result = Result.NO_PLAN; + if(response.hasResult()) { + result = DiskBalancerWorkStatus.Result.values()[ + response.getResult()]; } + + return new DiskBalancerWorkStatus(result, + response.hasPlanID() ? response.getPlanID() : null, + response.hasPlanFile() ? response.getPlanFile() : null, + response.hasCurrentStatus() ? response.getCurrentStatus() : null); } @Override public String getDiskBalancerSetting(String key) throws IOException { - try { - DiskBalancerSettingRequestProto request = - DiskBalancerSettingRequestProto.newBuilder().setKey(key).build(); - DiskBalancerSettingResponseProto response = - rpcProxy.getDiskBalancerSetting(NULL_CONTROLLER, request); - return response.hasValue() ? response.getValue() : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + DiskBalancerSettingRequestProto request = + DiskBalancerSettingRequestProto.newBuilder().setKey(key).build(); + DiskBalancerSettingResponseProto response = + ipc(() -> rpcProxy.getDiskBalancerSetting(NULL_CONTROLLER, request)); + return response.hasValue() ? response.getValue() : null; } @Override public List getVolumeReport() throws IOException { - try { - List volumeInfoList = new ArrayList<>(); - GetVolumeReportResponseProto volumeReport = rpcProxy.getVolumeReport( - NULL_CONTROLLER, VOID_GET_DATANODE_STORAGE_INFO); - List volumeProtoList = volumeReport - .getVolumeInfoList(); - for (DatanodeVolumeInfoProto proto : volumeProtoList) { - volumeInfoList.add(new DatanodeVolumeInfo(proto.getPath(), proto - .getUsedSpace(), proto.getFreeSpace(), proto.getReservedSpace(), - proto.getReservedSpaceForReplicas(), proto.getNumBlocks(), - PBHelperClient.convertStorageType(proto.getStorageType()))); - } - return volumeInfoList; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + List volumeInfoList = new ArrayList<>(); + GetVolumeReportResponseProto volumeReport = ipc(() -> rpcProxy.getVolumeReport( + NULL_CONTROLLER, VOID_GET_DATANODE_STORAGE_INFO)); + List volumeProtoList = volumeReport + .getVolumeInfoList(); + for (DatanodeVolumeInfoProto proto : volumeProtoList) { + volumeInfoList.add(new DatanodeVolumeInfo(proto.getPath(), proto + .getUsedSpace(), proto.getFreeSpace(), proto.getReservedSpace(), + proto.getReservedSpaceForReplicas(), proto.getNumBlocks(), + PBHelperClient.convertStorageType(proto.getStorageType()))); } + return volumeInfoList; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 541a4361896dc..ae4a84ead6552 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -238,7 +238,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.retry.AsyncCallHandler; import org.apache.hadoop.ipc.Client; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.ProtocolTranslator; @@ -258,6 +257,9 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.concurrent.AsyncGet; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.getRemoteException; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; + /** * This class forwards NN's ClientProtocol calls as RPC calls to the NN server * while translating from the parameter types used in ClientProtocol to the @@ -332,25 +334,17 @@ public LocatedBlocks getBlockLocations(String src, long offset, long length) .setOffset(offset) .setLength(length) .build(); - try { - GetBlockLocationsResponseProto resp = rpcProxy.getBlockLocations(null, - req); - return resp.hasLocations() ? - PBHelperClient.convert(resp.getLocations()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetBlockLocationsResponseProto resp = ipc(() -> rpcProxy.getBlockLocations(null, + req)); + return resp.hasLocations() ? + PBHelperClient.convert(resp.getLocations()) : null; } @Override public FsServerDefaults getServerDefaults() throws IOException { GetServerDefaultsRequestProto req = VOID_GET_SERVER_DEFAULT_REQUEST; - try { - return PBHelperClient - .convert(rpcProxy.getServerDefaults(null, req).getServerDefaults()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient + .convert(ipc(() -> rpcProxy.getServerDefaults(null, req).getServerDefaults())); } @Override @@ -381,13 +375,8 @@ public HdfsFileStatus create(String src, FsPermission masked, builder.addAllCryptoProtocolVersion( PBHelperClient.convert(supportedVersions)); CreateRequestProto req = builder.build(); - try { - CreateResponseProto res = rpcProxy.create(null, req); - return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - + CreateResponseProto res = ipc(() -> rpcProxy.create(null, req)); + return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null; } @Override @@ -398,11 +387,7 @@ public boolean truncate(String src, long newLength, String clientName) .setNewLength(newLength) .setClientName(clientName) .build(); - try { - return rpcProxy.truncate(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.truncate(null, req).getResult()); } @Override @@ -412,16 +397,12 @@ public LastBlockWithStatus append(String src, String clientName, .setClientName(clientName).setFlag( PBHelperClient.convertCreateFlag(flag)) .build(); - try { - AppendResponseProto res = rpcProxy.append(null, req); - LocatedBlock lastBlock = res.hasBlock() ? PBHelperClient - .convertLocatedBlockProto(res.getBlock()) : null; - HdfsFileStatus stat = (res.hasStat()) ? - PBHelperClient.convert(res.getStat()) : null; - return new LastBlockWithStatus(lastBlock, stat); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + AppendResponseProto res = ipc(() -> rpcProxy.append(null, req)); + LocatedBlock lastBlock = res.hasBlock() ? PBHelperClient + .convertLocatedBlockProto(res.getBlock()) : null; + HdfsFileStatus stat = (res.hasStat()) ? + PBHelperClient.convert(res.getStat()) : null; + return new LastBlockWithStatus(lastBlock, stat); } @Override @@ -431,11 +412,7 @@ public boolean setReplication(String src, short replication) .setSrc(src) .setReplication(replication) .build(); - try { - return rpcProxy.setReplication(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.setReplication(null, req)).getResult(); } @Override @@ -445,15 +422,11 @@ public void setPermission(String src, FsPermission permission) .setSrc(src) .setPermission(PBHelperClient.convert(permission)) .build(); - try { - if (Client.isAsynchronousMode()) { - rpcProxy.setPermission(null, req); - setAsyncReturnValue(); - } else { - rpcProxy.setPermission(null, req); - } - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (Client.isAsynchronousMode()) { + ipc(() -> rpcProxy.setPermission(null, req)); + setAsyncReturnValue(); + } else { + ipc(() -> rpcProxy.setPermission(null, req)); } } @@ -485,15 +458,11 @@ public void setOwner(String src, String username, String groupname) req.setUsername(username); if (groupname != null) req.setGroupname(groupname); - try { - if (Client.isAsynchronousMode()) { - rpcProxy.setOwner(null, req.build()); - setAsyncReturnValue(); - } else { - rpcProxy.setOwner(null, req.build()); - } - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (Client.isAsynchronousMode()) { + ipc(() -> rpcProxy.setOwner(null, req.build())); + setAsyncReturnValue(); + } else { + ipc(() -> rpcProxy.setOwner(null, req.build())); } } @@ -503,11 +472,7 @@ public void abandonBlock(ExtendedBlock b, long fileId, String src, AbandonBlockRequestProto req = AbandonBlockRequestProto.newBuilder() .setB(PBHelperClient.convert(b)).setSrc(src).setHolder(holder) .setFileId(fileId).build(); - try { - rpcProxy.abandonBlock(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.abandonBlock(null, req)); } @Override @@ -528,12 +493,8 @@ public LocatedBlock addBlock(String src, String clientName, req.addAllFlags(PBHelperClient.convertAddBlockFlags( addBlockFlags)); } - try { - return PBHelperClient.convertLocatedBlockProto( - rpcProxy.addBlock(null, req.build()).getBlock()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convertLocatedBlockProto( + ipc(() -> rpcProxy.addBlock(null, req.build())).getBlock()); } @Override @@ -552,12 +513,8 @@ public LocatedBlock getAdditionalDatanode(String src, long fileId, .setNumAdditionalNodes(numAdditionalNodes) .setClientName(clientName) .build(); - try { - return PBHelperClient.convertLocatedBlockProto( - rpcProxy.getAdditionalDatanode(null, req).getBlock()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convertLocatedBlockProto( + ipc(() -> rpcProxy.getAdditionalDatanode(null, req)).getBlock()); } @Override @@ -569,11 +526,7 @@ public boolean complete(String src, String clientName, .setFileId(fileId); if (last != null) req.setLast(PBHelperClient.convert(last)); - try { - return rpcProxy.complete(null, req.build()).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.complete(null, req.build())).getResult(); } @Override @@ -582,11 +535,7 @@ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException { .addAllBlocks(Arrays.asList( PBHelperClient.convertLocatedBlocks(blocks))) .build(); - try { - rpcProxy.reportBadBlocks(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.reportBadBlocks(null, req)); } @Override @@ -595,11 +544,7 @@ public boolean rename(String src, String dst) throws IOException { .setSrc(src) .setDst(dst).build(); - try { - return rpcProxy.rename(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.rename(null, req)).getResult(); } @@ -624,17 +569,12 @@ public void rename2(String src, String dst, Rename... options) setOverwriteDest(overwrite). setMoveToTrash(toTrash). build(); - try { - if (Client.isAsynchronousMode()) { - rpcProxy.rename2(null, req); - setAsyncReturnValue(); - } else { - rpcProxy.rename2(null, req); - } - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (Client.isAsynchronousMode()) { + ipc(() -> rpcProxy.rename2(null, req)); + setAsyncReturnValue(); + } else { + ipc(() -> rpcProxy.rename2(null, req)); } - } @Override @@ -642,11 +582,7 @@ public void concat(String trg, String[] srcs) throws IOException { ConcatRequestProto req = ConcatRequestProto.newBuilder(). setTrg(trg). addAllSrcs(Arrays.asList(srcs)).build(); - try { - rpcProxy.concat(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.concat(null, req)); } @@ -654,11 +590,7 @@ public void concat(String trg, String[] srcs) throws IOException { public boolean delete(String src, boolean recursive) throws IOException { DeleteRequestProto req = DeleteRequestProto.newBuilder().setSrc(src) .setRecursive(recursive).build(); - try { - return rpcProxy.delete(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.delete(null, req).getResult()); } @Override @@ -673,11 +605,7 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent) builder.setUnmasked(PBHelperClient.convert(unmasked)); } MkdirsRequestProto req = builder.build(); - try { - return rpcProxy.mkdirs(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.mkdirs(null, req)).getResult(); } @Override @@ -687,16 +615,11 @@ public DirectoryListing getListing(String src, byte[] startAfter, .setSrc(src) .setStartAfter(ByteString.copyFrom(startAfter)) .setNeedLocation(needLocation).build(); - try { - GetListingResponseProto result = rpcProxy.getListing(null, req); - - if (result.hasDirList()) { - return PBHelperClient.convert(result.getDirList()); - } - return null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetListingResponseProto result = ipc(() -> rpcProxy.getListing(null, req)); + if (result.hasDirList()) { + return PBHelperClient.convert(result.getDirList()); } + return null; } @Override @@ -708,38 +631,34 @@ public BatchedDirectoryListing getBatchedListing( .addAllPaths(Arrays.asList(srcs)) .setStartAfter(ByteString.copyFrom(startAfter)) .setNeedLocation(needLocation).build(); - try { - GetBatchedListingResponseProto result = - rpcProxy.getBatchedListing(null, req); - - if (result.getListingsCount() > 0) { - HdfsPartialListing[] listingArray = - new HdfsPartialListing[result.getListingsCount()]; - int listingIdx = 0; - for (BatchedDirectoryListingProto proto : result.getListingsList()) { - HdfsPartialListing listing; - if (proto.hasException()) { - HdfsProtos.RemoteExceptionProto reProto = proto.getException(); - RemoteException ex = new RemoteException( - reProto.getClassName(), reProto.getMessage()); - listing = new HdfsPartialListing(proto.getParentIdx(), ex); - } else { - List statuses = - PBHelperClient.convertHdfsFileStatus( - proto.getPartialListingList()); - listing = new HdfsPartialListing(proto.getParentIdx(), statuses); - } - listingArray[listingIdx++] = listing; + GetBatchedListingResponseProto result = + ipc(() -> rpcProxy.getBatchedListing(null, req)); + + if (result.getListingsCount() > 0) { + HdfsPartialListing[] listingArray = + new HdfsPartialListing[result.getListingsCount()]; + int listingIdx = 0; + for (BatchedDirectoryListingProto proto : result.getListingsList()) { + HdfsPartialListing listing; + if (proto.hasException()) { + HdfsProtos.RemoteExceptionProto reProto = proto.getException(); + RemoteException ex = new RemoteException( + reProto.getClassName(), reProto.getMessage()); + listing = new HdfsPartialListing(proto.getParentIdx(), ex); + } else { + List statuses = + PBHelperClient.convertHdfsFileStatus( + proto.getPartialListingList()); + listing = new HdfsPartialListing(proto.getParentIdx(), statuses); } - BatchedDirectoryListing batchedListing = - new BatchedDirectoryListing(listingArray, result.getHasMore(), - result.getStartAfter().toByteArray()); - return batchedListing; + listingArray[listingIdx++] = listing; } - return null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + BatchedDirectoryListing batchedListing = + new BatchedDirectoryListing(listingArray, result.getHasMore(), + result.getStartAfter().toByteArray()); + return batchedListing; } + return null; } @@ -751,11 +670,7 @@ public void renewLease(String clientName, List namespaces) if (namespaces != null && !namespaces.isEmpty()) { builder.addAllNamespaces(namespaces); } - try { - rpcProxy.renewLease(null, builder.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.renewLease(null, builder.build())); } @Override @@ -764,41 +679,25 @@ public boolean recoverLease(String src, String clientName) RecoverLeaseRequestProto req = RecoverLeaseRequestProto.newBuilder() .setSrc(src) .setClientName(clientName).build(); - try { - return rpcProxy.recoverLease(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.recoverLease(null, req)).getResult(); } @Override public long[] getStats() throws IOException { - try { - return PBHelperClient.convert(rpcProxy.getFsStats(null, - VOID_GET_FSSTATUS_REQUEST)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getFsStats(null, + VOID_GET_FSSTATUS_REQUEST))); } @Override public ReplicatedBlockStats getReplicatedBlockStats() throws IOException { - try { - return PBHelperClient.convert(rpcProxy.getFsReplicatedBlockStats(null, - VOID_GET_FS_REPLICATED_BLOCK_STATS_REQUEST)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getFsReplicatedBlockStats(null, + VOID_GET_FS_REPLICATED_BLOCK_STATS_REQUEST))); } @Override public ECBlockGroupStats getECBlockGroupStats() throws IOException { - try { - return PBHelperClient.convert(rpcProxy.getFsECBlockGroupStats(null, - VOID_GET_FS_ECBLOCKGROUP_STATS_REQUEST)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getFsECBlockGroupStats(null, + VOID_GET_FS_ECBLOCKGROUP_STATS_REQUEST))); } @Override @@ -807,12 +706,8 @@ public DatanodeInfo[] getDatanodeReport(DatanodeReportType type) GetDatanodeReportRequestProto req = GetDatanodeReportRequestProto .newBuilder() .setType(PBHelperClient.convert(type)).build(); - try { - return PBHelperClient.convert( - rpcProxy.getDatanodeReport(null, req).getDiList()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert( + ipc(() -> rpcProxy.getDatanodeReport(null, req)).getDiList()); } @Override @@ -821,13 +716,9 @@ public DatanodeStorageReport[] getDatanodeStorageReport( final GetDatanodeStorageReportRequestProto req = GetDatanodeStorageReportRequestProto.newBuilder() .setType(PBHelperClient.convert(type)).build(); - try { - return PBHelperClient.convertDatanodeStorageReports( - rpcProxy.getDatanodeStorageReport(null, req) - .getDatanodeStorageReportsList()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convertDatanodeStorageReports( + ipc(() -> rpcProxy.getDatanodeStorageReport(null, req) + .getDatanodeStorageReportsList())); } @Override @@ -836,11 +727,7 @@ public long getPreferredBlockSize(String filename) throws IOException { .newBuilder() .setFilename(filename) .build(); - try { - return rpcProxy.getPreferredBlockSize(null, req).getBsize(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.getPreferredBlockSize(null, req)).getBsize(); } @Override @@ -849,33 +736,21 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) SetSafeModeRequestProto req = SetSafeModeRequestProto.newBuilder() .setAction(PBHelperClient.convert(action)) .setChecked(isChecked).build(); - try { - return rpcProxy.setSafeMode(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.setSafeMode(null, req)).getResult(); } @Override public boolean saveNamespace(long timeWindow, long txGap) throws IOException { - try { - SaveNamespaceRequestProto req = SaveNamespaceRequestProto.newBuilder() - .setTimeWindow(timeWindow).setTxGap(txGap).build(); - return rpcProxy.saveNamespace(null, req).getSaved(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + SaveNamespaceRequestProto req = SaveNamespaceRequestProto.newBuilder() + .setTimeWindow(timeWindow).setTxGap(txGap).build(); + return ipc(() -> rpcProxy.saveNamespace(null, req)).getSaved(); } @Override public long rollEdits() throws IOException { - try { - RollEditsResponseProto resp = rpcProxy.rollEdits(null, - VOID_ROLLEDITS_REQUEST); - return resp.getNewSegmentTxId(); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + RollEditsResponseProto resp = ipc(() -> rpcProxy.rollEdits(null, + VOID_ROLLEDITS_REQUEST)); + return resp.getNewSegmentTxId(); } @Override @@ -883,40 +758,24 @@ public boolean restoreFailedStorage(String arg) throws IOException{ RestoreFailedStorageRequestProto req = RestoreFailedStorageRequestProto .newBuilder() .setArg(arg).build(); - try { - return rpcProxy.restoreFailedStorage(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.restoreFailedStorage(null, req)).getResult(); } @Override public void refreshNodes() throws IOException { - try { - rpcProxy.refreshNodes(null, VOID_REFRESH_NODES_REQUEST); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.refreshNodes(null, VOID_REFRESH_NODES_REQUEST)); } @Override public void finalizeUpgrade() throws IOException { - try { - rpcProxy.finalizeUpgrade(null, VOID_FINALIZE_UPGRADE_REQUEST); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.finalizeUpgrade(null, VOID_FINALIZE_UPGRADE_REQUEST)); } @Override public boolean upgradeStatus() throws IOException { - try { - final UpgradeStatusResponseProto proto = rpcProxy.upgradeStatus( - null, VOID_UPGRADE_STATUS_REQUEST); - return proto.getUpgradeFinalized(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + final UpgradeStatusResponseProto proto = ipc(() -> rpcProxy.upgradeStatus( + null, VOID_UPGRADE_STATUS_REQUEST)); + return proto.getUpgradeFinalized(); } @Override @@ -924,16 +783,12 @@ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException { final RollingUpgradeRequestProto r = RollingUpgradeRequestProto.newBuilder() .setAction(PBHelperClient.convert(action)).build(); - try { - final RollingUpgradeResponseProto proto = - rpcProxy.rollingUpgrade(null, r); - if (proto.hasRollingUpgradeInfo()) { - return PBHelperClient.convert(proto.getRollingUpgradeInfo()); - } - return null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + final RollingUpgradeResponseProto proto = + ipc(() -> rpcProxy.rollingUpgrade(null, r)); + if (proto.hasRollingUpgradeInfo()) { + return PBHelperClient.convert(proto.getRollingUpgradeInfo()); } + return null; } @Override @@ -943,24 +798,15 @@ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie) ListCorruptFileBlocksRequestProto.newBuilder().setPath(path); if (cookie != null) req.setCookie(cookie); - try { - return PBHelperClient.convert( - rpcProxy.listCorruptFileBlocks(null, req.build()).getCorrupt()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert( + ipc(() -> rpcProxy.listCorruptFileBlocks(null, req.build())).getCorrupt()); } @Override public void metaSave(String filename) throws IOException { MetaSaveRequestProto req = MetaSaveRequestProto.newBuilder() .setFilename(filename).build(); - try { - rpcProxy.metaSave(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - + ipc(() -> rpcProxy.metaSave(null, req)); } @Override @@ -968,12 +814,8 @@ public HdfsFileStatus getFileInfo(String src) throws IOException { GetFileInfoRequestProto req = GetFileInfoRequestProto.newBuilder() .setSrc(src) .build(); - try { - GetFileInfoResponseProto res = rpcProxy.getFileInfo(null, req); - return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetFileInfoResponseProto res = ipc(() -> rpcProxy.getFileInfo(null, req)); + return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null; } @Override @@ -984,27 +826,19 @@ public HdfsLocatedFileStatus getLocatedFileInfo(String src, .setSrc(src) .setNeedBlockToken(needBlockToken) .build(); - try { - GetLocatedFileInfoResponseProto res = - rpcProxy.getLocatedFileInfo(null, req); - return (HdfsLocatedFileStatus) (res.hasFs() - ? PBHelperClient.convert(res.getFs()) - : null); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetLocatedFileInfoResponseProto res = + ipc(() -> rpcProxy.getLocatedFileInfo(null, req)); + return (HdfsLocatedFileStatus) (res.hasFs() + ? PBHelperClient.convert(res.getFs()) + : null); } @Override public HdfsFileStatus getFileLinkInfo(String src) throws IOException { GetFileLinkInfoRequestProto req = GetFileLinkInfoRequestProto.newBuilder() .setSrc(src).build(); - try { - GetFileLinkInfoResponseProto result = rpcProxy.getFileLinkInfo(null, req); - return result.hasFs() ? PBHelperClient.convert(result.getFs()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetFileLinkInfoResponseProto result = ipc(() -> rpcProxy.getFileLinkInfo(null, req)); + return result.hasFs() ? PBHelperClient.convert(result.getFs()) : null; } @Override @@ -1013,12 +847,8 @@ public ContentSummary getContentSummary(String path) throws IOException { .newBuilder() .setPath(path) .build(); - try { - return PBHelperClient.convert(rpcProxy.getContentSummary(null, req) - .getSummary()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getContentSummary(null, req)) + .getSummary()); } @Override @@ -1033,11 +863,7 @@ public void setQuota(String path, long namespaceQuota, long storagespaceQuota, builder.setStorageType(PBHelperClient.convertStorageType(type)); } final SetQuotaRequestProto req = builder.build(); - try { - rpcProxy.setQuota(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.setQuota(null, req)); } @Override @@ -1046,11 +872,7 @@ public void fsync(String src, long fileId, String client, FsyncRequestProto req = FsyncRequestProto.newBuilder().setSrc(src) .setClient(client).setLastBlockLength(lastBlockLength) .setFileId(fileId).build(); - try { - rpcProxy.fsync(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.fsync(null, req)); } @Override @@ -1060,11 +882,7 @@ public void setTimes(String src, long mtime, long atime) throws IOException { .setMtime(mtime) .setAtime(atime) .build(); - try { - rpcProxy.setTimes(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.setTimes(null, req)); } @Override @@ -1076,23 +894,15 @@ public void createSymlink(String target, String link, FsPermission dirPerm, .setDirPerm(PBHelperClient.convert(dirPerm)) .setCreateParent(createParent) .build(); - try { - rpcProxy.createSymlink(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.createSymlink(null, req)); } @Override public String getLinkTarget(String path) throws IOException { GetLinkTargetRequestProto req = GetLinkTargetRequestProto.newBuilder() .setPath(path).build(); - try { - GetLinkTargetResponseProto rsp = rpcProxy.getLinkTarget(null, req); - return rsp.hasTargetPath() ? rsp.getTargetPath() : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetLinkTargetResponseProto rsp = ipc(() -> rpcProxy.getLinkTarget(null, req)); + return rsp.hasTargetPath() ? rsp.getTargetPath() : null; } @Override @@ -1103,12 +913,8 @@ public LocatedBlock updateBlockForPipeline(ExtendedBlock block, .setBlock(PBHelperClient.convert(block)) .setClientName(clientName) .build(); - try { - return PBHelperClient.convertLocatedBlockProto( - rpcProxy.updateBlockForPipeline(null, req).getBlock()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convertLocatedBlockProto( + ipc(() -> rpcProxy.updateBlockForPipeline(null, req)).getBlock()); } @Override @@ -1122,11 +928,7 @@ public void updatePipeline(String clientName, ExtendedBlock oldBlock, .addAllNewNodes(Arrays.asList(PBHelperClient.convert(newNodes))) .addAllStorageIDs(storageIDs == null ? null : Arrays.asList(storageIDs)) .build(); - try { - rpcProxy.updatePipeline(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.updatePipeline(null, req)); } @Override @@ -1136,14 +938,10 @@ public Token getDelegationToken(Text renewer) .newBuilder() .setRenewer(renewer == null ? "" : renewer.toString()) .build(); - try { - GetDelegationTokenResponseProto resp = - rpcProxy.getDelegationToken(null, req); - return resp.hasToken() ? - PBHelperClient.convertDelegationToken(resp.getToken()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetDelegationTokenResponseProto resp = + ipc(() -> rpcProxy.getDelegationToken(null, req)); + return resp.hasToken() ? + PBHelperClient.convertDelegationToken(resp.getToken()) : null; } @Override @@ -1153,11 +951,7 @@ public long renewDelegationToken(Token token) RenewDelegationTokenRequestProto.newBuilder(). setToken(PBHelperClient.convert(token)). build(); - try { - return rpcProxy.renewDelegationToken(null, req).getNewExpiryTime(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.renewDelegationToken(null, req)).getNewExpiryTime(); } @Override @@ -1167,11 +961,7 @@ public void cancelDelegationToken(Token token) .newBuilder() .setToken(PBHelperClient.convert(token)) .build(); - try { - rpcProxy.cancelDelegationToken(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.cancelDelegationToken(null, req)); } @Override @@ -1180,11 +970,7 @@ public void setBalancerBandwidth(long bandwidth) throws IOException { SetBalancerBandwidthRequestProto.newBuilder() .setBandwidth(bandwidth) .build(); - try { - rpcProxy.setBalancerBandwidth(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.setBalancerBandwidth(null, req)); } @Override @@ -1196,14 +982,10 @@ public boolean isMethodSupported(String methodName) throws IOException { @Override public DataEncryptionKey getDataEncryptionKey() throws IOException { - try { - GetDataEncryptionKeyResponseProto rsp = rpcProxy.getDataEncryptionKey( - null, VOID_GET_DATA_ENCRYPTIONKEY_REQUEST); - return rsp.hasDataEncryptionKey() ? - PBHelperClient.convert(rsp.getDataEncryptionKey()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetDataEncryptionKeyResponseProto rsp = ipc(() -> rpcProxy.getDataEncryptionKey( + null, VOID_GET_DATA_ENCRYPTIONKEY_REQUEST)); + return rsp.hasDataEncryptionKey() ? + PBHelperClient.convert(rsp.getDataEncryptionKey()) : null; } @@ -1211,11 +993,7 @@ public DataEncryptionKey getDataEncryptionKey() throws IOException { public boolean isFileClosed(String src) throws IOException { IsFileClosedRequestProto req = IsFileClosedRequestProto.newBuilder() .setSrc(src).build(); - try { - return rpcProxy.isFileClosed(null, req).getResult(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.isFileClosed(null, req)).getResult(); } @Override @@ -1232,11 +1010,7 @@ public String createSnapshot(String snapshotRoot, String snapshotName) builder.setSnapshotName(snapshotName); } final CreateSnapshotRequestProto req = builder.build(); - try { - return rpcProxy.createSnapshot(null, req).getSnapshotPath(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.createSnapshot(null, req)).getSnapshotPath(); } @Override @@ -1244,33 +1018,21 @@ public void deleteSnapshot(String snapshotRoot, String snapshotName) throws IOException { DeleteSnapshotRequestProto req = DeleteSnapshotRequestProto.newBuilder() .setSnapshotRoot(snapshotRoot).setSnapshotName(snapshotName).build(); - try { - rpcProxy.deleteSnapshot(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.deleteSnapshot(null, req)); } @Override public void allowSnapshot(String snapshotRoot) throws IOException { AllowSnapshotRequestProto req = AllowSnapshotRequestProto.newBuilder() .setSnapshotRoot(snapshotRoot).build(); - try { - rpcProxy.allowSnapshot(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.allowSnapshot(null, req)); } @Override public void disallowSnapshot(String snapshotRoot) throws IOException { DisallowSnapshotRequestProto req = DisallowSnapshotRequestProto .newBuilder().setSnapshotRoot(snapshotRoot).build(); - try { - rpcProxy.disallowSnapshot(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.disallowSnapshot(null, req)); } @Override @@ -1279,11 +1041,7 @@ public void renameSnapshot(String snapshotRoot, String snapshotOldName, RenameSnapshotRequestProto req = RenameSnapshotRequestProto.newBuilder() .setSnapshotRoot(snapshotRoot).setSnapshotOldName(snapshotOldName) .setSnapshotNewName(snapshotNewName).build(); - try { - rpcProxy.renameSnapshot(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.renameSnapshot(null, req)); } @Override @@ -1291,17 +1049,13 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() throws IOException { GetSnapshottableDirListingRequestProto req = GetSnapshottableDirListingRequestProto.newBuilder().build(); - try { - GetSnapshottableDirListingResponseProto result = rpcProxy - .getSnapshottableDirListing(null, req); + GetSnapshottableDirListingResponseProto result = ipc(() -> rpcProxy + .getSnapshottableDirListing(null, req)); - if (result.hasSnapshottableDirList()) { - return PBHelperClient.convert(result.getSnapshottableDirList()); - } - return null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (result.hasSnapshottableDirList()) { + return PBHelperClient.convert(result.getSnapshottableDirList()); } + return null; } @Override @@ -1310,17 +1064,13 @@ public SnapshotStatus[] getSnapshotListing(String path) GetSnapshotListingRequestProto req = GetSnapshotListingRequestProto.newBuilder() .setSnapshotRoot(path).build(); - try { - GetSnapshotListingResponseProto result = rpcProxy - .getSnapshotListing(null, req); + GetSnapshotListingResponseProto result = ipc(() -> rpcProxy + .getSnapshotListing(null, req)); - if (result.hasSnapshotList()) { - return PBHelperClient.convert(result.getSnapshotList()); - } - return null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (result.hasSnapshotList()) { + return PBHelperClient.convert(result.getSnapshotList()); } + return null; } @Override @@ -1329,14 +1079,10 @@ public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot, GetSnapshotDiffReportRequestProto req = GetSnapshotDiffReportRequestProto .newBuilder().setSnapshotRoot(snapshotRoot) .setFromSnapshot(fromSnapshot).setToSnapshot(toSnapshot).build(); - try { - GetSnapshotDiffReportResponseProto result = - rpcProxy.getSnapshotDiffReport(null, req); + GetSnapshotDiffReportResponseProto result = + ipc(() -> rpcProxy.getSnapshotDiffReport(null, req)); - return PBHelperClient.convert(result.getDiffReport()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(result.getDiffReport()); } @Override @@ -1350,58 +1096,42 @@ public SnapshotDiffReportListing getSnapshotDiffReportListing( HdfsProtos.SnapshotDiffReportCursorProto.newBuilder() .setStartPath(PBHelperClient.getByteString(startPath)) .setIndex(index).build()).build(); - try { - GetSnapshotDiffReportListingResponseProto result = - rpcProxy.getSnapshotDiffReportListing(null, req); + GetSnapshotDiffReportListingResponseProto result = + ipc(() -> rpcProxy.getSnapshotDiffReportListing(null, req)); - return PBHelperClient.convert(result.getDiffReport()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(result.getDiffReport()); } @Override public long addCacheDirective(CacheDirectiveInfo directive, EnumSet flags) throws IOException { - try { - AddCacheDirectiveRequestProto.Builder builder = - AddCacheDirectiveRequestProto.newBuilder(). - setInfo(PBHelperClient.convert(directive)); - if (!flags.isEmpty()) { - builder.setCacheFlags(PBHelperClient.convertCacheFlags(flags)); - } - return rpcProxy.addCacheDirective(null, builder.build()).getId(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + AddCacheDirectiveRequestProto.Builder builder = + AddCacheDirectiveRequestProto.newBuilder(). + setInfo(PBHelperClient.convert(directive)); + if (!flags.isEmpty()) { + builder.setCacheFlags(PBHelperClient.convertCacheFlags(flags)); } + return ipc(() -> rpcProxy.addCacheDirective(null, builder.build())).getId(); } @Override public void modifyCacheDirective(CacheDirectiveInfo directive, EnumSet flags) throws IOException { - try { - ModifyCacheDirectiveRequestProto.Builder builder = - ModifyCacheDirectiveRequestProto.newBuilder(). - setInfo(PBHelperClient.convert(directive)); - if (!flags.isEmpty()) { - builder.setCacheFlags(PBHelperClient.convertCacheFlags(flags)); - } - rpcProxy.modifyCacheDirective(null, builder.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + ModifyCacheDirectiveRequestProto.Builder builder = + ModifyCacheDirectiveRequestProto.newBuilder(). + setInfo(PBHelperClient.convert(directive)); + if (!flags.isEmpty()) { + builder.setCacheFlags(PBHelperClient.convertCacheFlags(flags)); } + ipc(() -> rpcProxy.modifyCacheDirective(null, builder.build())); } @Override public void removeCacheDirective(long id) throws IOException { - try { - rpcProxy.removeCacheDirective(null, - RemoveCacheDirectiveRequestProto.newBuilder(). - setId(id).build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeCacheDirective(null, + RemoveCacheDirectiveRequestProto.newBuilder(). + setId(id).build())); } private static class BatchedCacheEntries @@ -1435,16 +1165,13 @@ public BatchedEntries listCacheDirectives(long prevId, if (filter == null) { filter = new CacheDirectiveInfo.Builder().build(); } - try { - return new BatchedCacheEntries( - rpcProxy.listCacheDirectives(null, - ListCacheDirectivesRequestProto.newBuilder(). - setPrevId(prevId). - setFilter(PBHelperClient.convert(filter)). - build())); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + CacheDirectiveInfo f = filter; + return new BatchedCacheEntries( + ipc(() -> rpcProxy.listCacheDirectives(null, + ListCacheDirectivesRequestProto.newBuilder(). + setPrevId(prevId). + setFilter(PBHelperClient.convert(f)). + build()))); } @Override @@ -1452,11 +1179,7 @@ public void addCachePool(CachePoolInfo info) throws IOException { AddCachePoolRequestProto.Builder builder = AddCachePoolRequestProto.newBuilder(); builder.setInfo(PBHelperClient.convert(info)); - try { - rpcProxy.addCachePool(null, builder.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.addCachePool(null, builder.build())); } @Override @@ -1464,22 +1187,14 @@ public void modifyCachePool(CachePoolInfo req) throws IOException { ModifyCachePoolRequestProto.Builder builder = ModifyCachePoolRequestProto.newBuilder(); builder.setInfo(PBHelperClient.convert(req)); - try { - rpcProxy.modifyCachePool(null, builder.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.modifyCachePool(null, builder.build())); } @Override public void removeCachePool(String cachePoolName) throws IOException { - try { - rpcProxy.removeCachePool(null, - RemoveCachePoolRequestProto.newBuilder(). - setPoolName(cachePoolName).build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeCachePool(null, + RemoveCachePoolRequestProto.newBuilder(). + setPoolName(cachePoolName).build())); } private static class BatchedCachePoolEntries @@ -1510,14 +1225,10 @@ public boolean hasMore() { @Override public BatchedEntries listCachePools(String prevKey) throws IOException { - try { - return new BatchedCachePoolEntries( - rpcProxy.listCachePools(null, - ListCachePoolsRequestProto.newBuilder(). - setPrevPoolName(prevKey).build())); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return new BatchedCachePoolEntries( + ipc(() -> rpcProxy.listCachePools(null, + ListCachePoolsRequestProto.newBuilder(). + setPrevPoolName(prevKey).build()))); } @Override @@ -1526,11 +1237,7 @@ public void modifyAclEntries(String src, List aclSpec) ModifyAclEntriesRequestProto req = ModifyAclEntriesRequestProto .newBuilder().setSrc(src) .addAllAclSpec(PBHelperClient.convertAclEntryProto(aclSpec)).build(); - try { - rpcProxy.modifyAclEntries(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.modifyAclEntries(null, req)); } @Override @@ -1539,33 +1246,21 @@ public void removeAclEntries(String src, List aclSpec) RemoveAclEntriesRequestProto req = RemoveAclEntriesRequestProto .newBuilder().setSrc(src) .addAllAclSpec(PBHelperClient.convertAclEntryProto(aclSpec)).build(); - try { - rpcProxy.removeAclEntries(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeAclEntries(null, req)); } @Override public void removeDefaultAcl(String src) throws IOException { RemoveDefaultAclRequestProto req = RemoveDefaultAclRequestProto .newBuilder().setSrc(src).build(); - try { - rpcProxy.removeDefaultAcl(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeDefaultAcl(null, req)); } @Override public void removeAcl(String src) throws IOException { RemoveAclRequestProto req = RemoveAclRequestProto.newBuilder() .setSrc(src).build(); - try { - rpcProxy.removeAcl(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeAcl(null, req)); } @Override @@ -1574,15 +1269,11 @@ public void setAcl(String src, List aclSpec) throws IOException { .setSrc(src) .addAllAclSpec(PBHelperClient.convertAclEntryProto(aclSpec)) .build(); - try { - if (Client.isAsynchronousMode()) { - rpcProxy.setAcl(null, req); - setAsyncReturnValue(); - } else { - rpcProxy.setAcl(null, req); - } - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (Client.isAsynchronousMode()) { + ipc(() -> rpcProxy.setAcl(null, req)); + setAsyncReturnValue(); + } else { + ipc(() -> rpcProxy.setAcl(null, req)); } } @@ -1614,7 +1305,7 @@ public boolean isDone() { return PBHelperClient.convert(rpcProxy.getAclStatus(null, req)); } } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + throw getRemoteException(e); } } @@ -1628,11 +1319,7 @@ public void createEncryptionZone(String src, String keyName) builder.setKeyName(keyName); } CreateEncryptionZoneRequestProto req = builder.build(); - try { - rpcProxy.createEncryptionZone(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.createEncryptionZone(null, req)); } @Override @@ -1641,16 +1328,12 @@ public EncryptionZone getEZForPath(String src) throws IOException { GetEZForPathRequestProto.newBuilder(); builder.setSrc(src); final GetEZForPathRequestProto req = builder.build(); - try { - final EncryptionZonesProtos.GetEZForPathResponseProto response = - rpcProxy.getEZForPath(null, req); - if (response.hasZone()) { - return PBHelperClient.convert(response.getZone()); - } else { - return null; - } - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + final EncryptionZonesProtos.GetEZForPathResponseProto response = + ipc(() -> rpcProxy.getEZForPath(null, req)); + if (response.hasZone()) { + return PBHelperClient.convert(response.getZone()); + } else { + return null; } } @@ -1661,18 +1344,14 @@ public BatchedEntries listEncryptionZones(long id) ListEncryptionZonesRequestProto.newBuilder() .setId(id) .build(); - try { - EncryptionZonesProtos.ListEncryptionZonesResponseProto response = - rpcProxy.listEncryptionZones(null, req); - List elements = - Lists.newArrayListWithCapacity(response.getZonesCount()); - for (EncryptionZoneProto p : response.getZonesList()) { - elements.add(PBHelperClient.convert(p)); - } - return new BatchedListEntries<>(elements, response.getHasMore()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + EncryptionZonesProtos.ListEncryptionZonesResponseProto response = + ipc(() -> rpcProxy.listEncryptionZones(null, req)); + List elements = + Lists.newArrayListWithCapacity(response.getZonesCount()); + for (EncryptionZoneProto p : response.getZonesList()) { + elements.add(PBHelperClient.convert(p)); } + return new BatchedListEntries<>(elements, response.getHasMore()); } @Override @@ -1685,11 +1364,7 @@ public void setErasureCodingPolicy(String src, String ecPolicyName) builder.setEcPolicyName(ecPolicyName); } SetErasureCodingPolicyRequestProto req = builder.build(); - try { - rpcProxy.setErasureCodingPolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.setErasureCodingPolicy(null, req)); } @Override @@ -1698,11 +1373,7 @@ public void unsetErasureCodingPolicy(String src) throws IOException { UnsetErasureCodingPolicyRequestProto.newBuilder(); builder.setSrc(src); UnsetErasureCodingPolicyRequestProto req = builder.build(); - try { - rpcProxy.unsetErasureCodingPolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.unsetErasureCodingPolicy(null, req)); } @Override @@ -1712,14 +1383,10 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies( GetECTopologyResultForPoliciesRequestProto.newBuilder(); builder.addAllPolicies(Arrays.asList(policyNames)); GetECTopologyResultForPoliciesRequestProto req = builder.build(); - try { - GetECTopologyResultForPoliciesResponseProto response = - rpcProxy.getECTopologyResultForPolicies(null, req); - return PBHelperClient - .convertECTopologyVerifierResultProto(response.getResponse()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetECTopologyResultForPoliciesResponseProto response = + ipc(() -> rpcProxy.getECTopologyResultForPolicies(null, req)); + return PBHelperClient + .convertECTopologyVerifierResultProto(response.getResponse()); } @Override @@ -1729,11 +1396,7 @@ public void reencryptEncryptionZone(String zone, ReencryptAction action) ReencryptEncryptionZoneRequestProto.newBuilder(); builder.setZone(zone).setAction(PBHelperClient.convert(action)); ReencryptEncryptionZoneRequestProto req = builder.build(); - try { - rpcProxy.reencryptEncryptionZone(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.reencryptEncryptionZone(null, req)); } @Override @@ -1741,18 +1404,14 @@ public BatchedEntries listReencryptionStatus(long id) throws IOException { final ListReencryptionStatusRequestProto req = ListReencryptionStatusRequestProto.newBuilder().setId(id).build(); - try { - ListReencryptionStatusResponseProto response = - rpcProxy.listReencryptionStatus(null, req); - List elements = - Lists.newArrayListWithCapacity(response.getStatusesCount()); - for (ZoneReencryptionStatusProto p : response.getStatusesList()) { - elements.add(PBHelperClient.convert(p)); - } - return new BatchedListEntries<>(elements, response.getHasMore()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + ListReencryptionStatusResponseProto response = + ipc(() -> rpcProxy.listReencryptionStatus(null, req)); + List elements = + Lists.newArrayListWithCapacity(response.getStatusesCount()); + for (ZoneReencryptionStatusProto p : response.getStatusesList()) { + elements.add(PBHelperClient.convert(p)); } + return new BatchedListEntries<>(elements, response.getHasMore()); } @Override @@ -1763,11 +1422,7 @@ public void setXAttr(String src, XAttr xAttr, EnumSet flag) .setXAttr(PBHelperClient.convertXAttrProto(xAttr)) .setFlag(PBHelperClient.convert(flag)) .build(); - try { - rpcProxy.setXAttr(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.setXAttr(null, req)); } @Override @@ -1779,11 +1434,7 @@ public List getXAttrs(String src, List xAttrs) builder.addAllXAttrs(PBHelperClient.convertXAttrProto(xAttrs)); } GetXAttrsRequestProto req = builder.build(); - try { - return PBHelperClient.convert(rpcProxy.getXAttrs(null, req)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getXAttrs(null, req))); } @Override @@ -1792,11 +1443,7 @@ public List listXAttrs(String src) throws IOException { ListXAttrsRequestProto.newBuilder(); builder.setSrc(src); ListXAttrsRequestProto req = builder.build(); - try { - return PBHelperClient.convert(rpcProxy.listXAttrs(null, req)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.listXAttrs(null, req))); } @Override @@ -1804,22 +1451,14 @@ public void removeXAttr(String src, XAttr xAttr) throws IOException { RemoveXAttrRequestProto req = RemoveXAttrRequestProto .newBuilder().setSrc(src) .setXAttr(PBHelperClient.convertXAttrProto(xAttr)).build(); - try { - rpcProxy.removeXAttr(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeXAttr(null, req)); } @Override public void checkAccess(String path, FsAction mode) throws IOException { CheckAccessRequestProto req = CheckAccessRequestProto.newBuilder() .setPath(path).setMode(PBHelperClient.convert(mode)).build(); - try { - rpcProxy.checkAccess(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.checkAccess(null, req)); } @Override @@ -1827,66 +1466,42 @@ public void setStoragePolicy(String src, String policyName) throws IOException { SetStoragePolicyRequestProto req = SetStoragePolicyRequestProto .newBuilder().setSrc(src).setPolicyName(policyName).build(); - try { - rpcProxy.setStoragePolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.setStoragePolicy(null, req)); } @Override public void unsetStoragePolicy(String src) throws IOException { UnsetStoragePolicyRequestProto req = UnsetStoragePolicyRequestProto .newBuilder().setSrc(src).build(); - try { - rpcProxy.unsetStoragePolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.unsetStoragePolicy(null, req)); } @Override public BlockStoragePolicy getStoragePolicy(String path) throws IOException { GetStoragePolicyRequestProto request = GetStoragePolicyRequestProto .newBuilder().setPath(path).build(); - try { - return PBHelperClient.convert(rpcProxy.getStoragePolicy(null, request) - .getStoragePolicy()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getStoragePolicy(null, request)) + .getStoragePolicy()); } @Override public BlockStoragePolicy[] getStoragePolicies() throws IOException { - try { - GetStoragePoliciesResponseProto response = rpcProxy - .getStoragePolicies(null, VOID_GET_STORAGE_POLICIES_REQUEST); - return PBHelperClient.convertStoragePolicies(response.getPoliciesList()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetStoragePoliciesResponseProto response = ipc(() -> rpcProxy + .getStoragePolicies(null, VOID_GET_STORAGE_POLICIES_REQUEST)); + return PBHelperClient.convertStoragePolicies(response.getPoliciesList()); } public long getCurrentEditLogTxid() throws IOException { GetCurrentEditLogTxidRequestProto req = GetCurrentEditLogTxidRequestProto .getDefaultInstance(); - try { - return rpcProxy.getCurrentEditLogTxid(null, req).getTxid(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.getCurrentEditLogTxid(null, req)).getTxid(); } @Override public EventBatchList getEditsFromTxid(long txid) throws IOException { GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder() .setTxid(txid).build(); - try { - return PBHelperClient.convert(rpcProxy.getEditsFromTxid(null, req)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getEditsFromTxid(null, req))); } @Override @@ -1898,17 +1513,13 @@ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( AddErasureCodingPoliciesRequestProto req = AddErasureCodingPoliciesRequestProto.newBuilder() .addAllEcPolicies(protos).build(); - try { - AddErasureCodingPoliciesResponseProto rep = rpcProxy - .addErasureCodingPolicies(null, req); - AddErasureCodingPolicyResponse[] responses = - rep.getResponsesList().stream() - .map(PBHelperClient::convertAddErasureCodingPolicyResponse) - .toArray(AddErasureCodingPolicyResponse[]::new); - return responses; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + AddErasureCodingPoliciesResponseProto rep = ipc(() -> rpcProxy + .addErasureCodingPolicies(null, req)); + AddErasureCodingPolicyResponse[] responses = + rep.getResponsesList().stream() + .map(PBHelperClient::convertAddErasureCodingPolicyResponse) + .toArray(AddErasureCodingPolicyResponse[]::new); + return responses; } @Override @@ -1918,11 +1529,7 @@ public void removeErasureCodingPolicy(String ecPolicyName) RemoveErasureCodingPolicyRequestProto.newBuilder(); builder.setEcPolicyName(ecPolicyName); RemoveErasureCodingPolicyRequestProto req = builder.build(); - try { - rpcProxy.removeErasureCodingPolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.removeErasureCodingPolicy(null, req)); } @Override @@ -1932,11 +1539,7 @@ public void enableErasureCodingPolicy(String ecPolicyName) EnableErasureCodingPolicyRequestProto.newBuilder(); builder.setEcPolicyName(ecPolicyName); EnableErasureCodingPolicyRequestProto req = builder.build(); - try { - rpcProxy.enableErasureCodingPolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.enableErasureCodingPolicy(null, req)); } @Override @@ -1946,45 +1549,33 @@ public void disableErasureCodingPolicy(String ecPolicyName) DisableErasureCodingPolicyRequestProto.newBuilder(); builder.setEcPolicyName(ecPolicyName); DisableErasureCodingPolicyRequestProto req = builder.build(); - try { - rpcProxy.disableErasureCodingPolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.disableErasureCodingPolicy(null, req)); } @Override public ErasureCodingPolicyInfo[] getErasureCodingPolicies() throws IOException { - try { - GetErasureCodingPoliciesResponseProto response = rpcProxy - .getErasureCodingPolicies(null, VOID_GET_EC_POLICIES_REQUEST); - ErasureCodingPolicyInfo[] ecPolicies = - new ErasureCodingPolicyInfo[response.getEcPoliciesCount()]; - int i = 0; - for (ErasureCodingPolicyProto proto : response.getEcPoliciesList()) { - ecPolicies[i++] = - PBHelperClient.convertErasureCodingPolicyInfo(proto); - } - return ecPolicies; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetErasureCodingPoliciesResponseProto response = ipc(() -> rpcProxy + .getErasureCodingPolicies(null, VOID_GET_EC_POLICIES_REQUEST)); + ErasureCodingPolicyInfo[] ecPolicies = + new ErasureCodingPolicyInfo[response.getEcPoliciesCount()]; + int i = 0; + for (ErasureCodingPolicyProto proto : response.getEcPoliciesList()) { + ecPolicies[i++] = + PBHelperClient.convertErasureCodingPolicyInfo(proto); } + return ecPolicies; } @Override public Map getErasureCodingCodecs() throws IOException { - try { - GetErasureCodingCodecsResponseProto response = rpcProxy - .getErasureCodingCodecs(null, VOID_GET_EC_CODEC_REQUEST); - Map ecCodecs = new HashMap<>(); - for (CodecProto codec : response.getCodecList()) { - ecCodecs.put(codec.getCodec(), codec.getCoders()); - } - return ecCodecs; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetErasureCodingCodecsResponseProto response = ipc(() -> rpcProxy + .getErasureCodingCodecs(null, VOID_GET_EC_CODEC_REQUEST)); + Map ecCodecs = new HashMap<>(); + for (CodecProto codec : response.getCodecList()) { + ecCodecs.put(codec.getCodec(), codec.getCoders()); } + return ecCodecs; } @Override @@ -1992,29 +1583,21 @@ public ErasureCodingPolicy getErasureCodingPolicy(String src) throws IOException { GetErasureCodingPolicyRequestProto req = GetErasureCodingPolicyRequestProto.newBuilder().setSrc(src).build(); - try { - GetErasureCodingPolicyResponseProto response = - rpcProxy.getErasureCodingPolicy(null, req); - if (response.hasEcPolicy()) { - return PBHelperClient.convertErasureCodingPolicy( - response.getEcPolicy()); - } - return null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetErasureCodingPolicyResponseProto response = + ipc(() -> rpcProxy.getErasureCodingPolicy(null, req)); + if (response.hasEcPolicy()) { + return PBHelperClient.convertErasureCodingPolicy( + response.getEcPolicy()); } + return null; } @Override public QuotaUsage getQuotaUsage(String path) throws IOException { GetQuotaUsageRequestProto req = GetQuotaUsageRequestProto.newBuilder().setPath(path).build(); - try { - return PBHelperClient.convert(rpcProxy.getQuotaUsage(null, req) - .getUsage()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert(ipc(() -> rpcProxy.getQuotaUsage(null, req)) + .getUsage()); } @Deprecated @@ -2035,51 +1618,35 @@ public BatchedEntries listOpenFiles(long prevId, } req.setPath(path); - try { - ListOpenFilesResponseProto response = - rpcProxy.listOpenFiles(null, req.build()); - List openFileEntries = - Lists.newArrayListWithCapacity(response.getEntriesCount()); - for (OpenFilesBatchResponseProto p : response.getEntriesList()) { - openFileEntries.add(PBHelperClient.convert(p)); - } - return new BatchedListEntries<>(openFileEntries, response.getHasMore()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + ListOpenFilesResponseProto response = + ipc(() -> rpcProxy.listOpenFiles(null, req.build())); + List openFileEntries = + Lists.newArrayListWithCapacity(response.getEntriesCount()); + for (OpenFilesBatchResponseProto p : response.getEntriesList()) { + openFileEntries.add(PBHelperClient.convert(p)); } + return new BatchedListEntries<>(openFileEntries, response.getHasMore()); } @Override public void msync() throws IOException { MsyncRequestProto.Builder req = MsyncRequestProto.newBuilder(); - try { - rpcProxy.msync(null, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.msync(null, req.build())); } @Override public void satisfyStoragePolicy(String src) throws IOException { SatisfyStoragePolicyRequestProto req = SatisfyStoragePolicyRequestProto.newBuilder().setSrc(src).build(); - try { - rpcProxy.satisfyStoragePolicy(null, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.satisfyStoragePolicy(null, req)); } @Override public DatanodeInfo[] getSlowDatanodeReport() throws IOException { GetSlowDatanodeReportRequestProto req = GetSlowDatanodeReportRequestProto.newBuilder().build(); - try { - return PBHelperClient.convert( - rpcProxy.getSlowDatanodeReport(null, req).getDatanodeInfoProtoList()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelperClient.convert( + ipc(() -> rpcProxy.getSlowDatanodeReport(null, req)).getDatanodeInfoProtoList()); } @Override @@ -2087,22 +1654,18 @@ public HAServiceProtocol.HAServiceState getHAServiceState() throws IOException { HAServiceStateRequestProto req = HAServiceStateRequestProto.newBuilder().build(); - try { - HAServiceStateProto res = - rpcProxy.getHAServiceState(null, req).getState(); - switch(res) { - case ACTIVE: - return HAServiceProtocol.HAServiceState.ACTIVE; - case STANDBY: - return HAServiceProtocol.HAServiceState.STANDBY; - case OBSERVER: - return HAServiceProtocol.HAServiceState.OBSERVER; - case INITIALIZING: - default: - return HAServiceProtocol.HAServiceState.INITIALIZING; - } - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + HAServiceStateProto res = + ipc(() -> rpcProxy.getHAServiceState(null, req)).getState(); + switch(res) { + case ACTIVE: + return HAServiceProtocol.HAServiceState.ACTIVE; + case STANDBY: + return HAServiceProtocol.HAServiceState.STANDBY; + case OBSERVER: + return HAServiceProtocol.HAServiceState.OBSERVER; + case INITIALIZING: + default: + return HAServiceProtocol.HAServiceState.INITIALIZING; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 496a5cf46146d..26ee5de2886e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -209,7 +209,7 @@ import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.erasurecode.ECSchema; -import org.apache.hadoop.ipc.ProtobufHelper; +import org.apache.hadoop.ipc.internal.ShadedProtobufHelper; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ChunkedArrayList; @@ -237,7 +237,7 @@ public class PBHelperClient { FsAction.values(); private static ByteString getFixedByteString(String key) { - return ProtobufHelper.getFixedByteString(key); + return ShadedProtobufHelper.getFixedByteString(key); } /** @@ -260,7 +260,8 @@ private PBHelperClient() { public static ByteString getByteString(byte[] bytes) { // return singleton to reduce object allocation - return ProtobufHelper.getByteString(bytes); + // return singleton to reduce object allocation + return ShadedProtobufHelper.getByteString(bytes); } public static ShmId convert(ShortCircuitShmIdProto shmId) { @@ -328,7 +329,7 @@ public static ExtendedBlockProto convert(final ExtendedBlock b) { } public static TokenProto convert(Token tok) { - return ProtobufHelper.protoFromToken(tok); + return ShadedProtobufHelper.protoFromToken(tok); } public static ShortCircuitShmIdProto convert(ShmId shmId) { @@ -814,8 +815,8 @@ public static StorageType[] convertStorageTypes( public static Token convert( TokenProto blockToken) { - return (Token) ProtobufHelper - .tokenFromProto(blockToken); + return (Token) ShadedProtobufHelper.tokenFromProto( + blockToken); } // DatanodeId diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolTranslatorPB.java index ce8a89b84acce..eadee121e975b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ReconfigurationProtocolTranslatorPB.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ListReconfigurablePropertiesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ListReconfigurablePropertiesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.StartReconfigurationRequestProto; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.ProtocolTranslator; @@ -44,7 +43,8 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -102,37 +102,25 @@ public Object getUnderlyingProxyObject() { @Override public void startReconfiguration() throws IOException { - try { - rpcProxy.startReconfiguration(NULL_CONTROLLER, VOID_START_RECONFIG); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.startReconfiguration(NULL_CONTROLLER, VOID_START_RECONFIG)); } @Override public ReconfigurationTaskStatus getReconfigurationStatus() throws IOException { - try { - return ReconfigurationProtocolUtils.getReconfigurationStatus( - rpcProxy - .getReconfigurationStatus( - NULL_CONTROLLER, - VOID_GET_RECONFIG_STATUS)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ReconfigurationProtocolUtils.getReconfigurationStatus( + ipc(() -> rpcProxy + .getReconfigurationStatus( + NULL_CONTROLLER, + VOID_GET_RECONFIG_STATUS))); } @Override public List listReconfigurableProperties() throws IOException { ListReconfigurablePropertiesResponseProto response; - try { - response = rpcProxy.listReconfigurableProperties(NULL_CONTROLLER, - VOID_LIST_RECONFIGURABLE_PROPERTIES); - return response.getNameList(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + response = ipc(() -> rpcProxy.listReconfigurableProperties(NULL_CONTROLLER, + VOID_LIST_RECONFIGURABLE_PROPERTIES)); + return response.getNameList(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index 39bc6683fcfd1..c234caf46e677 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -146,7 +146,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> com.google.protobuf protobuf-java - compile + ${transient.protobuf2.scope} javax.servlet diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java index db71c1cb70cf9..0291b511b7d38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java @@ -103,7 +103,6 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.ProtocolTranslator; import org.apache.hadoop.ipc.RPC; @@ -111,6 +110,8 @@ import org.apache.hadoop.thirdparty.protobuf.ServiceException; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.getRemoteException; + /** * This class forwards RouterAdminProtocol calls as RPC calls to the RouterAdmin server * while translating from the parameter types used in RouterAdminProtocol to the @@ -156,7 +157,8 @@ public AddMountTableEntryResponse addMountTableEntry( rpcProxy.addMountTableEntry(null, proto); return new AddMountTableEntryResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -169,7 +171,7 @@ public AddMountTableEntriesResponse addMountTableEntries(AddMountTableEntriesReq AddMountTableEntriesResponseProto response = rpcProxy.addMountTableEntries(null, proto); return new AddMountTableEntriesResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + throw new IOException(getRemoteException(e).getMessage()); } } @@ -184,7 +186,8 @@ public UpdateMountTableEntryResponse updateMountTableEntry( rpcProxy.updateMountTableEntry(null, proto); return new UpdateMountTableEntryResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -199,7 +202,8 @@ public RemoveMountTableEntryResponse removeMountTableEntry( rpcProxy.removeMountTableEntry(null, proto); return new RemoveMountTableEntryResponsePBImpl(responseProto); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -214,7 +218,8 @@ public GetMountTableEntriesResponse getMountTableEntries( rpcProxy.getMountTableEntries(null, proto); return new GetMountTableEntriesResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -228,7 +233,8 @@ public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request) rpcProxy.enterSafeMode(null, proto); return new EnterSafeModeResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -242,7 +248,8 @@ public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request) rpcProxy.leaveSafeMode(null, proto); return new LeaveSafeModeResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -256,7 +263,8 @@ public GetSafeModeResponse getSafeMode(GetSafeModeRequest request) rpcProxy.getSafeMode(null, proto); return new GetSafeModeResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -271,7 +279,8 @@ public DisableNameserviceResponse disableNameservice( rpcProxy.disableNameservice(null, proto); return new DisableNameserviceResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -286,7 +295,8 @@ public EnableNameserviceResponse enableNameservice( rpcProxy.enableNameservice(null, proto); return new EnableNameserviceResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -300,7 +310,8 @@ public GetDisabledNameservicesResponse getDisabledNameservices( rpcProxy.getDisabledNameservices(null, proto); return new GetDisabledNameservicesResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -315,7 +326,8 @@ public RefreshMountTableEntriesResponse refreshMountTableEntries( rpcProxy.refreshMountTableEntries(null, proto); return new RefreshMountTableEntriesResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -330,7 +342,8 @@ public GetDestinationResponse getDestination( rpcProxy.getDestination(null, proto); return new GetDestinationResponsePBImpl(response); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } @@ -344,7 +357,8 @@ public boolean refreshSuperUserGroupsConfiguration() throws IOException { return new RefreshSuperUserGroupsConfigurationResponsePBImpl(response) .getStatus(); } catch (ServiceException e) { - throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + + throw new IOException(getRemoteException(e).getMessage()); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index 21736e520fddf..2d96ab1be359b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -211,9 +211,9 @@ public RouterAdminServer(Configuration conf, Router router) RefreshCallQueueProtocolProtos.RefreshCallQueueProtocolService. newReflectiveBlockingService(refreshCallQueueXlator); - DFSUtil.addPBProtocol(conf, GenericRefreshProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, GenericRefreshProtocolPB.class, genericRefreshService, adminServer); - DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshCallQueueProtocolPB.class, refreshCallQueueService, adminServer); registerRefreshFairnessPolicyControllerHandler(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 1c3a28a9924d5..cae61b7d927dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -341,11 +341,11 @@ public RouterRpcServer(Configuration conf, Router router, .build(); // Add all the RPC protocols that the Router implements - DFSUtil.addPBProtocol( + DFSUtil.addInternalPBProtocol( conf, NamenodeProtocolPB.class, nnPbService, this.rpcServer); - DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshUserMappingsProtocolPB.class, refreshUserMappingService, this.rpcServer); - DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, GetUserMappingsProtocolPB.class, getUserMappingService, this.rpcServer); // Set service-level authorization security policy diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java index 27fcf8726b6c8..87461aaf49137 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java @@ -196,7 +196,7 @@ private void setupRPCServer(final Configuration conf) throws IOException { BlockingService nnProtoPbService = NamenodeProtocolService.newReflectiveBlockingService( nnProtoXlator); - DFSUtil.addPBProtocol( + DFSUtil.addInternalPBProtocol( conf, NamenodeProtocolPB.class, nnProtoPbService, rpcServer); DatanodeProtocolServerSideTranslatorPB dnProtoPbXlator = @@ -204,7 +204,7 @@ private void setupRPCServer(final Configuration conf) throws IOException { BlockingService dnProtoPbService = DatanodeProtocolService.newReflectiveBlockingService( dnProtoPbXlator); - DFSUtil.addPBProtocol( + DFSUtil.addInternalPBProtocol( conf, DatanodeProtocolPB.class, dnProtoPbService, rpcServer); HAServiceProtocolServerSideTranslatorPB haServiceProtoXlator = @@ -212,7 +212,7 @@ private void setupRPCServer(final Configuration conf) throws IOException { BlockingService haProtoPbService = HAServiceProtocolService.newReflectiveBlockingService( haServiceProtoXlator); - DFSUtil.addPBProtocol( + DFSUtil.addInternalPBProtocol( conf, HAServiceProtocolPB.class, haProtoPbService, rpcServer); this.rpcServer.addTerseExceptions( diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 9efea6b6d093a..3abff73e76f0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -130,7 +130,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> com.google.protobuf protobuf-java - compile + ${transient.protobuf2.scope} javax.servlet diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 3a3219447dfae..1b3e77e0fe465 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -67,6 +67,7 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; @@ -1361,7 +1362,30 @@ static URI trimUri(URI uri) { } /** - * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server} + * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server}. + * This method is for exclusive use by the hadoop libraries, as its signature + * changes with the version of the shaded protobuf library it has been built with. + * @param conf configuration + * @param protocol Protocol interface + * @param service service that implements the protocol + * @param server RPC server to which the protocol & implementation is + * added to + * @throws IOException failure + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static void addInternalPBProtocol(Configuration conf, + Class protocol, + BlockingService service, + RPC.Server server) throws IOException { + RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine2.class); + server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service); + } + + /** + * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server}. + * Deprecated as it will only reliably compile if an unshaded protobuf library + * is also on the classpath. * @param conf configuration * @param protocol Protocol interface * @param service service that implements the protocol @@ -1369,17 +1393,17 @@ static URI trimUri(URI uri) { * added to * @throws IOException */ + @Deprecated public static void addPBProtocol(Configuration conf, Class protocol, BlockingService service, RPC.Server server) throws IOException { - RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine2.class); - server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service); + addInternalPBProtocol(conf, protocol, service, server); } /** * Add protobuf based protocol to the {@link RPC.Server}. * This engine uses Protobuf 2.5.0. Recommended to upgrade to * Protobuf 3.x from hadoop-thirdparty and use - * {@link DFSUtil#addPBProtocol(Configuration, Class, BlockingService, + * {@link DFSUtil#addInternalPBProtocol(Configuration, Class, BlockingService, * RPC.Server)}. * @param conf configuration * @param protocol Protocol interface diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeLifelineProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeLifelineProtocolClientSideTranslatorPB.java index 220e9e2835625..cdeb8275418f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeLifelineProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeLifelineProtocolClientSideTranslatorPB.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; @@ -38,7 +37,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -96,11 +96,7 @@ public void sendLifeline(DatanodeRegistration registration, builder.setVolumeFailureSummary(PBHelper.convertVolumeFailureSummary( volumeFailureSummary)); } - try { - rpcProxy.sendLifeline(NULL_CONTROLLER, builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.sendLifeline(NULL_CONTROLLER, builder.build())); } @Override // ProtocolMetaInterface diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index fd58c0c7ca289..1bc32f85ea4d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -61,7 +61,6 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; @@ -71,10 +70,11 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; import javax.annotation.Nonnull; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; + /** * This class is the client side translator to translate the requests made on * {@link DatanodeProtocol} interfaces to the RPC server implementing @@ -123,11 +123,8 @@ public DatanodeRegistration registerDatanode(DatanodeRegistration registration RegisterDatanodeRequestProto.Builder builder = RegisterDatanodeRequestProto .newBuilder().setRegistration(PBHelper.convert(registration)); RegisterDatanodeResponseProto resp; - try { - resp = rpcProxy.registerDatanode(NULL_CONTROLLER, builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + resp = ipc(() -> rpcProxy.registerDatanode(NULL_CONTROLLER, builder.build())); + return PBHelper.convert(resp.getRegistration()); } @@ -164,11 +161,8 @@ public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration, } HeartbeatResponseProto resp; - try { - resp = rpcProxy.sendHeartbeat(NULL_CONTROLLER, builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + resp = ipc(() -> rpcProxy.sendHeartbeat(NULL_CONTROLLER, builder.build())); + DatanodeCommand[] cmds = new DatanodeCommand[resp.getCmdsList().size()]; int index = 0; for (DatanodeCommandProto p : resp.getCmdsList()) { @@ -215,11 +209,7 @@ public DatanodeCommand blockReport(DatanodeRegistration registration, } builder.setContext(PBHelper.convert(context)); BlockReportResponseProto resp; - try { - resp = rpcProxy.blockReport(NULL_CONTROLLER, builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + resp = ipc(() -> rpcProxy.blockReport(NULL_CONTROLLER, builder.build())); return resp.hasCmd() ? PBHelper.convert(resp.getCmd()) : null; } @@ -235,11 +225,7 @@ public DatanodeCommand cacheReport(DatanodeRegistration registration, } CacheReportResponseProto resp; - try { - resp = rpcProxy.cacheReport(NULL_CONTROLLER, builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + resp = ipc(() -> rpcProxy.cacheReport(NULL_CONTROLLER, builder.build())); if (resp.hasCmd()) { return PBHelper.convert(resp.getCmd()); } @@ -264,11 +250,7 @@ public void blockReceivedAndDeleted(DatanodeRegistration registration, } builder.addBlocks(repBuilder.build()); } - try { - rpcProxy.blockReceivedAndDeleted(NULL_CONTROLLER, builder.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.blockReceivedAndDeleted(NULL_CONTROLLER, builder.build())); } @Override @@ -277,21 +259,13 @@ public void errorReport(DatanodeRegistration registration, int errorCode, ErrorReportRequestProto req = ErrorReportRequestProto.newBuilder() .setRegistartion(PBHelper.convert(registration)) .setErrorCode(errorCode).setMsg(msg).build(); - try { - rpcProxy.errorReport(NULL_CONTROLLER, req); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.errorReport(NULL_CONTROLLER, req)); } @Override public NamespaceInfo versionRequest() throws IOException { - try { - return PBHelper.convert(rpcProxy.versionRequest(NULL_CONTROLLER, - VOID_VERSION_REQUEST).getInfo()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelper.convert(ipc(() -> rpcProxy.versionRequest(NULL_CONTROLLER, + VOID_VERSION_REQUEST).getInfo())); } @Override @@ -302,11 +276,7 @@ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException { builder.addBlocks(i, PBHelperClient.convertLocatedBlock(blocks[i])); } ReportBadBlocksRequestProto req = builder.build(); - try { - rpcProxy.reportBadBlocks(NULL_CONTROLLER, req); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.reportBadBlocks(NULL_CONTROLLER, req)); } @Override @@ -327,11 +297,7 @@ public void commitBlockSynchronization(ExtendedBlock block, builder.addNewTargetStorages(newtargetstorages[i]); } CommitBlockSynchronizationRequestProto req = builder.build(); - try { - rpcProxy.commitBlockSynchronization(NULL_CONTROLLER, req); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.commitBlockSynchronization(NULL_CONTROLLER, req)); } @Override // ProtocolMetaInterface diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InMemoryAliasMapProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InMemoryAliasMapProtocolClientSideTranslatorPB.java index f00cfd45a3bfe..9d29e09d0cd99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InMemoryAliasMapProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InMemoryAliasMapProtocolClientSideTranslatorPB.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.hdfs.protocolPB; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -30,7 +29,6 @@ import org.apache.hadoop.hdfs.server.common.FileRegion; import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.InMemoryAliasMapFailoverProxyProvider; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.slf4j.Logger; @@ -54,6 +52,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX; import static org.apache.hadoop.hdfs.protocol.proto.AliasMapProtocolProtos.*; import static org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.*; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate requests made to the @@ -136,29 +135,24 @@ public InMemoryAliasMap.IterationResult list(Optional marker) builder.setMarker(PBHelperClient.convert(marker.get())); } ListRequestProto request = builder.build(); - try { - ListResponseProto response = rpcProxy.list(null, request); - List fileRegionsList = response.getFileRegionsList(); - - List fileRegions = fileRegionsList - .stream() - .map(kv -> new FileRegion( - PBHelperClient.convert(kv.getKey()), - PBHelperClient.convert(kv.getValue()) - )) - .collect(Collectors.toList()); - BlockProto nextMarker = response.getNextMarker(); - - if (nextMarker.isInitialized()) { - return new InMemoryAliasMap.IterationResult(fileRegions, - Optional.of(PBHelperClient.convert(nextMarker))); - } else { - return new InMemoryAliasMap.IterationResult(fileRegions, - Optional.empty()); - } - - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + ListResponseProto response = ipc(() -> rpcProxy.list(null, request)); + List fileRegionsList = response.getFileRegionsList(); + + List fileRegions = fileRegionsList + .stream() + .map(kv -> new FileRegion( + PBHelperClient.convert(kv.getKey()), + PBHelperClient.convert(kv.getValue()) + )) + .collect(Collectors.toList()); + BlockProto nextMarker = response.getNextMarker(); + + if (nextMarker.isInitialized()) { + return new InMemoryAliasMap.IterationResult(fileRegions, + Optional.of(PBHelperClient.convert(nextMarker))); + } else { + return new InMemoryAliasMap.IterationResult(fileRegions, + Optional.empty()); } } @@ -175,19 +169,15 @@ public Optional read(@Nonnull Block block) .newBuilder() .setKey(PBHelperClient.convert(block)) .build(); - try { - ReadResponseProto response = rpcProxy.read(null, request); - - ProvidedStorageLocationProto providedStorageLocation = - response.getValue(); - if (providedStorageLocation.isInitialized()) { - return Optional.of(PBHelperClient.convert(providedStorageLocation)); - } - return Optional.empty(); + ReadResponseProto response = ipc(() -> rpcProxy.read(null, request)); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + ProvidedStorageLocationProto providedStorageLocation = + response.getValue(); + if (providedStorageLocation.isInitialized()) { + return Optional.of(PBHelperClient.convert(providedStorageLocation)); } + return Optional.empty(); + } @Override @@ -206,22 +196,14 @@ public void write(@Nonnull Block block, .build()) .build(); - try { - rpcProxy.write(null, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.write(null, request)); } @Override public String getBlockPoolId() throws IOException { - try { - BlockPoolResponseProto response = rpcProxy.getBlockPoolId(null, - BlockPoolRequestProto.newBuilder().build()); - return response.getBlockPoolId(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + BlockPoolResponseProto response = ipc(() -> rpcProxy.getBlockPoolId(null, + BlockPoolRequestProto.newBuilder().build())); + return response.getBlockPoolId(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java index 031b0e4512ad3..ae81be9a90d65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; @@ -42,7 +41,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -79,11 +79,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) InitReplicaRecoveryRequestProto req = InitReplicaRecoveryRequestProto .newBuilder().setBlock(PBHelper.convert(rBlock)).build(); InitReplicaRecoveryResponseProto resp; - try { - resp = rpcProxy.initReplicaRecovery(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + resp = ipc(() -> rpcProxy.initReplicaRecovery(NULL_CONTROLLER, req)); if (!resp.getReplicaFound()) { // No replica found on the remote node. return null; @@ -108,12 +104,9 @@ public String updateReplicaUnderRecovery(ExtendedBlock oldBlock, .setBlock(PBHelperClient.convert(oldBlock)) .setNewLength(newLength).setNewBlockId(newBlockId) .setRecoveryId(recoveryId).build(); - try { - return rpcProxy.updateReplicaUnderRecovery(NULL_CONTROLLER, req - ).getStorageUuid(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.updateReplicaUnderRecovery(NULL_CONTROLLER, req) + .getStorageUuid()); + } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java index 9310dd360c8fe..b0a326e5fca34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java @@ -29,13 +29,12 @@ import org.apache.hadoop.hdfs.server.protocol.FenceResponse; import org.apache.hadoop.hdfs.server.protocol.JournalInfo; import org.apache.hadoop.hdfs.server.protocol.JournalProtocol; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -69,11 +68,17 @@ public void journal(JournalInfo journalInfo, long epoch, long firstTxnId, .setNumTxns(numTxns) .setRecords(PBHelperClient.getByteString(records)) .build(); - try { - rpcProxy.journal(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.journal(NULL_CONTROLLER, req)); + } + + @Override + public FenceResponse fence(JournalInfo journalInfo, long epoch, + String fencerInfo) throws IOException { + FenceRequestProto req = FenceRequestProto.newBuilder().setEpoch(epoch) + .setJournalInfo(PBHelper.convert(journalInfo)).build(); + FenceResponseProto resp = ipc(() -> rpcProxy.fence(NULL_CONTROLLER, req)); + return new FenceResponse(resp.getPreviousEpoch(), + resp.getLastTransactionId(), resp.getInSync()); } @Override @@ -84,25 +89,7 @@ public void startLogSegment(JournalInfo journalInfo, long epoch, long txid) .setEpoch(epoch) .setTxid(txid) .build(); - try { - rpcProxy.startLogSegment(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - - @Override - public FenceResponse fence(JournalInfo journalInfo, long epoch, - String fencerInfo) throws IOException { - FenceRequestProto req = FenceRequestProto.newBuilder().setEpoch(epoch) - .setJournalInfo(PBHelper.convert(journalInfo)).build(); - try { - FenceResponseProto resp = rpcProxy.fence(NULL_CONTROLLER, req); - return new FenceResponse(resp.getPreviousEpoch(), - resp.getLastTransactionId(), resp.getInSync()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.startLogSegment(NULL_CONTROLLER, req)); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java index 201004dc6f5b5..fd40e0ecef34c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java @@ -51,14 +51,13 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.ProtocolTranslator; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -107,63 +106,39 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long GetBlocksRequestProto req = GetBlocksRequestProto.newBuilder() .setDatanode(PBHelperClient.convert((DatanodeID)datanode)).setSize(size) .setMinBlockSize(minBlockSize).setTimeInterval(timeInterval).build(); - try { - return PBHelper.convert(rpcProxy.getBlocks(NULL_CONTROLLER, req) - .getBlocks()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelper.convert(ipc(() -> rpcProxy.getBlocks(NULL_CONTROLLER, req) + .getBlocks())); } @Override public ExportedBlockKeys getBlockKeys() throws IOException { - try { - GetBlockKeysResponseProto rsp = rpcProxy.getBlockKeys(NULL_CONTROLLER, - VOID_GET_BLOCKKEYS_REQUEST); - return rsp.hasKeys() ? PBHelper.convert(rsp.getKeys()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetBlockKeysResponseProto rsp = ipc(() -> rpcProxy.getBlockKeys(NULL_CONTROLLER, + VOID_GET_BLOCKKEYS_REQUEST)); + return rsp.hasKeys() ? PBHelper.convert(rsp.getKeys()) : null; } @Override public long getTransactionID() throws IOException { - try { - return rpcProxy.getTransactionId(NULL_CONTROLLER, - VOID_GET_TRANSACTIONID_REQUEST).getTxId(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.getTransactionId(NULL_CONTROLLER, + VOID_GET_TRANSACTIONID_REQUEST).getTxId()); } @Override public long getMostRecentCheckpointTxId() throws IOException { - try { - return rpcProxy.getMostRecentCheckpointTxId(NULL_CONTROLLER, - GetMostRecentCheckpointTxIdRequestProto.getDefaultInstance()).getTxId(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return ipc(() -> rpcProxy.getMostRecentCheckpointTxId(NULL_CONTROLLER, + GetMostRecentCheckpointTxIdRequestProto.getDefaultInstance()).getTxId()); } @Override public CheckpointSignature rollEditLog() throws IOException { - try { - return PBHelper.convert(rpcProxy.rollEditLog(NULL_CONTROLLER, - VOID_ROLL_EDITLOG_REQUEST).getSignature()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelper.convert(ipc(() -> rpcProxy.rollEditLog(NULL_CONTROLLER, + VOID_ROLL_EDITLOG_REQUEST).getSignature())); } @Override public NamespaceInfo versionRequest() throws IOException { - try { - return PBHelper.convert(rpcProxy.versionRequest(NULL_CONTROLLER, - VOID_VERSION_REQUEST).getInfo()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelper.convert(ipc(() -> rpcProxy.versionRequest(NULL_CONTROLLER, + VOID_VERSION_REQUEST).getInfo())); } @Override @@ -172,11 +147,7 @@ public void errorReport(NamenodeRegistration registration, int errorCode, ErrorReportRequestProto req = ErrorReportRequestProto.newBuilder() .setErrorCode(errorCode).setMsg(msg) .setRegistration(PBHelper.convert(registration)).build(); - try { - rpcProxy.errorReport(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.errorReport(NULL_CONTROLLER, req)); } @Override @@ -184,13 +155,9 @@ public NamenodeRegistration registerSubordinateNamenode( NamenodeRegistration registration) throws IOException { RegisterRequestProto req = RegisterRequestProto.newBuilder() .setRegistration(PBHelper.convert(registration)).build(); - try { - return PBHelper.convert( - rpcProxy.registerSubordinateNamenode(NULL_CONTROLLER, req) - .getRegistration()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelper.convert( + ipc(() -> rpcProxy.registerSubordinateNamenode(NULL_CONTROLLER, req) + .getRegistration())); } @Override @@ -199,11 +166,7 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration) StartCheckpointRequestProto req = StartCheckpointRequestProto.newBuilder() .setRegistration(PBHelper.convert(registration)).build(); NamenodeCommandProto cmd; - try { - cmd = rpcProxy.startCheckpoint(NULL_CONTROLLER, req).getCommand(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + cmd = ipc(() -> rpcProxy.startCheckpoint(NULL_CONTROLLER, req).getCommand()); return PBHelper.convert(cmd); } @@ -213,11 +176,7 @@ public void endCheckpoint(NamenodeRegistration registration, EndCheckpointRequestProto req = EndCheckpointRequestProto.newBuilder() .setRegistration(PBHelper.convert(registration)) .setSignature(PBHelper.convert(sig)).build(); - try { - rpcProxy.endCheckpoint(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.endCheckpoint(NULL_CONTROLLER, req)); } @Override @@ -225,12 +184,8 @@ public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException { GetEditLogManifestRequestProto req = GetEditLogManifestRequestProto .newBuilder().setSinceTxId(sinceTxId).build(); - try { - return PBHelper.convert(rpcProxy.getEditLogManifest(NULL_CONTROLLER, req) - .getManifest()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + return PBHelper.convert(ipc(() -> rpcProxy.getEditLogManifest(NULL_CONTROLLER, req) + .getManifest())); } @Override @@ -244,38 +199,26 @@ public boolean isMethodSupported(String methodName) throws IOException { public boolean isUpgradeFinalized() throws IOException { IsUpgradeFinalizedRequestProto req = IsUpgradeFinalizedRequestProto .newBuilder().build(); - try { - IsUpgradeFinalizedResponseProto response = rpcProxy.isUpgradeFinalized( - NULL_CONTROLLER, req); - return response.getIsUpgradeFinalized(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + IsUpgradeFinalizedResponseProto response = ipc(() -> rpcProxy.isUpgradeFinalized( + NULL_CONTROLLER, req)); + return response.getIsUpgradeFinalized(); } @Override public boolean isRollingUpgrade() throws IOException { IsRollingUpgradeRequestProto req = IsRollingUpgradeRequestProto .newBuilder().build(); - try { - IsRollingUpgradeResponseProto response = rpcProxy.isRollingUpgrade( - NULL_CONTROLLER, req); - return response.getIsRollingUpgrade(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + IsRollingUpgradeResponseProto response = ipc(() -> rpcProxy.isRollingUpgrade( + NULL_CONTROLLER, req)); + return response.getIsRollingUpgrade(); } @Override public Long getNextSPSPath() throws IOException { GetNextSPSPathRequestProto req = GetNextSPSPathRequestProto.newBuilder().build(); - try { - GetNextSPSPathResponseProto nextSPSPath = - rpcProxy.getNextSPSPath(NULL_CONTROLLER, req); - return nextSPSPath.hasSpsPath() ? nextSPSPath.getSpsPath() : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + GetNextSPSPathResponseProto nextSPSPath = + ipc(() -> rpcProxy.getNextSPSPath(NULL_CONTROLLER, req)); + return nextSPSPath.hasSpsPath() ? nextSPSPath.getSpsPath() : null; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/InterQJournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/InterQJournalProtocolTranslatorPB.java index 79a133a2abecc..4544308fff2fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/InterQJournalProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/InterQJournalProtocolTranslatorPB.java @@ -20,14 +20,12 @@ package org.apache.hadoop.hdfs.qjournal.protocolPB; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.qjournal.protocol.InterQJournalProtocol; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestRequestProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetEditLogManifestResponseProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; @@ -35,6 +33,8 @@ import java.io.Closeable; import java.io.IOException; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; + /** * This class is the client side translator to translate the requests made on * {@link InterQJournalProtocol} interfaces to the RPC server implementing @@ -63,21 +63,16 @@ public void close() { public GetEditLogManifestResponseProto getEditLogManifestFromJournal( String jid, String nameServiceId, long sinceTxId, boolean inProgressOk) throws IOException { - try { - GetEditLogManifestRequestProto.Builder req; - req = GetEditLogManifestRequestProto.newBuilder() - .setJid(convertJournalId(jid)) - .setSinceTxId(sinceTxId) - .setInProgressOk(inProgressOk); - if (nameServiceId !=null) { - req.setNameServiceId(nameServiceId); - } - return rpcProxy.getEditLogManifestFromJournal(NULL_CONTROLLER, - req.build() - ); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetEditLogManifestRequestProto.Builder req; + req = GetEditLogManifestRequestProto.newBuilder() + .setJid(convertJournalId(jid)) + .setSinceTxId(sinceTxId) + .setInProgressOk(inProgressOk); + if (nameServiceId !=null) { + req.setNameServiceId(nameServiceId); } + return ipc(() -> rpcProxy.getEditLogManifestFromJournal(NULL_CONTROLLER, + req.build())); } private QJournalProtocolProtos.JournalIdProto convertJournalId(String jid) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java index 7da009c5f4ad7..54b9a8d69f554 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java @@ -63,13 +63,12 @@ import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.protocol.JournalProtocol; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; - import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; /** * This class is the client side translator to translate the requests made on @@ -97,36 +96,28 @@ public void close() { @Override public boolean isFormatted(String journalId, String nameServiceId) throws IOException { - try { - IsFormattedRequestProto.Builder req = IsFormattedRequestProto.newBuilder() - .setJid(convertJournalId(journalId)); - if (nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - - IsFormattedResponseProto resp = rpcProxy.isFormatted( - NULL_CONTROLLER, req.build()); - return resp.getIsFormatted(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + IsFormattedRequestProto.Builder req = IsFormattedRequestProto.newBuilder() + .setJid(convertJournalId(journalId)); + if (nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + + IsFormattedResponseProto resp = ipc(() -> rpcProxy.isFormatted( + NULL_CONTROLLER, req.build())); + return resp.getIsFormatted(); } @Override public GetJournalStateResponseProto getJournalState(String jid, String nameServiceId) throws IOException { - try { - GetJournalStateRequestProto.Builder req = GetJournalStateRequestProto - .newBuilder() - .setJid(convertJournalId(jid)); - if (nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - return rpcProxy.getJournalState(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetJournalStateRequestProto.Builder req = GetJournalStateRequestProto + .newBuilder() + .setJid(convertJournalId(jid)); + if (nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + return ipc(() -> rpcProxy.getJournalState(NULL_CONTROLLER, req.build())); } private JournalIdProto convertJournalId(String jid) { @@ -140,19 +131,15 @@ public void format(String jid, String nameServiceId, NamespaceInfo nsInfo, boolean force) throws IOException { - try { - FormatRequestProto.Builder req = FormatRequestProto.newBuilder() - .setJid(convertJournalId(jid)) - .setNsInfo(PBHelper.convert(nsInfo)) - .setForce(force); - if(nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - - rpcProxy.format(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + FormatRequestProto.Builder req = FormatRequestProto.newBuilder() + .setJid(convertJournalId(jid)) + .setNsInfo(PBHelper.convert(nsInfo)) + .setForce(force); + if(nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + + ipc(() -> rpcProxy.format(NULL_CONTROLLER, req.build())); } @Override @@ -160,20 +147,16 @@ public NewEpochResponseProto newEpoch(String jid, String nameServiceId, NamespaceInfo nsInfo, long epoch) throws IOException { - try { - NewEpochRequestProto.Builder req = NewEpochRequestProto.newBuilder() - .setJid(convertJournalId(jid)) - .setNsInfo(PBHelper.convert(nsInfo)) - .setEpoch(epoch); - - if(nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } + NewEpochRequestProto.Builder req = NewEpochRequestProto.newBuilder() + .setJid(convertJournalId(jid)) + .setNsInfo(PBHelper.convert(nsInfo)) + .setEpoch(epoch); - return rpcProxy.newEpoch(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if(nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + + return ipc(() -> rpcProxy.newEpoch(NULL_CONTROLLER, req.build())); } @Override @@ -187,22 +170,14 @@ public void journal(RequestInfo reqInfo, .setNumTxns(numTxns) .setRecords(PBHelperClient.getByteString(records)) .build(); - try { - rpcProxy.journal(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.journal(NULL_CONTROLLER, req)); } @Override public void heartbeat(RequestInfo reqInfo) throws IOException { - try { - rpcProxy.heartbeat(NULL_CONTROLLER, HeartbeatRequestProto.newBuilder() - .setReqInfo(convert(reqInfo)) - .build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.heartbeat(NULL_CONTROLLER, HeartbeatRequestProto.newBuilder() + .setReqInfo(convert(reqInfo)) + .build())); } private QJournalProtocolProtos.RequestInfoProto convert( @@ -227,11 +202,7 @@ public void startLogSegment(RequestInfo reqInfo, long txid, int layoutVersion) .setReqInfo(convert(reqInfo)) .setTxid(txid).setLayoutVersion(layoutVersion) .build(); - try { - rpcProxy.startLogSegment(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.startLogSegment(NULL_CONTROLLER, req)); } @Override @@ -243,11 +214,7 @@ public void finalizeLogSegment(RequestInfo reqInfo, long startTxId, .setStartTxId(startTxId) .setEndTxId(endTxId) .build(); - try { - rpcProxy.finalizeLogSegment(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.finalizeLogSegment(NULL_CONTROLLER, req)); } @Override @@ -257,79 +224,58 @@ public void purgeLogsOlderThan(RequestInfo reqInfo, long minTxIdToKeep) .setReqInfo(convert(reqInfo)) .setMinTxIdToKeep(minTxIdToKeep) .build(); - try { - rpcProxy.purgeLogs(NULL_CONTROLLER, req); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + ipc(() -> rpcProxy.purgeLogs(NULL_CONTROLLER, req)); } @Override public GetEditLogManifestResponseProto getEditLogManifest( String jid, String nameServiceId, long sinceTxId, boolean inProgressOk) throws IOException { - try { - GetEditLogManifestRequestProto.Builder req; - req = GetEditLogManifestRequestProto.newBuilder() - .setJid(convertJournalId(jid)) - .setSinceTxId(sinceTxId) - .setInProgressOk(inProgressOk); - if (nameServiceId !=null) { - req.setNameServiceId(nameServiceId); - } - return rpcProxy.getEditLogManifest(NULL_CONTROLLER, - req.build() - ); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetEditLogManifestRequestProto.Builder req; + req = GetEditLogManifestRequestProto.newBuilder() + .setJid(convertJournalId(jid)) + .setSinceTxId(sinceTxId) + .setInProgressOk(inProgressOk); + if (nameServiceId !=null) { + req.setNameServiceId(nameServiceId); } + return ipc(() -> rpcProxy.getEditLogManifest(NULL_CONTROLLER, + req.build())); } @Override public GetJournaledEditsResponseProto getJournaledEdits(String jid, String nameServiceId, long sinceTxId, int maxTxns) throws IOException { - try { - GetJournaledEditsRequestProto.Builder req = - GetJournaledEditsRequestProto.newBuilder() - .setJid(convertJournalId(jid)) - .setSinceTxId(sinceTxId) - .setMaxTxns(maxTxns); - if (nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - return rpcProxy.getJournaledEdits(NULL_CONTROLLER, req.build()); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); + GetJournaledEditsRequestProto.Builder req = + GetJournaledEditsRequestProto.newBuilder() + .setJid(convertJournalId(jid)) + .setSinceTxId(sinceTxId) + .setMaxTxns(maxTxns); + if (nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + return ipc(() -> rpcProxy.getJournaledEdits(NULL_CONTROLLER, req.build())); } @Override public PrepareRecoveryResponseProto prepareRecovery(RequestInfo reqInfo, long segmentTxId) throws IOException { - try { - return rpcProxy.prepareRecovery(NULL_CONTROLLER, - PrepareRecoveryRequestProto.newBuilder() + return ipc(() -> rpcProxy.prepareRecovery(NULL_CONTROLLER, + PrepareRecoveryRequestProto.newBuilder() .setReqInfo(convert(reqInfo)) .setSegmentTxId(segmentTxId) - .build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + .build())); } @Override public void acceptRecovery(RequestInfo reqInfo, SegmentStateProto stateToAccept, URL fromUrl) throws IOException { - try { - rpcProxy.acceptRecovery(NULL_CONTROLLER, - AcceptRecoveryRequestProto.newBuilder() + ipc(() -> rpcProxy.acceptRecovery(NULL_CONTROLLER, + AcceptRecoveryRequestProto.newBuilder() .setReqInfo(convert(reqInfo)) .setStateToAccept(stateToAccept) .setFromURL(fromUrl.toExternalForm()) - .build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + .build())); } public boolean isMethodSupported(String methodName) throws IOException { @@ -340,42 +286,30 @@ public boolean isMethodSupported(String methodName) throws IOException { @Override public void doPreUpgrade(String jid) throws IOException { - try { - DoPreUpgradeRequestProto.Builder req; - req = DoPreUpgradeRequestProto.newBuilder() - .setJid(convertJournalId(jid)); - rpcProxy.doPreUpgrade(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + DoPreUpgradeRequestProto.Builder req; + req = DoPreUpgradeRequestProto.newBuilder() + .setJid(convertJournalId(jid)); + ipc(() -> rpcProxy.doPreUpgrade(NULL_CONTROLLER, req.build())); } @Override public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException { - try { - rpcProxy.doUpgrade(NULL_CONTROLLER, - DoUpgradeRequestProto.newBuilder() + ipc(() -> rpcProxy.doUpgrade(NULL_CONTROLLER, + DoUpgradeRequestProto.newBuilder() .setJid(convertJournalId(journalId)) .setSInfo(PBHelper.convert(sInfo)) - .build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } + .build())); } @Override public void doFinalize(String jid, String nameServiceId) throws IOException { - try { - DoFinalizeRequestProto.Builder req = DoFinalizeRequestProto - .newBuilder() - .setJid(convertJournalId(jid)); - if (nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - rpcProxy.doFinalize(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + DoFinalizeRequestProto.Builder req = DoFinalizeRequestProto + .newBuilder() + .setJid(convertJournalId(jid)); + if (nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + ipc(() -> rpcProxy.doFinalize(NULL_CONTROLLER, req.build())); } @Override @@ -384,37 +318,29 @@ public Boolean canRollBack(String journalId, StorageInfo storage, StorageInfo prevStorage, int targetLayoutVersion) throws IOException { - try { - CanRollBackRequestProto.Builder req = CanRollBackRequestProto.newBuilder() - .setJid(convertJournalId(journalId)) - .setStorage(PBHelper.convert(storage)) - .setPrevStorage(PBHelper.convert(prevStorage)) - .setTargetLayoutVersion(targetLayoutVersion); - if (nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - CanRollBackResponseProto response = rpcProxy.canRollBack( - NULL_CONTROLLER, req.build()); - return response.getCanRollBack(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + CanRollBackRequestProto.Builder req = CanRollBackRequestProto.newBuilder() + .setJid(convertJournalId(journalId)) + .setStorage(PBHelper.convert(storage)) + .setPrevStorage(PBHelper.convert(prevStorage)) + .setTargetLayoutVersion(targetLayoutVersion); + if (nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + CanRollBackResponseProto response = ipc(() -> rpcProxy.canRollBack( + NULL_CONTROLLER, req.build())); + return response.getCanRollBack(); } @Override public void doRollback(String journalId, String nameServiceId) throws IOException { - try { - DoRollbackRequestProto.Builder req = DoRollbackRequestProto.newBuilder() - .setJid(convertJournalId(journalId)); + DoRollbackRequestProto.Builder req = DoRollbackRequestProto.newBuilder() + .setJid(convertJournalId(journalId)); - if (nameServiceId != null) { - req.setNameserviceId(nameServiceId); - } - rpcProxy.doRollback(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (nameServiceId != null) { + req.setNameserviceId(nameServiceId); } + ipc(() -> rpcProxy.doRollback(NULL_CONTROLLER, req.build())); } @Override @@ -422,37 +348,28 @@ public void discardSegments(String journalId, String nameServiceId, long startTxId) throws IOException { - try { - DiscardSegmentsRequestProto.Builder req = DiscardSegmentsRequestProto - .newBuilder() - .setJid(convertJournalId(journalId)).setStartTxId(startTxId); + DiscardSegmentsRequestProto.Builder req = DiscardSegmentsRequestProto + .newBuilder() + .setJid(convertJournalId(journalId)).setStartTxId(startTxId); - if (nameServiceId != null) { - req.setNameServiceId(nameServiceId); - } - rpcProxy.discardSegments(NULL_CONTROLLER, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + if (nameServiceId != null) { + req.setNameServiceId(nameServiceId); } + ipc(() -> rpcProxy.discardSegments(NULL_CONTROLLER, req.build())); } @Override public Long getJournalCTime(String journalId, String nameServiceId) throws IOException { - try { - - GetJournalCTimeRequestProto.Builder req = GetJournalCTimeRequestProto - .newBuilder() - .setJid(convertJournalId(journalId)); - if(nameServiceId !=null) { - req.setNameServiceId(nameServiceId); - } - GetJournalCTimeResponseProto response = rpcProxy.getJournalCTime( - NULL_CONTROLLER, req.build()); - return response.getResultCTime(); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + GetJournalCTimeRequestProto.Builder req = GetJournalCTimeRequestProto + .newBuilder() + .setJid(convertJournalId(journalId)); + if(nameServiceId !=null) { + req.setNameServiceId(nameServiceId); } + GetJournalCTimeResponseProto response = ipc(() -> rpcProxy.getJournalCTime( + NULL_CONTROLLER, req.build())); + return response.getResultCTime(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index ab909aef2ecd8..7e33ab5c759f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -125,7 +125,7 @@ public class JournalNodeRpcServer implements QJournalProtocol, BlockingService interQJournalProtocolService = InterQJournalProtocolService .newReflectiveBlockingService(qJournalProtocolServerSideTranslatorPB); - DFSUtil.addPBProtocol(confCopy, InterQJournalProtocolPB.class, + DFSUtil.addInternalPBProtocol(confCopy, InterQJournalProtocolPB.class, interQJournalProtocolService, server); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 4613d37f606ed..8fb009dab850a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1528,14 +1528,14 @@ private void initIpcServer() throws IOException { = new ReconfigurationProtocolServerSideTranslatorPB(this); service = ReconfigurationProtocolService .newReflectiveBlockingService(reconfigurationProtocolXlator); - DFSUtil.addPBProtocol(getConf(), ReconfigurationProtocolPB.class, service, + DFSUtil.addInternalPBProtocol(getConf(), ReconfigurationProtocolPB.class, service, ipcServer); InterDatanodeProtocolServerSideTranslatorPB interDatanodeProtocolXlator = new InterDatanodeProtocolServerSideTranslatorPB(this); service = InterDatanodeProtocolService .newReflectiveBlockingService(interDatanodeProtocolXlator); - DFSUtil.addPBProtocol(getConf(), InterDatanodeProtocolPB.class, service, + DFSUtil.addInternalPBProtocol(getConf(), InterDatanodeProtocolPB.class, service, ipcServer); LOG.info("Opened IPC server at {}", ipcServer.getListenerAddress()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index a3d9746be1224..8815f898785a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -246,7 +246,7 @@ private BackupNodeRpcServer(Configuration conf, BackupNode nn) new JournalProtocolServerSideTranslatorPB(this); BlockingService service = JournalProtocolService .newReflectiveBlockingService(journalProtocolTranslator); - DFSUtil.addPBProtocol(conf, JournalProtocolPB.class, service, + DFSUtil.addInternalPBProtocol(conf, JournalProtocolPB.class, service, this.clientRpcServer); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index c5e6d0418593f..4a041dbec2758 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -371,24 +371,24 @@ public NameNodeRpcServer(Configuration conf, NameNode nn) .build(); // Add all the RPC protocols that the namenode implements - DFSUtil.addPBProtocol(conf, HAServiceProtocolPB.class, haPbService, + DFSUtil.addInternalPBProtocol(conf, HAServiceProtocolPB.class, haPbService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, ReconfigurationProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, ReconfigurationProtocolPB.class, reconfigurationPbService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService, + DFSUtil.addInternalPBProtocol(conf, NamenodeProtocolPB.class, NNPbService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, + DFSUtil.addInternalPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, RefreshAuthorizationPolicyProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshAuthorizationPolicyProtocolPB.class, refreshAuthService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshUserMappingsProtocolPB.class, refreshUserMappingService, serviceRpcServer); // We support Refreshing call queue here in case the client RPC queue is full - DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshCallQueueProtocolPB.class, refreshCallQueueService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, GenericRefreshProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, GenericRefreshProtocolPB.class, genericRefreshService, serviceRpcServer); - DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, GetUserMappingsProtocolPB.class, getUserMappingService, serviceRpcServer); // Update the address with the correct port @@ -431,7 +431,7 @@ public NameNodeRpcServer(Configuration conf, NameNode nn) .setSecretManager(namesystem.getDelegationTokenSecretManager()) .build(); - DFSUtil.addPBProtocol(conf, DatanodeLifelineProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, DatanodeLifelineProtocolPB.class, lifelineProtoPbService, lifelineRpcServer); // Update the address with the correct port @@ -474,23 +474,23 @@ public NameNodeRpcServer(Configuration conf, NameNode nn) .build(); // Add all the RPC protocols that the namenode implements - DFSUtil.addPBProtocol(conf, HAServiceProtocolPB.class, haPbService, + DFSUtil.addInternalPBProtocol(conf, HAServiceProtocolPB.class, haPbService, clientRpcServer); - DFSUtil.addPBProtocol(conf, ReconfigurationProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, ReconfigurationProtocolPB.class, reconfigurationPbService, clientRpcServer); - DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService, + DFSUtil.addInternalPBProtocol(conf, NamenodeProtocolPB.class, NNPbService, clientRpcServer); - DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, + DFSUtil.addInternalPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, clientRpcServer); - DFSUtil.addPBProtocol(conf, RefreshAuthorizationPolicyProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshAuthorizationPolicyProtocolPB.class, refreshAuthService, clientRpcServer); - DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshUserMappingsProtocolPB.class, refreshUserMappingService, clientRpcServer); - DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, RefreshCallQueueProtocolPB.class, refreshCallQueueService, clientRpcServer); - DFSUtil.addPBProtocol(conf, GenericRefreshProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, GenericRefreshProtocolPB.class, genericRefreshService, clientRpcServer); - DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class, + DFSUtil.addInternalPBProtocol(conf, GetUserMappingsProtocolPB.class, getUserMappingService, clientRpcServer); // set service-level authorization security policy diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index 79327a780fceb..21b93d87761ae 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -36,6 +36,10 @@ org.apache.hadoop hadoop-hdfs-client + + org.apache.hadoop + hadoop-common + org.apache.hadoop hadoop-hdfs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java index ea8c17de8879a..305e776180001 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; @@ -34,7 +33,8 @@ import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsRequestProto; import org.apache.hadoop.thirdparty.protobuf.RpcController; -import org.apache.hadoop.thirdparty.protobuf.ServiceException; + +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc; @Private public class HSAdminRefreshProtocolClientSideTranslatorPB implements @@ -73,43 +73,27 @@ public void close() throws IOException { @Override public void refreshAdminAcls() throws IOException { - try { - rpcProxy.refreshAdminAcls(NULL_CONTROLLER, - VOID_REFRESH_ADMIN_ACLS_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshAdminAcls(NULL_CONTROLLER, + VOID_REFRESH_ADMIN_ACLS_REQUEST)); } @Override public void refreshLoadedJobCache() throws IOException { - try { - rpcProxy.refreshLoadedJobCache(NULL_CONTROLLER, - VOID_REFRESH_LOADED_JOB_CACHE_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshLoadedJobCache(NULL_CONTROLLER, + VOID_REFRESH_LOADED_JOB_CACHE_REQUEST)); } @Override public void refreshJobRetentionSettings() throws IOException { - try { - rpcProxy.refreshJobRetentionSettings(NULL_CONTROLLER, - VOID_REFRESH_JOB_RETENTION_SETTINGS_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshJobRetentionSettings(NULL_CONTROLLER, + VOID_REFRESH_JOB_RETENTION_SETTINGS_REQUEST)); } @Override public void refreshLogRetentionSettings() throws IOException { - try { - rpcProxy.refreshLogRetentionSettings(NULL_CONTROLLER, - VOID_REFRESH_LOG_RETENTION_SETTINGS_REQUEST); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } + ipc(() -> rpcProxy.refreshLogRetentionSettings(NULL_CONTROLLER, + VOID_REFRESH_LOG_RETENTION_SETTINGS_REQUEST)); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index b9e181d1ae369..eb770c4ff1987 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -39,6 +39,7 @@ com.google.protobuf protobuf-java + ${transient.protobuf2.scope} org.apache.avro diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3fe8f8fa8f4d0..418c173816672 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -84,8 +84,14 @@ 1.1 - + + 2.5.0 + + + compile + + ${common.protobuf2.scope} 3.7.1 ${env.HADOOP_PROTOC_PATH} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 746abf385e6bd..1ff770cc42023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -191,6 +191,7 @@ com.google.protobuf protobuf-java + ${transient.protobuf2.scope} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index f85d875bf490b..4f6b40891d38e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -132,6 +132,7 @@ com.google.protobuf protobuf-java + ${transient.protobuf2.scope} org.bouncycastle diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index 36aecf141ad7c..c7c6dc510424c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -126,6 +125,8 @@ import org.apache.hadoop.thirdparty.protobuf.ServiceException; +import static org.apache.hadoop.ipc.internal.ShadedProtobufHelper.getRemoteException; + @Private public class ResourceManagerAdministrationProtocolPBClientImpl implements ResourceManagerAdministrationProtocol, Closeable { @@ -243,7 +244,7 @@ public String[] getGroupsForUser(String user) throws IOException { return (String[]) responseProto.getGroupsList().toArray( new String[responseProto.getGroupsCount()]); } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); + throw getRemoteException(e); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml index 953791e01e9ff..864067ce9746d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml @@ -38,6 +38,7 @@ com.google.protobuf protobuf-java ${hadoop.protobuf.version} + ${transient.protobuf2.scope} io.netty diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index 8ef7b0d03a36b..c5142c116c2aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -89,6 +89,7 @@ com.google.protobuf protobuf-java + ${transient.protobuf2.scope} junit diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 71981e02e7be0..fcd68ab2f52a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -81,6 +81,7 @@ com.google.protobuf protobuf-java + ${transient.protobuf2.scope} commons-io From 3d7b58d8a50b672e54d8fd08cfe7394b724efc0f Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Fri, 13 Oct 2023 20:01:44 +0100 Subject: [PATCH 091/155] HADOOP-18916. Exclude all module-info classes from uber jars (#6131) Removes java9 and java11 from all modules pulled into the hadoop-client and hadoop-client-minicluster modules. Contributed by PJ Fanning --- .../hadoop-client-minicluster/pom.xml | 15 ++------------- .../hadoop-client-runtime/pom.xml | 15 ++------------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 10c6d41dd6cce..9c9df2216fe8e 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -748,21 +748,10 @@ - com.fasterxml.jackson.*:* - - META-INF/versions/11/module-info.class - - - - com.google.code.gson:gson - - META-INF/versions/9/module-info.class - - - - org.apache.commons:commons-compress + *:* META-INF/versions/9/module-info.class + META-INF/versions/11/module-info.class diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 60790d932edaf..1391da71ffd3c 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -239,21 +239,10 @@ - com.fasterxml.jackson.*:* - - META-INF/versions/11/module-info.class - - - - com.google.code.gson:gson - - META-INF/versions/9/module-info.class - - - - org.apache.commons:commons-compress + *:* META-INF/versions/9/module-info.class + META-INF/versions/11/module-info.class From 8963b25ab3bedb482b75cc56695e5c88b69d7c4c Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sat, 14 Oct 2023 06:34:44 +0800 Subject: [PATCH 092/155] HADOOP-18926.Add some comments related to NodeFencer. (#6162) --- .../src/main/java/org/apache/hadoop/ha/NodeFencer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java index f42db1d0c5561..cb192a6ebb9a3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java @@ -46,7 +46,7 @@ * The fencing methods that ship with Hadoop may also be referred to * by shortened names:
    *

      - *
    • shell(/path/to/some/script.sh args...)
      • + *
    • shell(/path/to/some/script.sh args...) (see {@link ShellCommandFencer}) *
    • sshfence(...) (see {@link SshFenceByTcpPort}) *
    • powershell(...) (see {@link PowerShellFencer}) *
    From 00f8cdcb0f20fceee3af322a20689bb52e2d4d9d Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 14 Oct 2023 10:00:28 +0800 Subject: [PATCH 093/155] YARN-11571. [GPG] Add Information About YARN GPG in Federation.md (#6158) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../src/site/markdown/Federation.md | 85 ++++++++++++++++++- .../src/site/markdown/YarnUI2.md | 11 +-- 2 files changed, 89 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index a61325238447f..66c79c94cc9f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -384,7 +384,7 @@ Optional: |`yarn.federation.state-store.heartbeat-interval-secs` | `60` | The rate at which RMs report their membership to the federation to the central state-store. | -###ON ROUTER: +### ON ROUTER: These are extra configurations that should appear in the **conf/yarn-site.xml** at each Router. @@ -465,7 +465,88 @@ If we want to use JCache, we can configure `yarn.federation.cache.class` to `org This is a Cache implemented based on the Guava framework. If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`. -###ON NMs: +### ON GPG: + +GlobalPolicyGenerator, abbreviated as "GPG," is used for the automatic generation of global policies for subClusters. + +These are extra configurations that should appear in the **conf/yarn-site.xml** for GPG. We allow only one GPG. + +Optional: + +| Property | Example | Description | +|:------------------------------------------------------------------|:---------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.federation.gpg.scheduled.executor.threads` | `10` | The number of threads to use for the GPG scheduled executor service. default is 10. | +| `yarn.federation.gpg.subcluster.cleaner.interval-ms` | `-1` | The interval at which the subcluster cleaner runs, -1 means disabled | +| `yarn.federation.gpg.subcluster.heartbeat.expiration-ms` | `30m` | The expiration time for a subcluster heartbeat, default is 30 minutes. | +| `yarn.federation.gpg.application.cleaner.class` | `org.apache.hadoop.yarn.server.globalpolicygenerator.DefaultApplicationCleaner` | The application cleaner class to use. | +| `yarn.federation.gpg.application.cleaner.interval-ms` | `-1` | The interval at which the application cleaner runs, -1 means disabled | +| `yarn.federation.gpg.application.cleaner.contact.router.spec` | `3,10,600000` | Should have three values separated by comma: minimal success retries, maximum total retry, retry interval (ms). | +| `yarn.federation.gpg.policy.generator.interval` | `1h` | The interval at which the policy generator runs, default is one hour. | +| `yarn.federation.gpg.policy.generator.class` | `org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.NoOpGlobalPolicy` | The configured policy generator class, runs NoOpGlobalPolicy by default. | +| `yarn.federation.gpg.policy.generator.readonly` | `false` | Whether or not the policy generator is running in read only (won't modify policies), default is false.` | +| `yarn.federation.gpg.policy.generator.blacklist` | | Which sub-clusters the policy generator should blacklist. | +| `yarn.federation.gpg.policy.generator.load-based.pending.minimum` | `100` | The minimum number of pending applications in the subCluster. | +| `yarn.federation.gpg.policy.generator.load-based.pending.maximum` | `1000` | The maximum number of pending applications in the subCluster. | +| `yarn.federation.gpg.policy.generator.load-based.weight.minimum` | `0` | If a subCluster has a very high load, we will assign this value to the subCluster. The default value is 0, which means that we no longer assign appliaction to this subCluster. | +| `yarn.federation.gpg.policy.generator.load-based.edit.maximum` | `3` | This value represents the number of subClusters we want to calculate. default is 3. | +| `yarn.federation.gpg.policy.generator.load-based.scaling` | `LINEAR` | We provide 4 calculation methods: NONE, LINEAR, QUADRATIC, LOG. | +| `yarn.federation.gpg.webapp.address` | `0.0.0.0:8069` | The address of the GPG web application. | +| `yarn.federation.gpg.webapp.https.address` | `0.0.0.0:8070` | The https address of the GPG web application. | + +- yarn.federation.gpg.application.cleaner.contact.router.spec + +Specifications on how (many times) to contact Router for apps. We need to +do this because Router might return partial application list because some +sub-cluster RM is not responsive (e.g. failing over). Should have three values separated by comma: minimal success retries, +maximum total retry, retry interval (ms). + +- yarn.federation.gpg.policy.generator.load-based.scaling + +Note, this calculation method is when the number of Pending Applications in +the subCluster is less than yarn.federation.gpg.policy.generator.load-based.pending.maximum. + +maxPendingVal = `yarn.federation.gpg.policy.generator.load-based.pending.maximum` - +`yarn.federation.gpg.policy.generator.load-based.pending.minimum` + +curPendingVal = `Pending Applications in the subCluster` - +`yarn.federation.gpg.policy.generator.load-based.pending.minimum` + +No calculation is required, and the weight is 1 at this time. + +- LINEAR: + For linear computation, + we will use (maxPendingVal - curPendingVal) / (maxPendingVal). + +- QUADRATIC: + Calculated using quadratic, We will calculate quadratic for maxPendingVal, curPendingVal, + then use this formula = (maxPendingVal - curPendingVal) / (maxPendingVal). + +- LOG(LOGARITHM): + Calculated using logarithm, We will calculate logarithm for maxPendingVal, curPendingVal, + then use this formula = (maxPendingVal - curPendingVal) / (maxPendingVal). + +LINEAR is used by default. + +Security: + +Kerberos supported in GPG. + +| Property | Example | Description | +|:--------------------------------------------------|:--------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.federation.gpg.keytab.file` | | The keytab file used by GPG to login as its service principal. The principal name is configured with 'yarn.federation.gpg.kerberos.principal.hostname'. | +| `yarn.federation.gpg.kerberos.principal` | | The GPG service principal. This is typically set to GPG/_HOST@REALM.TLD. GPG will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on GPG in setup. | +| `yarn.federation.gpg.kerberos.principal.hostname` | | Optional. The hostname for the GPG containing this configuration file. Will be different for each machine. Defaults to current hostname. | + +Enabling CORS support: + +To enable cross-origin support (CORS) for the Yarn Router, please set the following configuration parameters: + +| Property | Example | Description | +|---------------------------------------------------|---------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| +| `hadoop.http.filter.initializers` | `org.apache.hadoop.security.HttpCrossOriginFilterInitializer` | Optional. Set the filter to HttpCrossOriginFilterInitializer, Configure this parameter in core-site.xml. | +| `yarn.federation.gpg.webapp.cross-origin.enabled` | `true` | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. | + +### ON NMs: These are extra configurations that should appear in the **conf/yarn-site.xml** at each NodeManager. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index b4d5386a7987f..0b309b6c376f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -36,12 +36,13 @@ Please note that, If you run YARN daemons locally in your machine for test purpo you need the following configurations added to `yarn-site.xml` to enable cross origin (CORS) support. -| Configuration Property | Value | Description | -|:---- |:---- |:---- | +| Configuration Property | Value | Description | +|:---- |:---- |:-----------------------------------------| | `yarn.timeline-service.http-cross-origin.enabled` | true | Enable CORS support for Timeline Server | -| `yarn.resourcemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Resource Manager | -| `yarn.nodemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Node Manager | -| `yarn.router.webapp.cross-origin.enabled` | true | Enable CORS support for Yarn Router | +| `yarn.resourcemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Resource Manager | +| `yarn.nodemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Node Manager | +| `yarn.router.webapp.cross-origin.enabled` | true | Enable CORS support for Yarn Router | +| `yarn.federation.gpg.webapp.cross-origin.enabled` | true | Enable CORS support for Yarn GPG | Also please ensure that CORS related configurations are enabled in `core-site.xml`. Kindly refer [here](../../hadoop-project-dist/hadoop-common/HttpAuthentication.html) From c8abca30045258668867eeab89fa05fe88fe5be6 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:34:40 +0800 Subject: [PATCH 094/155] HDFS-17210. Optimize AvailableSpaceBlockPlacementPolicy. (#6113). Contributed by GuoPhilipse. Reviewed-by: He Xiaoqiao Signed-off-by: Shuyan Zhang --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++ .../AvailableSpaceBlockPlacementPolicy.java | 33 +++++++- .../src/main/resources/hdfs-default.xml | 11 +++ ...estAvailableSpaceBlockPlacementPolicy.java | 81 ++++++++++++++++++- 4 files changed, 124 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 6b7e4fdb406db..c783fc76d091b 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1241,6 +1241,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = 5; + public static final String + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_KEY = + "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance-limit"; + public static final int + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_DEFAULT = + 100; public static final String DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 9e349423daa77..9eba57830ea4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_DEFAULT; import java.util.Collection; import java.util.EnumMap; @@ -50,7 +52,10 @@ public class AvailableSpaceBlockPlacementPolicy extends private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); private int balancedSpaceTolerance = - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; + + private int balancedSpaceToleranceLimit = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_DEFAULT; private boolean optimizeLocal; @Override @@ -59,8 +64,8 @@ public void initialize(Configuration conf, FSClusterStats stats, super.initialize(conf, stats, clusterMap, host2datanodeMap); float balancedPreferencePercent = conf.getFloat( - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); LOG.info("Available space block placement policy initialized: " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY @@ -71,6 +76,11 @@ public void initialize(Configuration conf, FSClusterStats stats, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); + balancedSpaceToleranceLimit = + conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_DEFAULT); + optimizeLocal = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY, DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_DEFAULT); @@ -87,6 +97,17 @@ public void initialize(Configuration conf, FSClusterStats stats, + " receive more block allocations."); } + if (balancedSpaceToleranceLimit > 100 || balancedSpaceToleranceLimit < 0) { + LOG.warn("The value of " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_KEY + + " is invalid, Current value is " + balancedSpaceToleranceLimit + ", Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_DEFAULT + + " will be used instead."); + + balancedSpaceToleranceLimit = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_DEFAULT; + } + if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { LOG.warn("The value of " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY @@ -201,8 +222,12 @@ private DatanodeDescriptor select(DatanodeDescriptor a, DatanodeDescriptor b, */ protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b, boolean isBalanceLocal) { + + boolean toleranceLimit = Math.max(a.getDfsUsedPercent(), b.getDfsUsedPercent()) + < balancedSpaceToleranceLimit; if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance || (( + || (toleranceLimit && Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) + < balancedSpaceTolerance) || (( isBalanceLocal && a.getDfsUsedPercent() < 50))) { return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 678908d901ec9..4ff825d642d54 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5173,6 +5173,17 @@ + + dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance-limit + 100 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy. + Special value between 0 and 100, inclusive. if the value is set beyond the scope, + this value will be set as 100 by default. + + + dfs.namenode.available-space-block-placement-policy.balance-local-node diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index af7690d681e49..34ac6751cb75e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -58,9 +58,12 @@ public class TestAvailableSpaceBlockPlacementPolicy { @BeforeClass public static void setupCluster() throws Exception { conf = new HdfsConfiguration(); - conf.setFloat( - DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, - 0.6f); + conf.setFloat(DFSConfigKeys. + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, + 0.6f); + conf.setInt(DFSConfigKeys. + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_LIMIT_KEY, + 93); String[] racks = new String[numRacks]; for (int i = 0; i < numRacks; i++) { racks[i] = "/rack" + i; @@ -219,6 +222,78 @@ public void testChooseSimilarDataNode() { tolerateDataNodes[0], false) == 1); } + + @Test + public void testCompareDataNode() { + DatanodeDescriptor[] tolerateDataNodes; + DatanodeStorageInfo[] tolerateStorages; + int capacity = 5; + Collection allTolerateNodes = new ArrayList<>(capacity); + String[] ownerRackOfTolerateNodes = new String[capacity]; + for (int i = 0; i < capacity; i++) { + ownerRackOfTolerateNodes[i] = "rack"+i; + } + tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages); + + Collections.addAll(allTolerateNodes, tolerateDataNodes); + final BlockManager bm = namenode.getNamesystem().getBlockManager(); + AvailableSpaceBlockPlacementPolicy toleratePlacementPolicy = + (AvailableSpaceBlockPlacementPolicy)bm.getBlockPlacementPolicy(); + + //96.6% + updateHeartbeatWithUsage(tolerateDataNodes[0], + 30 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 29 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + //93.3% + updateHeartbeatWithUsage(tolerateDataNodes[1], + 30 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 28 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + //90.0% + updateHeartbeatWithUsage(tolerateDataNodes[2], + 30 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 27 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + //86.6% + updateHeartbeatWithUsage(tolerateDataNodes[3], + 30 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 26 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + //83.3% + updateHeartbeatWithUsage(tolerateDataNodes[4], + 30 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 25 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[1], false) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], + tolerateDataNodes[0], false) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], + tolerateDataNodes[2], false) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[1], false) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[3], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[3], + tolerateDataNodes[2], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[4], false) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[4], + tolerateDataNodes[2], false) == -1); + } + @AfterClass public static void teardownCluster() { if (namenode != null) { From 2736f8856144c8361be3821cd1c158ef2d9a2dec Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Mon, 16 Oct 2023 15:17:58 -0400 Subject: [PATCH 095/155] YARN.11590. RM process stuck after calling confStore.format() when ZK SSL/TLS is enabled, as netty thread waits indefinitely. Contributed by Ferenc Erdelyi --- .../yarn/server/resourcemanager/ResourceManager.java | 9 +++++---- .../scheduler/capacity/conf/YarnConfigurationStore.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 90eaed3d8a02a..f358a22cee715 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1903,10 +1903,11 @@ static void deleteRMConfStore(Configuration conf) throws Exception { } if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) { - YarnConfigurationStore confStore = YarnConfigurationStoreFactory - .getStore(conf); - confStore.initialize(conf, conf, rmContext); - confStore.format(); + try (YarnConfigurationStore confStore = YarnConfigurationStoreFactory + .getStore(conf)) { + confStore.initialize(conf, conf, rmContext); + confStore.format(); + } } else { System.out.println(String.format("Scheduler Configuration format only " + "supported by %s.", MutableConfScheduler.class.getSimpleName())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 0f44f439056bb..2ec2ea442a64e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -44,7 +44,7 @@ * {@code getPendingMutations}, and replay/confirm them via * {@code confirmMutation} as in the normal case. */ -public abstract class YarnConfigurationStore { +public abstract class YarnConfigurationStore implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(YarnConfigurationStore.class); From 42e695d510befabf3b7cfc52349b0d78faadb249 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 17 Oct 2023 12:59:50 +0100 Subject: [PATCH 096/155] HADOOP-18932. S3A. upgrade AWS v2 SDK to 2.20.160 and v1 to 1.12.565 (#6178) v1 => 1.12.565 v2 => 2.20.160 Only the v2 one is distributed; v1 is needed in deployments only to support v1 credential providers Contributed by Steve Loughran --- LICENSE-binary | 2 +- hadoop-project/pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 4916fda109239..5daa51efd8945 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -360,7 +360,7 @@ org.objenesis:objenesis:2.6 org.xerial.snappy:snappy-java:1.1.10.4 org.yaml:snakeyaml:2.0 org.wildfly.openssl:wildfly-openssl:1.1.3.Final -software.amazon.awssdk:bundle:jar:2.20.128 +software.amazon.awssdk:bundle:jar:2.20.160 -------------------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 418c173816672..30e5132c21b30 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -186,10 +186,10 @@ 1.3.1 1.0-beta-1 900 - 1.12.499 - 2.7.1 - 2.20.128 + 1.12.565 + 2.20.160 1.0.1 + 2.7.1 1.11.2 2.1 0.7 From e5eb404bb3e16fab460d6e4b90f82b3398612485 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 17 Oct 2023 15:17:16 +0100 Subject: [PATCH 097/155] HADOOP-18939. NPE in AWS v2 SDK RetryOnErrorCodeCondition.shouldRetry() (#6193) MultiObjectDeleteException to fill in the error details See also: https://github.com/aws/aws-sdk-java-v2/issues/4600 Contributed by Steve Loughran --- .../s3a/impl/MultiObjectDeleteException.java | 33 ++++++++++++++++++- .../fs/s3a/impl/TestErrorTranslation.java | 22 +++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java index 6082c2f08daff..72ead1fb151fc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteException.java @@ -22,6 +22,8 @@ import java.nio.file.AccessDeniedException; import java.util.List; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.services.s3.model.S3Error; import software.amazon.awssdk.services.s3.model.S3Exception; import org.slf4j.Logger; @@ -55,10 +57,39 @@ public class MultiObjectDeleteException extends S3Exception { */ public static final String ACCESS_DENIED = "AccessDenied"; + /** + * Field value for the superclass builder: {@value}. + */ + private static final int STATUS_CODE = SC_200_OK; + + /** + * Field value for the superclass builder: {@value}. + */ + private static final String ERROR_CODE = "MultiObjectDeleteException"; + + /** + * Field value for the superclass builder: {@value}. + */ + private static final String SERVICE_NAME = "Amazon S3"; + + /** + * Extracted error list. + */ private final List errors; public MultiObjectDeleteException(List errors) { - super(builder().message(errors.toString()).statusCode(SC_200_OK)); + super(builder() + .message(errors.toString()) + .awsErrorDetails( + AwsErrorDetails.builder() + .errorCode(ERROR_CODE) + .errorMessage(ERROR_CODE) + .serviceName(SERVICE_NAME) + .sdkHttpResponse(SdkHttpResponse.builder() + .statusCode(STATUS_CODE) + .build()) + .build()) + .statusCode(STATUS_CODE)); this.errors = errors; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java index 71536880dd5cb..dd4abe64c65cb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java @@ -22,15 +22,20 @@ import java.net.ConnectException; import java.net.NoRouteToHostException; import java.net.UnknownHostException; +import java.util.Collections; +import org.assertj.core.api.Assertions; import org.junit.Test; +import software.amazon.awssdk.awscore.retry.conditions.RetryOnErrorCodeCondition; import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.retry.RetryPolicyContext; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.test.AbstractHadoopTestBase; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException; import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertTrue; /** * Unit tests related to the {@link ErrorTranslation} class. @@ -112,4 +117,21 @@ public NoConstructorIOE() { } } + + @Test + public void testMultiObjectExceptionFilledIn() throws Throwable { + + MultiObjectDeleteException ase = + new MultiObjectDeleteException(Collections.emptyList()); + RetryPolicyContext context = RetryPolicyContext.builder() + .exception(ase) + .build(); + RetryOnErrorCodeCondition retry = RetryOnErrorCodeCondition.create(""); + assertTrue("retry policy of MultiObjectException", + retry.shouldRetry(context)); + + Assertions.assertThat(retry.shouldRetry(context)) + .describedAs("retry policy of MultiObjectException") + .isFalse(); + } } From e0563fed50f447a31e683d91e33767fed8f7445e Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 17 Oct 2023 15:37:36 +0100 Subject: [PATCH 098/155] HADOOP-18908. Improve S3A region handling. (#6187) S3A region logic improved for better inference and to be compatible with previous releases 1. If you are using an AWS S3 AccessPoint, its region is determined from the ARN itself. 2. If fs.s3a.endpoint.region is set and non-empty, it is used. 3. If fs.s3a.endpoint is an s3.*.amazonaws.com url, the region is determined by by parsing the URL Note: vpce endpoints are not handled by this. 4. If fs.s3a.endpoint.region==null, and none could be determined from the endpoint, use us-east-2 as default. 5. If fs.s3a.endpoint.region=="" then it is handed off to The default AWS SDK resolution process. Consult the AWS SDK documentation for the details on its resolution process, knowing that it is complicated and may use environment variables, entries in ~/.aws/config, IAM instance information within EC2 deployments and possibly even JSON resources on the classpath. Put differently: it is somewhat brittle across deployments. Contributed by Ahmar Suhail --- .../fs/statistics/StoreStatisticNames.java | 4 - .../org/apache/hadoop/fs/s3a/Constants.java | 6 + .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 115 +++++++++++-- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 85 +--------- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 33 ++-- .../org/apache/hadoop/fs/s3a/Statistic.java | 5 - .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 7 + .../hadoop/fs/s3a/ITestS3AConfiguration.java | 17 ++ .../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 153 ++++++++++++------ .../ITestSessionDelegationInFilesystem.java | 5 +- .../src/test/resources/core-site.xml | 18 ++- 11 files changed, 269 insertions(+), 179 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java index 3a8927aba493e..c04c1bb47fcea 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java @@ -407,10 +407,6 @@ public final class StoreStatisticNames { public static final String MULTIPART_UPLOAD_LIST = "multipart_upload_list"; - /** Probe for store region: {@value}. */ - public static final String STORE_REGION_PROBE - = "store_region_probe"; - private StoreStatisticNames() { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 492508225fa3e..d69d01f99450f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1179,6 +1179,12 @@ private Constants() { */ public static final String AWS_S3_CENTRAL_REGION = "us-east-1"; + /** + * The default S3 region when using cross region client. + * Value {@value}. + */ + public static final String AWS_S3_DEFAULT_REGION = "us-east-2"; + /** * Require that all S3 access is made through Access Points. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index c85263f1903ab..6aaa0e2d06605 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.awscore.util.AwsHostNameUtils; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; @@ -48,6 +49,9 @@ import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector; import org.apache.hadoop.fs.store.LogExactlyOnce; +import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; +import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION; +import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT; import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS; import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS; @@ -66,12 +70,27 @@ public class DefaultS3ClientFactory extends Configured private static final String REQUESTER_PAYS_HEADER_VALUE = "requester"; + private static final String S3_SERVICE_NAME = "s3"; + /** * Subclasses refer to this. */ protected static final Logger LOG = LoggerFactory.getLogger(DefaultS3ClientFactory.class); + /** + * A one-off warning of default region chains in use. + */ + private static final LogExactlyOnce WARN_OF_DEFAULT_REGION_CHAIN = + new LogExactlyOnce(LOG); + + /** + * Warning message printed when the SDK Region chain is in use. + */ + private static final String SDK_REGION_CHAIN_IN_USE = + "S3A filesystem client is using" + + " the SDK region resolution chain."; + /** Exactly once log to inform about ignoring the AWS-SDK Warnings for CSE. */ private static final LogExactlyOnce IGNORE_CSE_WARN = new LogExactlyOnce(LOG); @@ -138,15 +157,7 @@ private , ClientT> Build BuilderT builder, S3ClientCreationParameters parameters, Configuration conf, String bucket) throws IOException { - Region region = parameters.getRegion(); - LOG.debug("Using region {}", region); - - URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf); - - if (endpoint != null) { - builder.endpointOverride(endpoint); - LOG.debug("Using endpoint {}", endpoint); - } + configureEndpointAndRegion(builder, parameters, conf); S3Configuration serviceConfiguration = S3Configuration.builder() .pathStyleAccessEnabled(parameters.isPathStyleAccess()) @@ -155,7 +166,6 @@ private , ClientT> Build return builder .overrideConfiguration(createClientOverrideConfiguration(parameters, conf)) .credentialsProvider(parameters.getCredentialSet()) - .region(region) .serviceConfiguration(serviceConfiguration); } @@ -201,6 +211,72 @@ protected ClientOverrideConfiguration createClientOverrideConfiguration( return clientOverrideConfigBuilder.build(); } + /** + * This method configures the endpoint and region for a S3 client. + * The order of configuration is: + * + *
      + *
    1. If region is configured via fs.s3a.endpoint.region, use it.
      2. + *
    2. If endpoint is configured via via fs.s3a.endpoint, set it. + * If no region is configured, try to parse region from endpoint.
      3. + *
    3. If no region is configured, and it could not be parsed from the endpoint, + * set the default region as US_EAST_2 and enable cross region access.
      4. + *
    4. If configured region is empty, fallback to SDK resolution chain.
      5. + *
    + * + * @param builder S3 client builder. + * @param parameters parameter object + * @param conf conf configuration object + * @param S3 client builder type + * @param S3 client type + */ + private , ClientT> void configureEndpointAndRegion( + BuilderT builder, S3ClientCreationParameters parameters, Configuration conf) { + URI endpoint = getS3Endpoint(parameters.getEndpoint(), conf); + + String configuredRegion = parameters.getRegion(); + Region region = null; + String origin = ""; + + // If the region was configured, set it. + if (configuredRegion != null && !configuredRegion.isEmpty()) { + origin = AWS_REGION; + region = Region.of(configuredRegion); + } + + if (endpoint != null) { + builder.endpointOverride(endpoint); + // No region was configured, try to determine it from the endpoint. + if (region == null) { + region = getS3RegionFromEndpoint(parameters.getEndpoint()); + if (region != null) { + origin = "endpoint"; + } + } + LOG.debug("Setting endpoint to {}", endpoint); + } + + if (region != null) { + builder.region(region); + } else if (configuredRegion == null) { + // no region is configured, and none could be determined from the endpoint. + // Use US_EAST_2 as default. + region = Region.of(AWS_S3_DEFAULT_REGION); + builder.crossRegionAccessEnabled(true); + builder.region(region); + origin = "cross region access fallback"; + } else if (configuredRegion.isEmpty()) { + // region configuration was set to empty string. + // allow this if people really want it; it is OK to rely on this + // when deployed in EC2. + WARN_OF_DEFAULT_REGION_CHAIN.warn(SDK_REGION_CHAIN_IN_USE); + LOG.debug(SDK_REGION_CHAIN_IN_USE); + origin = "SDK region chain"; + } + + LOG.debug("Setting region to {} from {}", region, origin); + } + /** * Given a endpoint string, create the endpoint URI. * @@ -229,4 +305,23 @@ private static URI getS3Endpoint(String endpoint, final Configuration conf) { throw new IllegalArgumentException(e); } } + + /** + * Parses the endpoint to get the region. + * If endpoint is the central one, use US_EAST_1. + * + * @param endpoint the configure endpoint. + * @return the S3 region, null if unable to resolve from endpoint. + */ + private static Region getS3RegionFromEndpoint(String endpoint) { + + if(!endpoint.endsWith(CENTRAL_ENDPOINT)) { + LOG.debug("Endpoint {} is not the default; parsing", endpoint); + return AwsHostNameUtils.parseSigningRegion(endpoint, S3_SERVICE_NAME).orElse(null); + } + + // endpoint is for US_EAST_1; + return Region.US_EAST_1; + } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index a56f1db0b4556..8ab8d22cc6d84 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -34,7 +34,6 @@ import java.util.Collections; import java.util.Date; import java.util.EnumSet; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -54,7 +53,6 @@ import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; @@ -83,7 +81,6 @@ import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Error; -import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.SelectObjectContentRequest; import software.amazon.awssdk.services.s3.model.SelectObjectContentResponseHandler; @@ -98,7 +95,6 @@ import software.amazon.awssdk.transfer.s3.model.FileUpload; import software.amazon.awssdk.transfer.s3.model.UploadFileRequest; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -246,7 +242,6 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_301_MOVED_PERMANENTLY; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT; @@ -332,8 +327,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private int executorCapacity; private long multiPartThreshold; public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); - /** Exactly once log to warn about setting the region in config to avoid probe. */ - private static final LogExactlyOnce SET_REGION_WARNING = new LogExactlyOnce(LOG); /** Log to warn of storage class configuration problems. */ private static final LogExactlyOnce STORAGE_CLASS_WARNING = new LogExactlyOnce(LOG); @@ -461,8 +454,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private String scheme = FS_S3A; - private final static Map BUCKET_REGIONS = new HashMap<>(); - /** Add any deprecated keys. */ @SuppressWarnings("deprecation") private static void addDeprecatedKeys() { @@ -870,9 +861,6 @@ protected void verifyBucketExists() throws UnknownStoreException, IOException { STORE_EXISTS_PROBE, bucket, null, () -> invoker.retry("doesBucketExist", bucket, true, () -> { try { - if (BUCKET_REGIONS.containsKey(bucket)) { - return true; - } s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()); return true; } catch (AwsServiceException ex) { @@ -982,8 +970,6 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { ? conf.getTrimmed(AWS_REGION) : accessPoint.getRegion(); - Region region = getS3Region(configuredRegion); - S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(credentials) @@ -998,7 +984,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { .withMultipartCopyEnabled(isMultipartCopyEnabled) .withMultipartThreshold(multiPartThreshold) .withTransferManagerExecutor(unboundedThreadPool) - .withRegion(region); + .withRegion(configuredRegion); S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf); s3Client = clientFactory.createS3Client(getUri(), parameters); @@ -1019,75 +1005,6 @@ private void createS3AsyncClient(S3ClientFactory clientFactory, s3AsyncClient = clientFactory.createS3AsyncClient(getUri(), parameters); } - /** - * Get the bucket region. - * - * @param region AWS S3 Region set in the config. This property may not be set, in which case - * ask S3 for the region. - * @return region of the bucket. - */ - private Region getS3Region(String region) throws IOException { - - if (!StringUtils.isBlank(region)) { - return Region.of(region); - } - - Region cachedRegion = BUCKET_REGIONS.get(bucket); - - if (cachedRegion != null) { - LOG.debug("Got region {} for bucket {} from cache", cachedRegion, bucket); - return cachedRegion; - } - - Region s3Region = trackDurationAndSpan(STORE_REGION_PROBE, bucket, null, - () -> invoker.retry("getS3Region", bucket, true, () -> { - try { - - SET_REGION_WARNING.warn( - "Getting region for bucket {} from S3, this will slow down FS initialisation. " - + "To avoid this, set the region using property {}", bucket, - FS_S3A_BUCKET_PREFIX + bucket + ".endpoint.region"); - - // build a s3 client with region eu-west-1 that can be used to get the region of the - // bucket. Using eu-west-1, as headBucket() doesn't work with us-east-1. This is because - // us-east-1 uses the endpoint s3.amazonaws.com, which resolves bucket.s3.amazonaws.com - // to the actual region the bucket is in. As the request is signed with us-east-1 and - // not the bucket's region, it fails. - S3Client getRegionS3Client = - S3Client.builder().region(Region.EU_WEST_1).credentialsProvider(credentials) - .build(); - - HeadBucketResponse headBucketResponse = - getRegionS3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()); - - Region bucketRegion = Region.of( - headBucketResponse.sdkHttpResponse().headers().get(BUCKET_REGION_HEADER).get(0)); - BUCKET_REGIONS.put(bucket, bucketRegion); - - return bucketRegion; - } catch (S3Exception exception) { - if (exception.statusCode() == SC_301_MOVED_PERMANENTLY) { - Region bucketRegion = Region.of( - exception.awsErrorDetails().sdkHttpResponse().headers().get(BUCKET_REGION_HEADER) - .get(0)); - BUCKET_REGIONS.put(bucket, bucketRegion); - - return bucketRegion; - } - - if (exception.statusCode() == SC_404_NOT_FOUND) { - throw new UnknownStoreException("s3a://" + bucket + "/", - " Bucket does not exist: " + exception, - exception); - } - - throw exception; - } - })); - - return s3Region; - } - /** * Initialize and launch the audit manager and service. * As this takes the FS IOStatistics store, it must be invoked diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index e2e792ebfb668..451eab8e9ad58 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -28,7 +28,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.transfer.s3.S3TransferManager; @@ -169,7 +168,7 @@ final class S3ClientCreationParameters { /** * Region of the S3 bucket. */ - private Region region; + private String region; /** @@ -386,42 +385,42 @@ public S3ClientCreationParameters withTransferManagerExecutor( } /** - * Set region. + * Set the multipart flag.. * * @param value new value * @return the builder */ - public S3ClientCreationParameters withRegion( - final Region value) { - region = value; + public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) { + this.multipartCopy = value; return this; } /** - * Get the region. - * @return invoker + * Get the multipart flag. + * @return multipart flag */ - public Region getRegion() { - return region; + public boolean isMultipartCopy() { + return multipartCopy; } /** - * Set the multipart flag.. + * Set region. * * @param value new value * @return the builder */ - public S3ClientCreationParameters withMultipartCopyEnabled(final boolean value) { - this.multipartCopy = value; + public S3ClientCreationParameters withRegion( + final String value) { + region = value; return this; } /** - * Get the multipart flag. - * @return multipart flag + * Get the region. + * @return invoker */ - public boolean isMultipartCopy() { - return multipartCopy; + public String getRegion() { + return region; } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index f54113af6c5c1..f4e28aa62783e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -548,11 +548,6 @@ public enum Statistic { StoreStatisticNames.STORE_IO_THROTTLE_RATE, "Rate of S3 request throttling", TYPE_QUANTILE), - STORE_REGION_PROBE( - StoreStatisticNames.STORE_REGION_PROBE, - "Store Region Probe", - TYPE_DURATION - ), /* * Delegation Token Operations. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 330b92186dde1..22fc630dad1f5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -406,6 +406,13 @@ public int run(String[] args, PrintStream out) // Note and continue. LOG.debug("failed to get bucket location", e); println(out, LOCATION_UNKNOWN); + + // it may be the bucket is not found; we can't differentiate + // that and handle third party store issues where the API may + // not work. + // Fallback to looking for bucket root attributes. + println(out, "Probing for bucket existence"); + fs.listXAttrs(new Path("/")); } // print any auth paths for directory marker info diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 60037e58cd611..d9c847d4b2b0a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -59,6 +59,7 @@ import org.apache.http.HttpStatus; import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3ATestConstants.EU_WEST_1; import static org.apache.hadoop.fs.s3a.S3AUtils.*; @@ -366,6 +367,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() throws Exception { conf = new Configuration(); + skipIfCrossRegionClient(conf); conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true)); assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false)); @@ -404,6 +406,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() @Test public void testDefaultUserAgent() throws Exception { conf = new Configuration(); + skipIfCrossRegionClient(conf); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); S3Client s3 = getS3Client("User Agent"); @@ -417,6 +420,7 @@ public void testDefaultUserAgent() throws Exception { @Test public void testCustomUserAgent() throws Exception { conf = new Configuration(); + skipIfCrossRegionClient(conf); conf.set(Constants.USER_AGENT_PREFIX, "MyApp"); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); @@ -431,6 +435,7 @@ public void testCustomUserAgent() throws Exception { @Test public void testRequestTimeout() throws Exception { conf = new Configuration(); + skipIfCrossRegionClient(conf); conf.set(REQUEST_TIMEOUT, "120"); fs = S3ATestUtils.createTestFileSystem(conf); S3Client s3 = getS3Client("Request timeout (ms)"); @@ -610,4 +615,16 @@ public static boolean isSTSSignerCalled() { return stsSignerCalled; } } + + /** + * Skip a test if client created is cross region. + * @param configuration configuration to probe + */ + private static void skipIfCrossRegionClient( + Configuration configuration) { + if (configuration.get(ENDPOINT, null) == null + && configuration.get(AWS_REGION, null) == null) { + skip("Skipping test as cross region client is in use "); + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index eb010674beedf..5d10590dfe30f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -27,6 +27,7 @@ import java.util.List; import org.assertj.core.api.Assertions; +import org.junit.Ignore; import org.junit.Test; import software.amazon.awssdk.awscore.AwsExecutionAttribute; import software.amazon.awssdk.awscore.exception.AwsServiceException; @@ -42,8 +43,9 @@ import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; -import static org.apache.hadoop.fs.s3a.Statistic.STORE_REGION_PROBE; import static org.apache.hadoop.io.IOUtils.closeStream; +import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT; + import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -54,9 +56,17 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase { private static final String AWS_ENDPOINT_TEST = "test-endpoint"; - private static final String USW_2_BUCKET = "landsat-pds"; + private static final String US_EAST_1 = "us-east-1"; + + private static final String US_EAST_2 = "us-east-2"; + + private static final String US_WEST_2 = "us-west-2"; + + private static final String EU_WEST_2 = "eu-west-2"; - public static final String USW_2_STORE = "s3a://" + USW_2_BUCKET; + private static final String CN_NORTHWEST_1 = "cn-northwest-1"; + + private static final String US_GOV_EAST_1 = "us-gov-east-1"; /** * If anyone were ever to create a bucket with this UUID pair it would break the tests. @@ -64,6 +74,14 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase { public static final String UNKNOWN_BUCKET = "23FA76D4-5F17-48B8-9D7D-9050269D0E40" + "-8281BAF2-DBCF-47AA-8A27-F2FA3589656A"; + private static final String EU_WEST_2_ENDPOINT = "s3.eu-west-2.amazonaws.com"; + + private static final String CN_ENDPOINT = "s3.cn-northwest-1.amazonaws.com.cn"; + + private static final String GOV_ENDPOINT = "s3-fips.us-gov-east-1.amazonaws.com"; + + private static final String VPC_ENDPOINT = "vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com"; + /** * New FS instance which will be closed in teardown. */ @@ -75,11 +93,6 @@ public void teardown() throws Exception { super.teardown(); } - /** - * Test to verify that not setting the region config, will lead to the client factory making - * a HEAD bucket call to configure the correct region. If an incorrect region is set, the HEAD - * bucket call in this test will raise an exception. - */ @Test public void testWithoutRegionConfig() throws IOException { describe("Verify that region lookup takes place"); @@ -96,7 +109,6 @@ public void testWithoutRegionConfig() throws IOException { } catch (UnknownHostException | UnknownStoreException | AccessDeniedException allowed) { // these are all valid failure modes from different test environments. } - assertRegionProbeCount(1); } @Test @@ -115,82 +127,128 @@ public void testUnknownBucket() throws Exception { } catch (UnknownHostException | UnknownStoreException expected) { // this is good. } - assertRegionProbeCount(1); } + @Test + public void testEndpointOverride() throws Throwable { + describe("Create a client with a configured endpoint"); + Configuration conf = getConfiguration(); + + S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST, null, US_EAST_2); + + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); + } @Test - public void testWithRegionConfig() throws IOException, URISyntaxException { - describe("Verify that region lookup is skipped if the region property is set"); + public void testCentralEndpoint() throws Throwable { + describe("Create a client with the central endpoint"); Configuration conf = getConfiguration(); - removeBaseAndBucketOverrides(conf, AWS_REGION, PATH_STYLE_ACCESS); - conf.set(AWS_REGION, "us-east-2"); + S3Client client = createS3Client(conf, CENTRAL_ENDPOINT, null, US_EAST_1); - newFS = new S3AFileSystem(); - newFS.initialize(new URI(USW_2_STORE), conf); - assertRegionProbeCount(0); + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); } @Test - public void testRegionCache() throws IOException, URISyntaxException { - describe("Verify that region lookup is cached on the second attempt"); + public void testWithRegionConfig() throws Throwable { + describe("Create a client with a configured region"); Configuration conf = getConfiguration(); - removeBaseAndBucketOverrides(USW_2_BUCKET, conf, AWS_REGION, PATH_STYLE_ACCESS); - newFS = new S3AFileSystem(); + S3Client client = createS3Client(conf, null, EU_WEST_2, EU_WEST_2); - newFS.initialize(new URI(USW_2_STORE), conf); + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); + } - assertRegionProbeCount(1); - closeStream(newFS); - // create a new instance - newFS = new S3AFileSystem(); - newFS.initialize(new URI(USW_2_STORE), conf); + public void testEUWest2Endpoint() throws Throwable { + describe("Create a client with the eu west 2 endpoint"); + Configuration conf = getConfiguration(); + + S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, null, EU_WEST_2); - // value should already be cached. - assertRegionProbeCount(0); + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); } - private void assertRegionProbeCount(final int expected) { - Assertions.assertThat(newFS.getInstrumentation().getCounterValue(STORE_REGION_PROBE)) - .describedAs("Incorrect number of calls made to get bucket region").isEqualTo(expected); + @Test + public void testWithRegionAndEndpointConfig() throws Throwable { + describe("Test that when both region and endpoint are configured, region takes precedence"); + Configuration conf = getConfiguration(); + + S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, US_WEST_2, US_WEST_2); + + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); } @Test - public void testEndpointOverride() throws Throwable { - describe("Create a client with no region and the default endpoint"); + public void testWithChinaEndpoint() throws Throwable { + describe("Test with a china endpoint"); Configuration conf = getConfiguration(); - S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST); + S3Client client = createS3Client(conf, CN_ENDPOINT, null, CN_NORTHWEST_1); intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); } + @Test + public void testWithGovCloudEndpoint() throws Throwable { + describe("Test with a gov cloud endpoint"); + Configuration conf = getConfiguration(); + + S3Client client = createS3Client(conf, GOV_ENDPOINT, null, US_GOV_EAST_1); + + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); + } + + @Test + @Ignore("Pending HADOOP-18938. S3A region logic to handle vpce and non standard endpoints") + public void testWithVPCE() throws Throwable { + describe("Test with vpc endpoint"); + Configuration conf = getConfiguration(); + + S3Client client = createS3Client(conf, VPC_ENDPOINT, null, US_WEST_2); + + intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket( + HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build())); + } class RegionInterceptor implements ExecutionInterceptor { - private boolean endpointOverridden; + private String endpoint; + private String region; - RegionInterceptor(boolean endpointOverridden) { - this.endpointOverridden = endpointOverridden; + RegionInterceptor(String endpoint, String region) { + this.endpoint = endpoint; + this.region = region; } @Override public void beforeExecution(Context.BeforeExecution context, ExecutionAttributes executionAttributes) { - if (endpointOverridden) { + if (endpoint != null) { Assertions.assertThat( executionAttributes.getAttribute(AwsExecutionAttribute.ENDPOINT_OVERRIDDEN)) .describedAs("Endpoint not overridden").isTrue(); Assertions.assertThat( executionAttributes.getAttribute(AwsExecutionAttribute.CLIENT_ENDPOINT).toString()) - .describedAs("There is an endpoint mismatch").isEqualTo("https://" + AWS_ENDPOINT_TEST); + .describedAs("There is an endpoint mismatch").isEqualTo("https://" + endpoint); + } else { + Assertions.assertThat( + executionAttributes.getAttribute(AwsExecutionAttribute.ENDPOINT_OVERRIDDEN)) + .describedAs("Endpoint is overridden").isEqualTo(null); } + Assertions.assertThat( + executionAttributes.getAttribute(AwsExecutionAttribute.AWS_REGION).toString()) + .describedAs("Incorrect region set").isEqualTo(region); + // We don't actually want to make a request, so exit early. throw AwsServiceException.builder().message("Exception thrown by interceptor").build(); } @@ -202,23 +260,18 @@ public void beforeExecution(Context.BeforeExecution context, * value. * @param conf configuration to use. * @param endpoint endpoint. + * @param expectedRegion the region that should be set in the client. * @return the client. * @throws URISyntaxException parse problems. * @throws IOException IO problems */ @SuppressWarnings("deprecation") private S3Client createS3Client(Configuration conf, - String endpoint) + String endpoint, String configuredRegion, String expectedRegion) throws IOException { - boolean endpointOverridden = false; - - if (endpoint != null && !endpoint.isEmpty()) { - endpointOverridden = true; - } - List interceptors = new ArrayList<>(); - interceptors.add(new RegionInterceptor(endpointOverridden)); + interceptors.add(new RegionInterceptor(endpoint, expectedRegion)); DefaultS3ClientFactory factory = new DefaultS3ClientFactory(); @@ -229,7 +282,9 @@ private S3Client createS3Client(Configuration conf, .withEndpoint(endpoint) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) - .withExecutionInterceptors(interceptors); + .withExecutionInterceptors(interceptors) + .withRegion(configuredRegion); + S3Client client = factory.createS3Client( getFileSystem().getUri(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java index ebad90336f7d0..e85e33c844d52 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java @@ -26,7 +26,6 @@ import java.net.URI; import java.nio.file.AccessDeniedException; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadBucketResponse; import org.junit.AfterClass; @@ -596,8 +595,8 @@ protected HeadBucketResponse readLandsatMetadata(final S3AFileSystem delegatedFS .withPathUri(new URI("s3a://localhost/")) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) - .withUserAgentSuffix("ITestSessionDelegationInFilesystem") - .withRegion(Region.US_WEST_2); + .withUserAgentSuffix("ITestSessionDelegationInFilesystem"); + S3Client s3 = factory.createS3Client(landsat, parameters); return Invoker.once("HEAD", host, diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 12379fe6c8a6b..e6c3eeed3d7e9 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -31,11 +31,10 @@     - - fs.s3a.bucket.landsat-pds.endpoint - ${central.endpoint} - The endpoint for s3a://landsat-pds URLs + fs.s3a.bucket.landsat-pds.endpoint.region + us-west-2 + The region for s3a://landsat-pds @@ -58,10 +57,9 @@ - - fs.s3a.bucket.usgs-landsat.endpoint - ${central.endpoint} + fs.s3a.bucket.usgs-landsat.endpoint.region + us-west-2 @@ -82,6 +80,12 @@ false + + + fs.s3a.bucket.osm-pds.endpoint.region + us-east-1 + The region for s3a://osm-pds + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index f7303fb0f5e1a..b07ba76e8eec0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -149,6 +149,10 @@ public void initializeMemberVariables() { xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess"); xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix"); + // Properties in enable callqueue overflow trigger failover for stateless servers. + xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.overflow.trigger.failover"); + xmlPropsToSkipCompare.add("ipc.callqueue.overflow.trigger.failover"); + // FairCallQueue configs that includes dynamic ports in its keys xmlPropsToSkipCompare.add("ipc.[port_number].backoff.enable"); xmlPropsToSkipCompare.add("ipc.backoff.enable"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java index 4a60520a364b6..545ddb40ff5fe 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ipc; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -517,4 +518,29 @@ public void testCallQueueOverflowExceptions() throws Exception { verify(queue, times(0)).put(call); verify(queue, times(0)).add(call); } + + @Test + public void testCallQueueOverEnabled() { + // default ipc.callqueue.overflow.trigger.failover' configure false. + String ns = "ipc.8888"; + conf.setBoolean("ipc.callqueue.overflow.trigger.failover", false); + manager = new CallQueueManager<>(fcqueueClass, rpcSchedulerClass, false, + 10, ns, conf); + assertFalse(manager.isServerFailOverEnabled()); + assertFalse(manager.isServerFailOverEnabledByQueue()); + + // set ipc.8888.callqueue.overflow.trigger.failover configure true. + conf.setBoolean("ipc.8888.callqueue.overflow.trigger.failover", true); + manager = new CallQueueManager<>(fcqueueClass, rpcSchedulerClass, false, + 10, ns, conf); + assertTrue(manager.isServerFailOverEnabled()); + assertTrue(manager.isServerFailOverEnabledByQueue()); + + // set ipc.callqueue.overflow.trigger.failover' configure true. + conf.setBoolean("ipc.callqueue.overflow.trigger.failover", true); + manager = new CallQueueManager<>(fcqueueClass, rpcSchedulerClass, false, + 10, ns, conf); + assertTrue(manager.isServerFailOverEnabled()); + assertTrue(manager.isServerFailOverEnabledByQueue()); + } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java index 1fed9a317642a..06b65dc4df3c5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java @@ -28,7 +28,6 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.times; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -105,7 +104,7 @@ public void testTotalCapacityOfSubQueues() { fairCallQueue = new FairCallQueue(7, 1025, "ns", conf); assertThat(fairCallQueue.remainingCapacity()).isEqualTo(1025); fairCallQueue = new FairCallQueue(7, 1025, "ns", - new int[]{7, 6, 5, 4, 3, 2, 1}, conf); + new int[]{7, 6, 5, 4, 3, 2, 1}, false, conf); assertThat(fairCallQueue.remainingCapacity()).isEqualTo(1025); } @@ -170,7 +169,7 @@ public void testQueueCapacity() { // default weights i.e. all queues share capacity fcq = new FairCallQueue(numQueues, 4, "ns", conf); FairCallQueue fcq1 = new FairCallQueue( - numQueues, capacity, "ns", new int[]{1, 3}, conf); + numQueues, capacity, "ns", new int[]{1, 3}, false, conf); for (int i=0; i < capacity; i++) { Schedulable call = mockCall("u", i%2); @@ -221,11 +220,10 @@ public void testInsertionWithFailover() { Configuration conf = new Configuration(); // Config for server to throw StandbyException instead of the // regular RetriableException if call queue is full. - conf.setBoolean( - "ns." + CommonConfigurationKeys.IPC_CALLQUEUE_SERVER_FAILOVER_ENABLE, - true); + // 3 queues, 2 slots each. - fcq = Mockito.spy(new FairCallQueue<>(3, 6, "ns", conf)); + fcq = Mockito.spy(new FairCallQueue<>(3, 6, "ns", + true, conf)); Schedulable p0 = mockCall("a", 0); Schedulable p1 = mockCall("b", 1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java index e21a5a307308a..873a524c988c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java @@ -88,7 +88,7 @@ public void tearDown() throws IOException { @SuppressWarnings("serial") public static class MockCallQueue extends LinkedBlockingQueue { public MockCallQueue(int levels, int cap, String ns, int[] capacityWeights, - Configuration conf) { + boolean serverFailOverEnabled, Configuration conf) { super(cap); mockQueueConstructions++; } @@ -172,6 +172,6 @@ public void testRefreshCallQueueWithFairCallQueue() throws Exception { // check callQueueSize has changed assertEquals(150 * serviceHandlerCount, rpcServer.getClientRpcServer() .getMaxQueueSize()); - } + } } \ No newline at end of file From 80a22a736ee886d78ef809bc06aa27f1dff837cb Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:28:05 +0800 Subject: [PATCH 114/155] YARN-11500. [Addendum] Fix typos in hadoop-yarn-server-common#federation. (#6212) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../policies/AbstractConfigurableFederationPolicy.java | 2 +- .../FederationPolicyInitializationContextValidator.java | 8 ++++---- .../policies/amrmproxy/BroadcastAMRMProxyPolicy.java | 2 +- .../amrmproxy/LocalityMulticastAMRMProxyPolicy.java | 2 +- .../policies/manager/AbstractPolicyManager.java | 8 ++++---- .../policies/manager/FederationPolicyManager.java | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java index 7234d46b61261..c70b7b5eb5435 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java @@ -53,7 +53,7 @@ public void reinitialize( initializationContext.getSubClusterPolicyConfiguration().getParams()); // if nothing has changed skip the rest of initialization - // and signal to childs that the reinit is free via isDirty var. + // and signal to children that the reinit is free via isDirty var. if (policyInfo != null && policyInfo.equals(newPolicyInfo)) { isDirty = false; return; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java index da63bc1de468c..1d430751036af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java @@ -40,25 +40,25 @@ public static void validate( if (policyContext == null) { throw new FederationPolicyInitializationException( "The FederationPolicyInitializationContext provided is null. Cannot" - + " reinitalize " + "successfully."); + + " reinitialize " + "successfully."); } if (policyContext.getFederationStateStoreFacade() == null) { throw new FederationPolicyInitializationException( "The FederationStateStoreFacade provided is null. Cannot" - + " reinitalize successfully."); + + " reinitialize successfully."); } if (policyContext.getFederationSubclusterResolver() == null) { throw new FederationPolicyInitializationException( "The FederationSubclusterResolver provided is null. Cannot" - + " reinitalize successfully."); + + " reinitialize successfully."); } if (policyContext.getSubClusterPolicyConfiguration() == null) { throw new FederationPolicyInitializationException( "The SubClusterPolicyConfiguration provided is null. Cannot " - + "reinitalize successfully."); + + "reinitialize successfully."); } String intendedType = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java index 643bfa6da0117..36074f989fd1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java @@ -41,7 +41,7 @@ public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy { public void reinitialize( FederationPolicyInitializationContext policyContext) throws FederationPolicyInitializationException { - // overrides initialize to avoid weight checks that do no apply for + // overrides initialize to avoid weight checks that do not apply for // this policy. FederationPolicyInitializationContextValidator .validate(policyContext, this.getClass().getCanonicalName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java index a98ec138f604a..2cd1eacaa665f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java @@ -394,7 +394,7 @@ private void splitAnyRequests(List originalResourceRequests, targetSubclusters = allocationBookkeeper.getActiveAndEnabledSC(); } - // SECOND: pick how much to ask to each RM for each request + // SECOND: pick how much to ask each RM for each request splitIndividualAny(resourceRequest, targetSubclusters, allocationBookkeeper); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java index f7a89c614feaf..aa0742d090c2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java @@ -53,10 +53,10 @@ public abstract class AbstractPolicyManager implements * @param federationPolicyContext the current context * @param oldInstance the existing (possibly null) instance. * - * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy} + * @return a valid and fully reinitialized {@link FederationAMRMProxyPolicy} * instance * - * @throws FederationPolicyInitializationException if the reinitalization is + * @throws FederationPolicyInitializationException if the reinitialization is * not valid, and ensure * previous state is preserved */ @@ -89,10 +89,10 @@ public FederationAMRMProxyPolicy getAMRMPolicy( * @param federationPolicyContext the current context * @param oldInstance the existing (possibly null) instance. * - * @return a valid and fully reinitalized {@link FederationRouterPolicy} + * @return a valid and fully reinitialized {@link FederationRouterPolicy} * instance * - * @throws FederationPolicyInitializationException if the reinitalization is + * @throws FederationPolicyInitializationException if the reinitialization is * not valid, and ensure * previous state is preserved */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java index 23f7cf3ae38bc..3aeb7d718e2d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java @@ -48,7 +48,7 @@ public interface FederationPolicyManager { * instance of {@link FederationAMRMProxyPolicy} reinitialized with the * current context, otherwise a new instance initialized with the current * context is provided. If the instance is compatible with the current class - * the implementors should attempt to reinitalize (retaining state). To affect + * the implementors should attempt to reinitialize (retaining state). To affect * a complete policy reset oldInstance should be null. * * @param policyContext the current context @@ -70,7 +70,7 @@ FederationAMRMProxyPolicy getAMRMPolicy( * instance of {@link FederationRouterPolicy} reinitialized with the current * context, otherwise a new instance initialized with the current context is * provided. If the instance is compatible with the current class the - * implementors should attempt to reinitalize (retaining state). To affect a + * implementors should attempt to reinitialize (retaining state). To affect a * complete policy reset oldInstance should be set to null. * * @param policyContext the current context From 9c7e5b66fa9af9c750df640fca9680d97d7ecdc5 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:36:06 +0800 Subject: [PATCH 115/155] YARN-11576. Improve FederationInterceptorREST AuditLog. (#6117) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../yarn/server/router/RouterAuditLogger.java | 27 ++ .../webapp/FederationInterceptorREST.java | 437 +++++++++++++++++- 2 files changed, 460 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java index bb814b652831a..b0902a5e5c14e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java @@ -50,6 +50,7 @@ public static class AuditConstants { public static final String FORCE_KILL_APP = "Force Kill App"; public static final String GET_APP_REPORT = "Get Application Report"; public static final String TARGET_CLIENT_RM_SERVICE = "RouterClientRMService"; + public static final String TARGET_WEB_SERVICE = "RouterWebServices"; public static final String UNKNOWN = "UNKNOWN"; public static final String GET_APPLICATIONS = "Get Applications"; public static final String GET_CLUSTERMETRICS = "Get ClusterMetrics"; @@ -82,6 +83,32 @@ public static class AuditConstants { public static final String GET_ATTRIBUTESTONODES = "Get AttributesToNodes"; public static final String GET_CLUSTERNODEATTRIBUTES = "Get ClusterNodeAttributes"; public static final String GET_NODESTOATTRIBUTES = "Get NodesToAttributes"; + public static final String GET_CLUSTERINFO = "Get ClusterInfo"; + public static final String GET_CLUSTERUSERINFO = "Get ClusterUserInfo"; + public static final String GET_SCHEDULERINFO = "Get SchedulerInfo"; + public static final String DUMP_SCHEDULERLOGS = "Dump SchedulerLogs"; + public static final String GET_ACTIVITIES = "Get Activities"; + public static final String GET_BULKACTIVITIES = "Get BulkActivities"; + public static final String GET_APPACTIVITIES = "Get AppActivities"; + public static final String GET_APPSTATISTICS = "Get AppStatistics"; + public static final String GET_RMNODELABELS = "Get RMNodeLabels"; + public static final String REPLACE_LABELSONNODES = "Replace LabelsOnNodes"; + public static final String REPLACE_LABELSONNODE = "Replace LabelsOnNode"; + public static final String GET_CLUSTER_NODELABELS = "Get ClusterNodeLabels"; + public static final String ADD_TO_CLUSTER_NODELABELS = "Add To ClusterNodeLabels"; + public static final String REMOVE_FROM_CLUSTERNODELABELS = "Remove From ClusterNodeLabels"; + public static final String GET_LABELS_ON_NODE = "Get LabelsOnNode"; + public static final String GET_APP_PRIORITY = "Get AppPriority"; + public static final String UPDATE_APP_QUEUE = "Update AppQueue"; + public static final String POST_DELEGATION_TOKEN = "Post DelegationToken"; + public static final String POST_DELEGATION_TOKEN_EXPIRATION = "Post DelegationTokenExpiration"; + public static final String GET_APP_TIMEOUT = "Get App Timeout"; + public static final String GET_APP_TIMEOUTS = "Get App Timeouts"; + public static final String CHECK_USER_ACCESS_TO_QUEUE = "Check User AccessToQueue"; + public static final String GET_APP_ATTEMPT = "Get AppAttempt"; + public static final String GET_CONTAINER = "Get Container"; + public static final String UPDATE_SCHEDULER_CONFIGURATION = "Update SchedulerConfiguration"; + public static final String GET_SCHEDULER_CONFIGURATION = "Get SchedulerConfiguration"; } public static void logSuccess(String user, String operation, String target) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index 71268ac672d98..7f9446878b3e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -120,6 +120,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntry; +import org.apache.hadoop.yarn.server.router.RouterAuditLogger; import org.apache.hadoop.yarn.server.router.RouterMetrics; import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; @@ -151,6 +152,50 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import static org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade.getRandomActiveSubCluster; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_NEW_APP; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_CLUSTERINFO; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_CLUSTERUSERINFO; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_SCHEDULERINFO; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.DUMP_SCHEDULERLOGS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_ACTIVITIES; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_BULKACTIVITIES; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APPACTIVITIES; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APPSTATISTICS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_NODETOLABELS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_RMNODELABELS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_LABELSTONODES; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UNKNOWN; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.TARGET_WEB_SERVICE; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.REPLACE_LABELSONNODES; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.REPLACE_LABELSONNODE; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_CLUSTER_NODELABELS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.ADD_TO_CLUSTER_NODELABELS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.REMOVE_FROM_CLUSTERNODELABELS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_LABELS_ON_NODE; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APP_PRIORITY; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_QUEUEINFO; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UPDATE_APPLICATIONPRIORITY; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UPDATE_APP_QUEUE; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.POST_DELEGATION_TOKEN; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.POST_DELEGATION_TOKEN_EXPIRATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.CANCEL_DELEGATIONTOKEN; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_NEW_RESERVATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.SUBMIT_RESERVATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UPDATE_RESERVATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.DELETE_RESERVATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.LIST_RESERVATIONS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APP_TIMEOUT; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APP_TIMEOUTS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UPDATE_APPLICATIONTIMEOUTS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APPLICATION_ATTEMPTS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.CHECK_USER_ACCESS_TO_QUEUE; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APP_ATTEMPT; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_CONTAINERS; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_CONTAINER; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UPDATE_SCHEDULER_CONFIGURATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_SCHEDULER_CONFIGURATION; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.SIGNAL_TOCONTAINER; import static org.apache.hadoop.yarn.server.router.webapp.RouterWebServiceUtil.extractToken; import static org.apache.hadoop.yarn.server.router.webapp.RouterWebServiceUtil.getKerberosUserGroupInformation; @@ -360,9 +405,13 @@ public Response createNewApplication(HttpServletRequest hsr) } catch (FederationPolicyException e) { // If a FederationPolicyException is thrown, the service is unavailable. routerMetrics.incrAppsFailedCreated(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getLocalizedMessage()).build(); } catch (Exception e) { routerMetrics.incrAppsFailedCreated(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR).entity(e.getLocalizedMessage()).build(); } @@ -370,6 +419,8 @@ public Response createNewApplication(HttpServletRequest hsr) String errMsg = "Fail to create a new application."; LOG.error(errMsg); routerMetrics.incrAppsFailedCreated(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, errMsg); return Response.status(Status.INTERNAL_SERVER_ERROR).entity(errMsg).build(); } @@ -400,6 +451,9 @@ private Response invokeGetNewApplication(Map subCl try { Response response = interceptor.createNewApplication(hsr); if (response != null && response.getStatus() == HttpServletResponse.SC_OK) { + ApplicationId applicationId = ApplicationId.fromString(response.getEntity().toString()); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_NEW_APP, + TARGET_WEB_SERVICE, applicationId, subClusterId); return response; } } catch (Exception e) { @@ -490,6 +544,8 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, HttpS routerMetrics.incrAppsFailedSubmitted(); String errMsg = "Missing ApplicationSubmissionContextInfo or " + "applicationSubmissionContext information."; + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, errMsg); return Response.status(Status.BAD_REQUEST).entity(errMsg).build(); } @@ -498,6 +554,8 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, HttpS RouterServerUtil.validateApplicationId(applicationId); } catch (IllegalArgumentException e) { routerMetrics.incrAppsFailedSubmitted(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, e.getMessage()); return Response.status(Status.BAD_REQUEST).entity(e.getLocalizedMessage()).build(); } @@ -515,6 +573,8 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, HttpS } } catch (Exception e) { routerMetrics.incrAppsFailedSubmitted(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, e.getMessage()); return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getLocalizedMessage()).build(); } @@ -522,6 +582,8 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, HttpS String errMsg = String.format("Application %s with appId %s failed to be submitted.", newApp.getApplicationName(), newApp.getApplicationId()); LOG.error(errMsg); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, errMsg); return Response.status(Status.SERVICE_UNAVAILABLE).entity(errMsg).build(); } @@ -577,11 +639,15 @@ private Response invokeSubmitApplication(ApplicationSubmissionContextInfo submis if (response != null && response.getStatus() == HttpServletResponse.SC_ACCEPTED) { LOG.info("Application {} with appId {} submitted on {}.", context.getApplicationName(), applicationId, subClusterId); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), SUBMIT_NEW_APP, + TARGET_WEB_SERVICE, applicationId, subClusterId); return response; } String msg = String.format("application %s failed to be submitted.", applicationId); throw new YarnException(msg); } catch (Exception e) { + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_WEB_SERVICE, e.getMessage(), applicationId, subClusterId); LOG.warn("Unable to submit the application {} to SubCluster {}.", applicationId, subClusterId, e); if (subClusterId != null) { @@ -1103,16 +1169,24 @@ public ClusterInfo getClusterInfo() { federationClusterInfo.getList().add(clusterInfo); }); long stopTime = Time.now(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_CLUSTERINFO, + TARGET_WEB_SERVICE); routerMetrics.succeededGetClusterInfoRetrieved(stopTime - startTime); return federationClusterInfo; } catch (NotFoundException e) { routerMetrics.incrGetClusterInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTERINFO, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("Get all active sub cluster(s) error.", e); } catch (YarnException | IOException e) { routerMetrics.incrGetClusterInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTERINFO, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getClusterInfo error.", e); } routerMetrics.incrGetClusterInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTERINFO, UNKNOWN, + TARGET_WEB_SERVICE, "getClusterInfo error."); throw new RuntimeException("getClusterInfo error."); } @@ -1144,16 +1218,24 @@ public ClusterUserInfo getClusterUserInfo(HttpServletRequest hsr) { federationClusterUserInfo.getList().add(clusterUserInfo); }); long stopTime = Time.now(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_CLUSTERUSERINFO, + TARGET_WEB_SERVICE); routerMetrics.succeededGetClusterUserInfoRetrieved(stopTime - startTime); return federationClusterUserInfo; } catch (NotFoundException e) { routerMetrics.incrGetClusterUserInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTERUSERINFO, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("Get all active sub cluster(s) error.", e); } catch (YarnException | IOException e) { routerMetrics.incrGetClusterUserInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTERUSERINFO, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getClusterUserInfo error.", e); } routerMetrics.incrGetClusterUserInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTERUSERINFO, UNKNOWN, + TARGET_WEB_SERVICE, "getClusterUserInfo error."); throw new RuntimeException("getClusterUserInfo error."); } @@ -1182,16 +1264,24 @@ public SchedulerTypeInfo getSchedulerInfo() { federationSchedulerTypeInfo.getList().add(schedulerTypeInfo); }); long stopTime = Time.now(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_SCHEDULERINFO, + TARGET_WEB_SERVICE); routerMetrics.succeededGetSchedulerInfoRetrieved(stopTime - startTime); return federationSchedulerTypeInfo; } catch (NotFoundException e) { routerMetrics.incrGetSchedulerInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_SCHEDULERINFO, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("Get all active sub cluster(s) error.", e); } catch (YarnException | IOException e) { routerMetrics.incrGetSchedulerInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_SCHEDULERINFO, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getSchedulerInfo error.", e); } routerMetrics.incrGetSchedulerInfoFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_SCHEDULERINFO, UNKNOWN, + TARGET_WEB_SERVICE, "getSchedulerInfo error."); throw new RuntimeException("getSchedulerInfo error."); } @@ -1213,6 +1303,8 @@ public String dumpSchedulerLogs(String time, HttpServletRequest hsr) if (StringUtils.isBlank(time)) { routerMetrics.incrDumpSchedulerLogsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, UNKNOWN, + TARGET_WEB_SERVICE, "Parameter error, the time is empty or null."); throw new IllegalArgumentException("Parameter error, the time is empty or null."); } @@ -1223,9 +1315,13 @@ public String dumpSchedulerLogs(String time, HttpServletRequest hsr) } } catch (NumberFormatException e) { routerMetrics.incrDumpSchedulerLogsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw new IllegalArgumentException("time must be a number."); } catch (IllegalArgumentException e) { routerMetrics.incrDumpSchedulerLogsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -1246,19 +1342,27 @@ public String dumpSchedulerLogs(String time, HttpServletRequest hsr) .append(subClusterId).append(" : ").append(msg).append("; "); }); long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, + TARGET_WEB_SERVICE); routerMetrics.succeededDumpSchedulerLogsRetrieved(stopTime - startTime); return stringBuilder.toString(); } catch (IllegalArgumentException e) { routerMetrics.incrDumpSchedulerLogsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to dump SchedulerLogs by time: %s.", time); } catch (YarnException e) { routerMetrics.incrDumpSchedulerLogsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "dumpSchedulerLogs by time = %s error .", time); } routerMetrics.incrDumpSchedulerLogsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DUMP_SCHEDULERLOGS, UNKNOWN, + TARGET_WEB_SERVICE, "dumpSchedulerLogs Failed."); throw new RuntimeException("dumpSchedulerLogs Failed."); } @@ -1290,14 +1394,19 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, ActivitiesInfo activitiesInfo = interceptor.getActivities(hsrCopy, nodeId, groupBy); if (activitiesInfo != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_ACTIVITIES, + TARGET_WEB_SERVICE); routerMetrics.succeededGetActivitiesLatencyRetrieved(stopTime - startTime); return activitiesInfo; } } catch (IllegalArgumentException | NotFoundException e) { routerMetrics.incrGetActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_ACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } - + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_ACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, "getActivities Failed."); routerMetrics.incrGetActivitiesFailedRetrieved(); throw new RuntimeException("getActivities Failed."); } @@ -1337,28 +1446,40 @@ public BulkActivitiesInfo getBulkActivities(HttpServletRequest hsr, bulkActivitiesInfo.setSubClusterId(subClusterId.getId()); fedBulkActivitiesInfo.getList().add(bulkActivitiesInfo); }); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_BULKACTIVITIES, + TARGET_WEB_SERVICE); long stopTime = clock.getTime(); routerMetrics.succeededGetBulkActivitiesRetrieved(stopTime - startTime); return fedBulkActivitiesInfo; } catch (IllegalArgumentException e) { routerMetrics.incrGetBulkActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_BULKACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } catch (NotFoundException e) { routerMetrics.incrGetBulkActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_BULKACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("get all active sub cluster(s) error.", e); } catch (IOException e) { routerMetrics.incrGetBulkActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_BULKACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getBulkActivities by groupBy = %s, activitiesCount = %s with io error.", groupBy, String.valueOf(activitiesCount)); } catch (YarnException e) { routerMetrics.incrGetBulkActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_BULKACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getBulkActivities by groupBy = %s, activitiesCount = %s with yarn error.", groupBy, String.valueOf(activitiesCount)); } routerMetrics.incrGetBulkActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_BULKACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, "getBulkActivities Failed."); throw new RuntimeException("getBulkActivities Failed."); } @@ -1377,17 +1498,25 @@ public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, if (appActivitiesInfo != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetAppActivitiesRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APPACTIVITIES, + TARGET_WEB_SERVICE); return appActivitiesInfo; } } catch (IllegalArgumentException e) { routerMetrics.incrGetAppActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get subCluster by appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrGetAppActivitiesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getAppActivities by appId = %s error .", appId); } + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPACTIVITIES, UNKNOWN, + TARGET_WEB_SERVICE, "getAppActivities Failed."); routerMetrics.incrGetAppActivitiesFailedRetrieved(); throw new RuntimeException("getAppActivities Failed."); } @@ -1409,23 +1538,33 @@ public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, if (applicationStatisticsInfo != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetAppStatisticsRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APPSTATISTICS, + TARGET_WEB_SERVICE); return applicationStatisticsInfo; } } catch (NotFoundException e) { routerMetrics.incrGetAppStatisticsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPSTATISTICS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("get all active sub cluster(s) error.", e); } catch (IOException e) { routerMetrics.incrGetAppStatisticsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPSTATISTICS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getAppStatistics error by stateQueries = %s, typeQueries = %s with io error.", StringUtils.join(stateQueries, ","), StringUtils.join(typeQueries, ",")); } catch (YarnException e) { routerMetrics.incrGetAppStatisticsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPSTATISTICS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getAppStatistics by stateQueries = %s, typeQueries = %s with yarn error.", StringUtils.join(stateQueries, ","), StringUtils.join(typeQueries, ",")); } routerMetrics.incrGetAppStatisticsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPSTATISTICS, UNKNOWN, + TARGET_WEB_SERVICE, "getAppStatistics Failed."); throw RouterServerUtil.logAndReturnRunTimeException( "getAppStatistics by stateQueries = %s, typeQueries = %s Failed.", StringUtils.join(stateQueries, ","), StringUtils.join(typeQueries, ",")); @@ -1448,16 +1587,24 @@ public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) if (nodeToLabelsInfo != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetNodeToLabelsRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_NODETOLABELS, + TARGET_WEB_SERVICE); return nodeToLabelsInfo; } } catch (NotFoundException e) { routerMetrics.incrNodeToLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NODETOLABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrNodeToLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NODETOLABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("getNodeToLabels error.", e); } routerMetrics.incrNodeToLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NODETOLABELS, UNKNOWN, + TARGET_WEB_SERVICE, "getNodeToLabels Failed."); throw new RuntimeException("getNodeToLabels Failed."); } @@ -1477,16 +1624,24 @@ public NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) throws IOException if (nodeToLabelsInfo != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetRMNodeLabelsRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_RMNODELABELS, + TARGET_WEB_SERVICE); return nodeToLabelsInfo; } } catch (NotFoundException e) { routerMetrics.incrGetRMNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_RMNODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrGetRMNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_RMNODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("getRMNodeLabels error.", e); } routerMetrics.incrGetRMNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_RMNODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, "getRMNodeLabels Failed."); throw new RuntimeException("getRMNodeLabels Failed."); } @@ -1515,18 +1670,26 @@ public LabelsToNodesInfo getLabelsToNodes(Set labels) LabelsToNodesInfo labelsToNodesInfo = new LabelsToNodesInfo(labelToNodesMap); if (labelsToNodesInfo != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_LABELSTONODES, + TARGET_WEB_SERVICE); routerMetrics.succeededGetLabelsToNodesRetrieved(stopTime - startTime); return labelsToNodesInfo; } } catch (NotFoundException e) { routerMetrics.incrLabelsToNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_LABELSTONODES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrLabelsToNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_LABELSTONODES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException( e, "getLabelsToNodes by labels = %s with yarn error.", StringUtils.join(labels, ",")); } routerMetrics.incrLabelsToNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_LABELSTONODES, UNKNOWN, + TARGET_WEB_SERVICE, "getLabelsToNodes Failed."); throw RouterServerUtil.logAndReturnRunTimeException( "getLabelsToNodes by labels = %s Failed.", StringUtils.join(labels, ",")); } @@ -1553,6 +1716,9 @@ public Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels, List nodeToLabelsEntries = newNodeToLabels.getNodeToLabels(); if (CollectionUtils.isEmpty(nodeToLabelsEntries)) { routerMetrics.incrReplaceLabelsOnNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REPLACE_LABELSONNODES, UNKNOWN, + TARGET_WEB_SERVICE, "Parameter error, " + + "nodeToLabelsEntries must not be empty."); throw new IllegalArgumentException("Parameter error, " + "nodeToLabelsEntries must not be empty."); } @@ -1594,12 +1760,16 @@ public Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels, } }); long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), REPLACE_LABELSONNODES, + TARGET_WEB_SERVICE); routerMetrics.succeededReplaceLabelsOnNodesRetrieved(stopTime - startTime); // Step5. return call result. return Response.status(Status.OK).entity(builder.toString()).build(); } catch (Exception e) { routerMetrics.incrReplaceLabelsOnNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REPLACE_LABELSONNODES, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } } @@ -1623,10 +1793,14 @@ public Response replaceLabelsOnNode(Set newNodeLabelsName, // Step1. Check the parameters to ensure that the parameters are not empty. if (StringUtils.isBlank(nodeId)) { routerMetrics.incrReplaceLabelsOnNodeFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REPLACE_LABELSONNODE, UNKNOWN, + TARGET_WEB_SERVICE, "Parameter error, nodeId must not be null or empty."); throw new IllegalArgumentException("Parameter error, nodeId must not be null or empty."); } if (CollectionUtils.isEmpty(newNodeLabelsName)) { routerMetrics.incrReplaceLabelsOnNodeFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REPLACE_LABELSONNODE, UNKNOWN, + TARGET_WEB_SERVICE, "Parameter error, newNodeLabelsName must not be empty."); throw new IllegalArgumentException("Parameter error, newNodeLabelsName must not be empty."); } @@ -1641,11 +1815,15 @@ public Response replaceLabelsOnNode(Set newNodeLabelsName, // Step3. Return the response result. long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), REPLACE_LABELSONNODE, + TARGET_WEB_SERVICE); routerMetrics.succeededReplaceLabelsOnNodeRetrieved(stopTime - startTime); String msg = "subCluster#" + subClusterInfo.getSubClusterId().getId() + ":Success;"; return Response.status(Status.OK).entity(msg).build(); } catch (Exception e) { routerMetrics.incrReplaceLabelsOnNodeFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REPLACE_LABELSONNODE, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } } @@ -1668,16 +1846,24 @@ public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) if (nodeLabelsInfo != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetClusterNodeLabelsRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_CLUSTER_NODELABELS, + TARGET_WEB_SERVICE); return nodeLabelsInfo; } } catch (NotFoundException e) { routerMetrics.incrClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("getClusterNodeLabels with yarn error.", e); } routerMetrics.incrClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, "getClusterNodeLabels Failed."); throw new RuntimeException("getClusterNodeLabels Failed."); } @@ -1697,12 +1883,16 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, if (newNodeLabels == null) { routerMetrics.incrAddToClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), ADD_TO_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, "Parameter error, the newNodeLabels is null."); throw new IllegalArgumentException("Parameter error, the newNodeLabels is null."); } List nodeLabelInfos = newNodeLabels.getNodeLabelsInfo(); if (CollectionUtils.isEmpty(nodeLabelInfos)) { routerMetrics.incrAddToClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), ADD_TO_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, "Parameter error, the nodeLabelsInfo is null or empty."); throw new IllegalArgumentException("Parameter error, the nodeLabelsInfo is null or empty."); } @@ -1720,17 +1910,25 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, responseInfoMap.forEach((subClusterInfo, response) -> buildAppendMsg(subClusterInfo, buffer, response)); long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), ADD_TO_CLUSTER_NODELABELS, + TARGET_WEB_SERVICE); routerMetrics.succeededAddToClusterNodeLabelsRetrieved((stopTime - startTime)); return Response.status(Status.OK).entity(buffer.toString()).build(); } catch (NotFoundException e) { routerMetrics.incrAddToClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), ADD_TO_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrAddToClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), ADD_TO_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("addToClusterNodeLabels with yarn error.", e); } routerMetrics.incrAddToClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), ADD_TO_CLUSTER_NODELABELS, UNKNOWN, + TARGET_WEB_SERVICE, "addToClusterNodeLabels Failed."); throw new RuntimeException("addToClusterNodeLabels Failed."); } @@ -1750,6 +1948,8 @@ public Response removeFromClusterNodeLabels(Set oldNodeLabels, if (CollectionUtils.isEmpty(oldNodeLabels)) { routerMetrics.incrRemoveFromClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REMOVE_FROM_CLUSTERNODELABELS, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the oldNodeLabels is null or empty."); throw new IllegalArgumentException("Parameter error, the oldNodeLabels is null or empty."); } @@ -1768,13 +1968,19 @@ public Response removeFromClusterNodeLabels(Set oldNodeLabels, responseInfoMap.forEach((subClusterInfo, response) -> buildAppendMsg(subClusterInfo, buffer, response)); long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), REMOVE_FROM_CLUSTERNODELABELS, + TARGET_WEB_SERVICE); routerMetrics.succeededRemoveFromClusterNodeLabelsRetrieved(stopTime - startTime); return Response.status(Status.OK).entity(buffer.toString()).build(); } catch (NotFoundException e) { routerMetrics.incrRemoveFromClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REMOVE_FROM_CLUSTERNODELABELS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrRemoveFromClusterNodeLabelsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), REMOVE_FROM_CLUSTERNODELABELS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("removeFromClusterNodeLabels with yarn error.", e); } @@ -1819,17 +2025,25 @@ public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) if (nodeLabelsInfo != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetLabelsToNodesRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_LABELS_ON_NODE, + TARGET_WEB_SERVICE); return nodeLabelsInfo; } } catch (NotFoundException e) { routerMetrics.incrLabelsToNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_LABELS_ON_NODE, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e); } catch (YarnException e) { routerMetrics.incrLabelsToNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_LABELS_ON_NODE, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowIOException( e, "getLabelsOnNode nodeId = %s with yarn error.", nodeId); } routerMetrics.incrLabelsToNodesFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_LABELS_ON_NODE, + UNKNOWN, TARGET_WEB_SERVICE, "getLabelsOnNode by nodeId = " + nodeId + " Failed."); throw RouterServerUtil.logAndReturnRunTimeException( "getLabelsOnNode by nodeId = %s Failed.", nodeId); } @@ -1845,17 +2059,25 @@ public AppPriority getAppPriority(HttpServletRequest hsr, String appId) if (appPriority != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetAppPriorityRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APP_PRIORITY, + TARGET_WEB_SERVICE); return appPriority; } } catch (IllegalArgumentException e) { routerMetrics.incrGetAppPriorityFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_PRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get the getAppPriority appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrGetAppPriorityFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_PRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getAppPriority error.", e); } routerMetrics.incrGetAppPriorityFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_PRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, "getAppPriority Failed."); throw new RuntimeException("getAppPriority Failed."); } @@ -1866,6 +2088,8 @@ public Response updateApplicationPriority(AppPriority targetPriority, if (targetPriority == null) { routerMetrics.incrUpdateAppPriorityFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONPRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the targetPriority is empty or null."); throw new IllegalArgumentException("Parameter error, the targetPriority is empty or null."); } @@ -1876,16 +2100,24 @@ public Response updateApplicationPriority(AppPriority targetPriority, if (response != null) { long stopTime = clock.getTime(); routerMetrics.succeededUpdateAppPriorityRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), UPDATE_APPLICATIONPRIORITY, + TARGET_WEB_SERVICE); return response; } } catch (IllegalArgumentException e) { routerMetrics.incrUpdateAppPriorityFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONPRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get the updateApplicationPriority appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrUpdateAppPriorityFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONPRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("updateApplicationPriority error.", e); } + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONPRIORITY, + UNKNOWN, TARGET_WEB_SERVICE, "getAppPriority Failed."); routerMetrics.incrUpdateAppPriorityFailedRetrieved(); throw new RuntimeException("updateApplicationPriority Failed."); } @@ -1901,15 +2133,23 @@ public AppQueue getAppQueue(HttpServletRequest hsr, String appId) if (queue != null) { long stopTime = clock.getTime(); routerMetrics.succeededGetAppQueueRetrieved((stopTime - startTime)); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_QUEUEINFO, + TARGET_WEB_SERVICE); return queue; } } catch (IllegalArgumentException e) { routerMetrics.incrGetAppQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_QUEUEINFO, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get queue by appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrGetAppQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_QUEUEINFO, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getAppQueue error.", e); } + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_QUEUEINFO, + UNKNOWN, TARGET_WEB_SERVICE, "getAppQueue Failed."); routerMetrics.incrGetAppQueueFailedRetrieved(); throw new RuntimeException("getAppQueue Failed."); } @@ -1921,6 +2161,8 @@ public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, if (targetQueue == null) { routerMetrics.incrUpdateAppQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APP_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the targetQueue is null."); throw new IllegalArgumentException("Parameter error, the targetQueue is null."); } @@ -1931,16 +2173,24 @@ public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, if (response != null) { long stopTime = clock.getTime(); routerMetrics.succeededUpdateAppQueueRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), UPDATE_APP_QUEUE, + TARGET_WEB_SERVICE); return response; } } catch (IllegalArgumentException e) { routerMetrics.incrUpdateAppQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APP_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to update app queue by appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrUpdateAppQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APP_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("updateAppQueue error.", e); } + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APP_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, "updateAppQueue Failed."); routerMetrics.incrUpdateAppQueueFailedRetrieved(); throw new RuntimeException("updateAppQueue Failed."); } @@ -1961,6 +2211,8 @@ public Response postDelegationToken(DelegationToken tokenData, HttpServletReques throws AuthorizationException, IOException, InterruptedException, Exception { if (tokenData == null || hsr == null) { + RouterAuditLogger.logFailure(getUser().getShortUserName(), POST_DELEGATION_TOKEN, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the tokenData or hsr is null."); throw new IllegalArgumentException("Parameter error, the tokenData or hsr is null."); } @@ -1973,6 +2225,8 @@ public Response postDelegationToken(DelegationToken tokenData, HttpServletReques return createDelegationToken(tokenData, callerUGI); } catch (YarnException e) { LOG.error("Create delegation token request failed.", e); + RouterAuditLogger.logFailure(getUser().getShortUserName(), POST_DELEGATION_TOKEN, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.FORBIDDEN).entity(e.getMessage()).build(); } } @@ -1997,6 +2251,8 @@ private Response createDelegationToken(DelegationToken dtoken, UserGroupInformat }); DelegationToken respToken = getDelegationToken(renewer, resp); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), POST_DELEGATION_TOKEN, + TARGET_WEB_SERVICE); return Response.status(Status.OK).entity(respToken).build(); } @@ -2061,6 +2317,8 @@ public Response postDelegationTokenExpiration(HttpServletRequest hsr) throws AuthorizationException, IOException, InterruptedException, Exception { if (hsr == null) { + RouterAuditLogger.logFailure(getUser().getShortUserName(), POST_DELEGATION_TOKEN_EXPIRATION, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the hsr is null."); throw new IllegalArgumentException("Parameter error, the hsr is null."); } @@ -2071,6 +2329,8 @@ public Response postDelegationTokenExpiration(HttpServletRequest hsr) return renewDelegationToken(hsr, callerUGI); } catch (YarnException e) { LOG.error("Renew delegation token request failed.", e); + RouterAuditLogger.logFailure(getUser().getShortUserName(), POST_DELEGATION_TOKEN_EXPIRATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.FORBIDDEN).entity(e.getMessage()).build(); } } @@ -2109,6 +2369,8 @@ private Response renewDelegationToken(HttpServletRequest hsr, UserGroupInformati long renewTime = resp.getNextExpirationTime(); DelegationToken respToken = new DelegationToken(); respToken.setNextExpirationTime(renewTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), POST_DELEGATION_TOKEN_EXPIRATION, + TARGET_WEB_SERVICE); return Response.status(Status.OK).entity(respToken).build(); } @@ -2142,9 +2404,13 @@ public Response cancelDelegationToken(HttpServletRequest hsr) return this.getRouterClientRMService().cancelDelegationToken(req); }); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), CANCEL_DELEGATIONTOKEN, + TARGET_WEB_SERVICE); return Response.status(Status.OK).build(); } catch (YarnException e) { LOG.error("Cancel delegation token request failed.", e); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CANCEL_DELEGATIONTOKEN, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.FORBIDDEN).entity(e.getMessage()).build(); } } @@ -2166,15 +2432,21 @@ public Response createNewReservation(HttpServletRequest hsr) // this request can be returned directly. if (response != null && response.getStatus() == HttpServletResponse.SC_OK) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_NEW_RESERVATION, + TARGET_WEB_SERVICE); routerMetrics.succeededGetNewReservationRetrieved(stopTime - startTime); return response; } } catch (FederationPolicyException e) { // If a FederationPolicyException is thrown, the service is unavailable. routerMetrics.incrGetNewReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NEW_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getLocalizedMessage()).build(); } catch (Exception e) { routerMetrics.incrGetNewReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NEW_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR).entity(e.getLocalizedMessage()).build(); } @@ -2182,6 +2454,8 @@ public Response createNewReservation(HttpServletRequest hsr) String errMsg = "Fail to create a new reservation."; LOG.error(errMsg); routerMetrics.incrGetNewReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_NEW_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, errMsg); return Response.status(Status.INTERNAL_SERVER_ERROR).entity(errMsg).build(); } @@ -2217,6 +2491,8 @@ public Response submitReservation(ReservationSubmissionRequestInfo resContext, routerMetrics.incrSubmitReservationFailedRetrieved(); String errMsg = "Missing submitReservation resContext or reservationId " + "or reservation definition or queue."; + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, errMsg); return Response.status(Status.BAD_REQUEST).entity(errMsg).build(); } @@ -2226,6 +2502,8 @@ public Response submitReservation(ReservationSubmissionRequestInfo resContext, RouterServerUtil.validateReservationId(resId); } catch (IllegalArgumentException e) { routerMetrics.incrSubmitReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -2238,16 +2516,22 @@ public Response submitReservation(ReservationSubmissionRequestInfo resContext, runWithRetries(actualRetryNums, submitIntervalTime); if (response != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), SUBMIT_RESERVATION, + TARGET_WEB_SERVICE); routerMetrics.succeededSubmitReservationRetrieved(stopTime - startTime); return response; } } catch (Exception e) { routerMetrics.incrSubmitReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getLocalizedMessage()).build(); } routerMetrics.incrSubmitReservationFailedRetrieved(); String msg = String.format("Reservation %s failed to be submitted.", resId); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SUBMIT_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, msg); return Response.status(Status.SERVICE_UNAVAILABLE).entity(msg).build(); } @@ -2311,6 +2595,8 @@ public Response updateReservation(ReservationUpdateRequestInfo resContext, routerMetrics.incrUpdateReservationFailedRetrieved(); String errMsg = "Missing updateReservation resContext or reservationId " + "or reservation definition."; + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, errMsg); return Response.status(Status.BAD_REQUEST).entity(errMsg).build(); } @@ -2322,6 +2608,8 @@ public Response updateReservation(ReservationUpdateRequestInfo resContext, RouterServerUtil.validateReservationId(reservationId); } catch (IllegalArgumentException e) { routerMetrics.incrUpdateReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -2332,15 +2620,21 @@ public Response updateReservation(ReservationUpdateRequestInfo resContext, HttpServletRequest hsrCopy = clone(hsr); Response response = interceptor.updateReservation(resContext, hsrCopy); if (response != null) { + RouterAuditLogger.logSuccess(getUser().getShortUserName(), UPDATE_RESERVATION, + TARGET_WEB_SERVICE); return response; } } catch (Exception e) { routerMetrics.incrUpdateReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("updateReservation Failed.", e); } // throw an exception routerMetrics.incrUpdateReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, "updateReservation Failed, reservationId = " + reservationId); throw new YarnRuntimeException("updateReservation Failed, reservationId = " + reservationId); } @@ -2353,6 +2647,8 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, if (resContext == null || resContext.getReservationId() == null) { routerMetrics.incrDeleteReservationFailedRetrieved(); String errMsg = "Missing deleteReservation request or reservationId."; + RouterAuditLogger.logFailure(getUser().getShortUserName(), DELETE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, errMsg); return Response.status(Status.BAD_REQUEST).entity(errMsg).build(); } @@ -2364,6 +2660,8 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, RouterServerUtil.validateReservationId(reservationId); } catch (IllegalArgumentException e) { routerMetrics.incrDeleteReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DELETE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -2374,15 +2672,21 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, HttpServletRequest hsrCopy = clone(hsr); Response response = interceptor.deleteReservation(resContext, hsrCopy); if (response != null) { + RouterAuditLogger.logSuccess(getUser().getShortUserName(), DELETE_RESERVATION, + TARGET_WEB_SERVICE); return response; } } catch (Exception e) { routerMetrics.incrDeleteReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DELETE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("deleteReservation Failed.", e); } // throw an exception routerMetrics.incrDeleteReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), DELETE_RESERVATION, + UNKNOWN, TARGET_WEB_SERVICE, "deleteReservation Failed, reservationId = " + reservationId); throw new YarnRuntimeException("deleteReservation Failed, reservationId = " + reservationId); } @@ -2393,11 +2697,15 @@ public Response listReservation(String queue, String reservationId, if (queue == null || queue.isEmpty()) { routerMetrics.incrListReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), LIST_RESERVATIONS, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the queue is empty or null."); throw new IllegalArgumentException("Parameter error, the queue is empty or null."); } if (reservationId == null || reservationId.isEmpty()) { routerMetrics.incrListReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), LIST_RESERVATIONS, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the reservationId is empty or null."); throw new IllegalArgumentException("Parameter error, the reservationId is empty or null."); } @@ -2406,6 +2714,8 @@ public Response listReservation(String queue, String reservationId, RouterServerUtil.validateReservationId(reservationId); } catch (IllegalArgumentException e) { routerMetrics.incrListReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), LIST_RESERVATIONS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -2419,15 +2729,21 @@ public Response listReservation(String queue, String reservationId, includeResourceAllocations, hsrCopy); if (response != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), LIST_RESERVATIONS, + TARGET_WEB_SERVICE); routerMetrics.succeededListReservationRetrieved(stopTime - startTime1); return response; } } catch (YarnException e) { routerMetrics.incrListReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), LIST_RESERVATIONS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("listReservation error.", e); } routerMetrics.incrListReservationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), LIST_RESERVATIONS, + UNKNOWN, TARGET_WEB_SERVICE, "listReservation Failed."); throw new YarnException("listReservation Failed."); } @@ -2437,6 +2753,8 @@ public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId, if (type == null || type.isEmpty()) { routerMetrics.incrGetAppTimeoutFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUT, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the type is empty or null."); throw new IllegalArgumentException("Parameter error, the type is empty or null."); } @@ -2446,18 +2764,26 @@ public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId, AppTimeoutInfo appTimeoutInfo = interceptor.getAppTimeout(hsr, appId, type); if (appTimeoutInfo != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APP_TIMEOUT, + TARGET_WEB_SERVICE); routerMetrics.succeededGetAppTimeoutRetrieved((stopTime - startTime)); return appTimeoutInfo; } } catch (IllegalArgumentException e) { routerMetrics.incrGetAppTimeoutFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUT, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get the getAppTimeout appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrGetAppTimeoutFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUT, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getAppTimeout error.", e); } routerMetrics.incrGetAppTimeoutFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUT, + UNKNOWN, TARGET_WEB_SERVICE, "getAppTimeout Failed."); throw new RuntimeException("getAppTimeout Failed."); } @@ -2471,19 +2797,27 @@ public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId) AppTimeoutsInfo appTimeoutsInfo = interceptor.getAppTimeouts(hsr, appId); if (appTimeoutsInfo != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APP_TIMEOUTS, + TARGET_WEB_SERVICE); routerMetrics.succeededGetAppTimeoutsRetrieved((stopTime - startTime)); return appTimeoutsInfo; } } catch (IllegalArgumentException e) { routerMetrics.incrGetAppTimeoutsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get the getAppTimeouts appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrGetAppTimeoutsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getAppTimeouts error.", e); } routerMetrics.incrGetAppTimeoutsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_TIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, "getAppTimeouts Failed."); throw new RuntimeException("getAppTimeouts Failed."); } @@ -2494,6 +2828,8 @@ public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, if (appTimeout == null) { routerMetrics.incrUpdateApplicationTimeoutsRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONTIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the appTimeout is null."); throw new IllegalArgumentException("Parameter error, the appTimeout is null."); } @@ -2503,19 +2839,27 @@ public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, Response response = interceptor.updateApplicationTimeout(appTimeout, hsr, appId); if (response != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), UPDATE_APPLICATIONTIMEOUTS, + TARGET_WEB_SERVICE); routerMetrics.succeededUpdateAppTimeoutsRetrieved((stopTime - startTime)); return response; } } catch (IllegalArgumentException e) { routerMetrics.incrUpdateApplicationTimeoutsRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONTIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get the updateApplicationTimeout appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrUpdateApplicationTimeoutsRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONTIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("updateApplicationTimeout error.", e); } routerMetrics.incrUpdateApplicationTimeoutsRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_APPLICATIONTIMEOUTS, + UNKNOWN, TARGET_WEB_SERVICE, "updateApplicationTimeout Failed."); throw new RuntimeException("updateApplicationTimeout Failed."); } @@ -2529,18 +2873,26 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) { if (appAttemptsInfo != null) { long stopTime = Time.now(); routerMetrics.succeededAppAttemptsRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APPLICATION_ATTEMPTS, + TARGET_WEB_SERVICE); return appAttemptsInfo; } } catch (IllegalArgumentException e) { routerMetrics.incrAppAttemptsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPLICATION_ATTEMPTS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get the AppAttempt appId: %s.", appId); } catch (YarnException e) { routerMetrics.incrAppAttemptsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPLICATION_ATTEMPTS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getAppAttempts error.", e); } routerMetrics.incrAppAttemptsFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APPLICATION_ATTEMPTS, + UNKNOWN, TARGET_WEB_SERVICE, "getAppAttempts Failed."); throw new RuntimeException("getAppAttempts Failed."); } @@ -2551,16 +2903,22 @@ public RMQueueAclInfo checkUserAccessToQueue(String queue, String username, // Parameter Verification if (queue == null || queue.isEmpty()) { routerMetrics.incrCheckUserAccessToQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the queue is empty or null."); throw new IllegalArgumentException("Parameter error, the queue is empty or null."); } if (username == null || username.isEmpty()) { routerMetrics.incrCheckUserAccessToQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the username is empty or null."); throw new IllegalArgumentException("Parameter error, the username is empty or null."); } if (queueAclType == null || queueAclType.isEmpty()) { routerMetrics.incrCheckUserAccessToQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the queueAclType is empty or null."); throw new IllegalArgumentException("Parameter error, the queueAclType is empty or null."); } @@ -2582,17 +2940,25 @@ public RMQueueAclInfo checkUserAccessToQueue(String queue, String username, aclInfo.getList().add(rMQueueAclInfo); }); long stopTime = Time.now(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + TARGET_WEB_SERVICE); routerMetrics.succeededCheckUserAccessToQueueRetrieved(stopTime - startTime); return aclInfo; } catch (NotFoundException e) { routerMetrics.incrCheckUserAccessToQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("Get all active sub cluster(s) error.", e); } catch (YarnException | IOException e) { routerMetrics.incrCheckUserAccessToQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("checkUserAccessToQueue error.", e); } routerMetrics.incrCheckUserAccessToQueueFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), CHECK_USER_ACCESS_TO_QUEUE, + UNKNOWN, TARGET_WEB_SERVICE, "checkUserAccessToQueue error."); throw new RuntimeException("checkUserAccessToQueue error."); } @@ -2605,6 +2971,8 @@ public AppAttemptInfo getAppAttempt(HttpServletRequest req, RouterServerUtil.validateApplicationAttemptId(appAttemptId); } catch (IllegalArgumentException e) { routerMetrics.incrAppAttemptReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_ATTEMPT, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -2615,20 +2983,28 @@ public AppAttemptInfo getAppAttempt(HttpServletRequest req, AppAttemptInfo appAttemptInfo = interceptor.getAppAttempt(req, res, appId, appAttemptId); if (appAttemptInfo != null) { long stopTime = Time.now(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_APP_ATTEMPT, + TARGET_WEB_SERVICE); routerMetrics.succeededAppAttemptReportRetrieved(stopTime - startTime); return appAttemptInfo; } } catch (IllegalArgumentException e) { routerMetrics.incrAppAttemptReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_ATTEMPT, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Unable to getAppAttempt by appId: %s, appAttemptId: %s.", appId, appAttemptId); } catch (YarnException e) { routerMetrics.incrAppAttemptReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_ATTEMPT, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getAppAttempt error, appId: %s, appAttemptId: %s.", appId, appAttemptId); } routerMetrics.incrAppAttemptReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_APP_ATTEMPT, + UNKNOWN, TARGET_WEB_SERVICE, "getAppAttempt failed."); throw RouterServerUtil.logAndReturnRunTimeException( "getAppAttempt failed, appId: %s, appAttemptId: %s.", appId, appAttemptId); } @@ -2642,6 +3018,8 @@ public ContainersInfo getContainers(HttpServletRequest req, RouterServerUtil.validateApplicationId(appId); RouterServerUtil.validateApplicationAttemptId(appAttemptId); } catch (IllegalArgumentException e) { + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINERS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); routerMetrics.incrGetContainersFailedRetrieved(); throw e; } @@ -2662,20 +3040,28 @@ public ContainersInfo getContainers(HttpServletRequest req, } if (containersInfo != null) { long stopTime = clock.getTime(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_CONTAINERS, + TARGET_WEB_SERVICE); routerMetrics.succeededGetContainersRetrieved(stopTime - startTime); return containersInfo; } } catch (NotFoundException e) { routerMetrics.incrGetContainersFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINERS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getContainers error, appId = %s, " + " appAttemptId = %s, Probably getActiveSubclusters error.", appId, appAttemptId); } catch (IOException | YarnException e) { routerMetrics.incrGetContainersFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINERS, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "getContainers error, appId = %s, " + " appAttemptId = %s.", appId, appAttemptId); } routerMetrics.incrGetContainersFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINERS, + UNKNOWN, TARGET_WEB_SERVICE, "getContainers failed."); throw RouterServerUtil.logAndReturnRunTimeException( "getContainers failed, appId: %s, appAttemptId: %s.", appId, appAttemptId); } @@ -2695,6 +3081,8 @@ public ContainerInfo getContainer(HttpServletRequest req, RouterServerUtil.validateContainerId(containerId); } catch (IllegalArgumentException e) { routerMetrics.incrGetContainerReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } @@ -2706,6 +3094,8 @@ public ContainerInfo getContainer(HttpServletRequest req, if (containerInfo != null) { long stopTime = Time.now(); routerMetrics.succeededGetContainerReportRetrieved(stopTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_CONTAINER, + TARGET_WEB_SERVICE); return containerInfo; } } catch (IllegalArgumentException e) { @@ -2713,13 +3103,19 @@ public ContainerInfo getContainer(HttpServletRequest req, "Unable to get the AppAttempt appId: %s, appAttemptId: %s, containerId: %s.", appId, appAttemptId, containerId); routerMetrics.incrGetContainerReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(msg, e); } catch (YarnException e) { routerMetrics.incrGetContainerReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("getContainer Failed.", e); } routerMetrics.incrGetContainerReportFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_CONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, "getContainer Failed."); throw new RuntimeException("getContainer Failed."); } @@ -2743,6 +3139,9 @@ public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, // Make Sure mutationInfo is not null. if (mutationInfo == null) { routerMetrics.incrUpdateSchedulerConfigurationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, + "Parameter error, the schedConfUpdateInfo is empty or null."); throw new IllegalArgumentException( "Parameter error, the schedConfUpdateInfo is empty or null."); } @@ -2753,6 +3152,9 @@ public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, String pSubClusterId = mutationInfo.getSubClusterId(); if (StringUtils.isBlank(pSubClusterId)) { routerMetrics.incrUpdateSchedulerConfigurationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, + "Parameter error, the subClusterId is empty or null."); throw new IllegalArgumentException("Parameter error, " + "the subClusterId is empty or null."); } @@ -2767,19 +3169,27 @@ public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, if (response != null) { long endTime = clock.getTime(); routerMetrics.succeededUpdateSchedulerConfigurationRetrieved(endTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), UPDATE_SCHEDULER_CONFIGURATION, + TARGET_WEB_SERVICE); return Response.status(response.getStatus()).entity(response.getEntity()).build(); } } catch (NotFoundException e) { routerMetrics.incrUpdateSchedulerConfigurationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "Get subCluster error. subClusterId = %s", pSubClusterId); } catch (Exception e) { routerMetrics.incrUpdateSchedulerConfigurationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException(e, "UpdateSchedulerConfiguration error. subClusterId = %s", pSubClusterId); } routerMetrics.incrUpdateSchedulerConfigurationFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), UPDATE_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, "UpdateSchedulerConfiguration Failed."); throw new RuntimeException("UpdateSchedulerConfiguration error. subClusterId = " + pSubClusterId); } @@ -2822,18 +3232,25 @@ public Response getSchedulerConfiguration(HttpServletRequest hsr) }); long endTime = clock.getTime(); routerMetrics.succeededGetSchedulerConfigurationRetrieved(endTime - startTime); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), GET_SCHEDULER_CONFIGURATION, + TARGET_WEB_SERVICE); return Response.status(Status.OK).entity(federationConfInfo).build(); } catch (NotFoundException e) { - RouterServerUtil.logAndThrowRunTimeException("get all active sub cluster(s) error.", e); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); routerMetrics.incrGetSchedulerConfigurationFailedRetrieved(); + RouterServerUtil.logAndThrowRunTimeException("get all active sub cluster(s) error.", e); } catch (Exception e) { + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); routerMetrics.incrGetSchedulerConfigurationFailedRetrieved(); - RouterServerUtil.logAndThrowRunTimeException("getSchedulerConfiguration error.", e); return Response.status(Status.BAD_REQUEST).entity("getSchedulerConfiguration error.").build(); } routerMetrics.incrGetSchedulerConfigurationFailedRetrieved(); - throw new RuntimeException("getSchedulerConfiguration error."); + RouterAuditLogger.logFailure(getUser().getShortUserName(), GET_SCHEDULER_CONFIGURATION, + UNKNOWN, TARGET_WEB_SERVICE, "getSchedulerConfiguration Failed."); + throw new RuntimeException("getSchedulerConfiguration Failed."); } @Override @@ -2853,12 +3270,16 @@ public Response signalToContainer(String containerId, String command, RouterServerUtil.validateContainerId(containerId); } catch (IllegalArgumentException e) { routerMetrics.incrSignalToContainerFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SIGNAL_TOCONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); throw e; } // Check if command is empty or null if (command == null || command.isEmpty()) { routerMetrics.incrSignalToContainerFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SIGNAL_TOCONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, "Parameter error, the command is empty or null."); throw new IllegalArgumentException("Parameter error, the command is empty or null."); } @@ -2874,18 +3295,26 @@ public Response signalToContainer(String containerId, String command, Response response = interceptor.signalToContainer(containerId, command, req); if (response != null) { long stopTime = Time.now(); + RouterAuditLogger.logSuccess(getUser().getShortUserName(), SIGNAL_TOCONTAINER, + TARGET_WEB_SERVICE); routerMetrics.succeededSignalToContainerRetrieved(stopTime - startTime); return response; } } catch (YarnException e) { routerMetrics.incrSignalToContainerFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SIGNAL_TOCONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("signalToContainer Failed.", e); } catch (AuthorizationException e) { routerMetrics.incrSignalToContainerFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SIGNAL_TOCONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, e.getLocalizedMessage()); RouterServerUtil.logAndThrowRunTimeException("signalToContainer Author Failed.", e); } routerMetrics.incrSignalToContainerFailedRetrieved(); + RouterAuditLogger.logFailure(getUser().getShortUserName(), SIGNAL_TOCONTAINER, + UNKNOWN, TARGET_WEB_SERVICE, "signalToContainer Failed."); throw new RuntimeException("signalToContainer Failed."); } From 0042544bf2b3bcb89f1bbd3d792e489c28655432 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Tue, 24 Oct 2023 12:28:40 +0100 Subject: [PATCH 116/155] HADOOP-18949. upgrade maven dependency plugin due to CVE-2021-26291. (#6219) Addresses CVE-2021-26291. "Origin Validation Error in Apache Maven" Contributed by PJ Fanning. --- hadoop-maven-plugins/pom.xml | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml index 522c5a9468705..8765eb795b874 100644 --- a/hadoop-maven-plugins/pom.xml +++ b/hadoop-maven-plugins/pom.xml @@ -26,26 +26,56 @@ maven-plugin Apache Hadoop Maven Plugins - 3.0.5 - 3.6.0 + 3.9.5 + 3.10.1 + 2.7.0 + 0.3.5 org.apache.maven maven-plugin-api ${maven.dependency.version} + + + org.eclipse.sisu + org.eclipse.sisu.inject + + + org.codehaus.plexus + plexus-classworlds + + org.apache.maven maven-core ${maven.dependency.version} + + org.eclipse.sisu + org.eclipse.sisu.inject + org.sonatype.sisu sisu-inject-plexus + + org.codehaus.plexus + plexus-classworlds + + + org.codehaus.plexus + plexus-classworlds + ${plexus.classworlds.version} + + + org.eclipse.sisu + org.eclipse.sisu.inject + ${sisu.inject.version} + org.apache.maven.plugin-tools maven-plugin-annotations From 8b974bcc1f084ae77dccf99ebc243e7a571f2e11 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 24 Oct 2023 18:17:52 +0100 Subject: [PATCH 117/155] HADOOP-18889. Third party storage followup. (#6186) Followup to HADOOP-18889 third party store support; Fix some minor review comments which came in after the merge. --- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../org/apache/hadoop/fs/s3a/auth/SignerFactory.java | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 8ab8d22cc6d84..d7149d7dead6b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1357,7 +1357,7 @@ public String getBucketLocation() throws IOException { public String getBucketLocation(String bucketName) throws IOException { final String region = trackDurationAndSpan( STORE_EXISTS_PROBE, bucketName, null, () -> - once("getBucketLocation()", bucketName, () -> + invoker.retry("getBucketLocation()", bucketName, true, () -> // If accessPoint then region is known from Arn accessPoint != null ? accessPoint.getRegion() diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java index 5d34688cebe14..21c390c07940b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerFactory.java @@ -81,16 +81,6 @@ public static void registerSigner( SIGNERS.put(signerType, signerClass); } - /** - * Check if the signer has already been registered. - * @param signerType signer to get - * @throws IllegalArgumentException if the signer type is unknown. - */ - public static void verifySignerRegistered(String signerType) { - checkArgument(isSignerRegistered(signerType), - "unknown signer type: %s", signerType); - } - /** * Check if the signer has already been registered. * @param signerType signer to get From 882f08b4bc1d23ac3b0d78b339ddd3a5af53abdd Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Tue, 24 Oct 2023 21:39:03 +0100 Subject: [PATCH 118/155] HDFS-17237. Remove IPCLoggerChannelMetrics when the logger is closed (#6217) --- .../qjournal/client/IPCLoggerChannel.java | 1 + .../client/IPCLoggerChannelMetrics.java | 39 +++---------------- .../qjournal/client/TestIPCLoggerChannel.java | 21 +++++++++- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 4b7e59c51f13e..67fc85810278d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -206,6 +206,7 @@ public void close() { // making any more calls after this point (eg clear the queue) RPC.stopProxy(proxy); } + metrics.unregister(); } protected QJournalProtocol getProxy() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java index 6eef8ffd38620..c1e27e2e98a71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannelMetrics.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.qjournal.client; import java.net.InetSocketAddress; -import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -29,8 +28,6 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableQuantiles; -import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; - /** * The metrics for a journal from the writer's perspective. */ @@ -43,21 +40,6 @@ class IPCLoggerChannelMetrics { private final MutableQuantiles[] writeEndToEndLatencyQuantiles; private final MutableQuantiles[] writeRpcLatencyQuantiles; - - /** - * In the case of the NN transitioning between states, edit logs are closed - * and reopened. Thus, the IPCLoggerChannel instance that writes to a - * given JournalNode may change over the lifetime of the process. - * However, metrics2 doesn't have a function to unregister a set of metrics - * and fails if a new metrics class is registered with the same name - * as the existing one. Hence, we have to maintain our own registry - * ("multiton") here, so that we have exactly one metrics instance - * per JournalNode, and switch out the pointer to the underlying - * IPCLoggerChannel instance. - */ - private static final Map REGISTRY = - Maps.newHashMap(); - private IPCLoggerChannelMetrics(IPCLoggerChannel ch) { this.ch = ch; @@ -81,25 +63,16 @@ private IPCLoggerChannelMetrics(IPCLoggerChannel ch) { writeRpcLatencyQuantiles = null; } } - - private void setChannel(IPCLoggerChannel ch) { - assert ch.getRemoteAddress().equals(this.ch.getRemoteAddress()); - this.ch = ch; + + public void unregister() { + DefaultMetricsSystem.instance().unregisterSource(getName(ch)); } static IPCLoggerChannelMetrics create(IPCLoggerChannel ch) { String name = getName(ch); - synchronized (REGISTRY) { - IPCLoggerChannelMetrics m = REGISTRY.get(name); - if (m != null) { - m.setChannel(ch); - } else { - m = new IPCLoggerChannelMetrics(ch); - DefaultMetricsSystem.instance().register(name, null, m); - REGISTRY.put(name, m); - } - return m; - } + IPCLoggerChannelMetrics m = new IPCLoggerChannelMetrics(ch); + DefaultMetricsSystem.instance().register(name, null, m); + return m; } private static String getName(IPCLoggerChannel ch) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java index f2f46424cfd5a..06df99de1fe8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestIPCLoggerChannel.java @@ -24,12 +24,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.qjournal.client.IPCLoggerChannel; -import org.apache.hadoop.hdfs.qjournal.client.LoggerTooFarBehindException; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol; import org.apache.hadoop.hdfs.qjournal.protocol.RequestInfo; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; @@ -178,4 +179,20 @@ public void testStopSendingEditsWhenOutOfSync() throws Exception { ch.sendEdits(3L, 3L, 1, FAKE_DATA).get(); } + + @Test + public void testMetricsRemovedOnClose() { + MetricsSystem metricsSystem = DefaultMetricsSystem.instance(); + String sourceName = "IPCLoggerChannel-" + + FAKE_ADDR.getAddress().getHostAddress() + + "-" + FAKE_ADDR.getPort(); + // Ensure the metrics exist + MetricsSource source = metricsSystem.getSource(sourceName); + assertNotNull(source); + + ch.close(); + // ensure the metrics are removed. + source = metricsSystem.getSource(sourceName); + assertNull(source); + } } From a170d58501cf9f1bca0f111007122caf3ebe9419 Mon Sep 17 00:00:00 2001 From: gp1314 <814085234@qq.com> Date: Wed, 25 Oct 2023 11:43:12 +0800 Subject: [PATCH 119/155] HDFS-17231. HA: Safemode should exit when resources are from low to available. (#6207). Contributed by Gu Peng. Reviewed-by: Xing Lin Signed-off-by: He Xiaoqiao --- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 ++++++++++++ .../server/namenode/TestNameNodeResourceChecker.java | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index d9b165f96ee0c..3d360c6d0dd2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4534,6 +4534,11 @@ public void run () { LOG.warn(lowResourcesMsg + "Already in safe mode."); } enterSafeMode(true); + } else { + if (isNoManualAndResourceLowSafeMode()) { + LOG.info("Namenode has sufficient available resources, exiting safe mode."); + leaveSafeMode(false); + } } try { Thread.sleep(resourceRecheckInterval); @@ -5265,6 +5270,13 @@ private synchronized boolean isInManualOrResourceLowSafeMode() { return manualSafeMode || resourceLowSafeMode; } + /** + * @return true if it is not in manual safe mode and resource low safe mode. + */ + private synchronized boolean isNoManualAndResourceLowSafeMode() { + return !manualSafeMode && resourceLowSafeMode; + } + private synchronized void setManualAndResourceLowSafeMode(boolean manual, boolean resourceLow) { this.manualSafeMode = manual; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java index f86ce5fc06772..f3e187b5e3cd9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java @@ -130,6 +130,14 @@ public void testCheckThatNameNodeResourceMonitorIsRunning() assertTrue("NN should be in safe mode after resources crossed threshold", cluster.getNameNode().isInSafeMode()); + + mockResourceChecker.setResourcesAvailable(true); + while (cluster.getNameNode().isInSafeMode() && + Time.now() < startMillis + (60 * 1000)) { + Thread.sleep(1000); + } + assertTrue("NN should leave safe mode after resources not crossed threshold", + !cluster.getNameNode().isInSafeMode()); } finally { if (cluster != null) cluster.shutdown(); From f85ac5b60daf478c08b3301d7633db933f1d0834 Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Wed, 25 Oct 2023 13:56:39 +0800 Subject: [PATCH 120/155] HADOOP-18920. RPC Metrics : Optimize logic for log slow RPCs (#6146) --- .../fs/CommonConfigurationKeysPublic.java | 4 ++ .../java/org/apache/hadoop/ipc/Server.java | 52 +++++++++++++------ .../src/main/resources/core-default.xml | 9 ++++ .../apache/hadoop/ipc/TestProtoBufRpc.java | 9 +++- .../hadoop/hdfs/server/namenode/NameNode.java | 48 ++++++++++++++++- .../namenode/TestNameNodeReconfigure.java | 46 ++++++++++++++++ .../hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- 7 files changed, 150 insertions(+), 20 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 397d81f92f60b..006144e64ad15 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -504,6 +504,10 @@ public class CommonConfigurationKeysPublic { "ipc.server.log.slow.rpc"; public static final boolean IPC_SERVER_LOG_SLOW_RPC_DEFAULT = false; + public static final String IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY = + "ipc.server.log.slow.rpc.threshold.ms"; + public static final long IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_DEFAULT = 0; + public static final String IPC_SERVER_PURGE_INTERVAL_MINUTES_KEY = "ipc.server.purge.interval"; public static final int IPC_SERVER_PURGE_INTERVAL_MINUTES_DEFAULT = 15; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 73c86c09fc79e..53497e9707807 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -516,16 +516,22 @@ protected ResponseBuffer initialValue() { private final long metricsUpdaterInterval; private final ScheduledExecutorService scheduledExecutorService; - private boolean logSlowRPC = false; + private volatile boolean logSlowRPC = false; + /** Threshold time for log slow rpc. */ + private volatile long logSlowRPCThresholdTime; /** * Checks if LogSlowRPC is set true. * @return true, if LogSlowRPC is set true, false, otherwise. */ - protected boolean isLogSlowRPC() { + public boolean isLogSlowRPC() { return logSlowRPC; } + public long getLogSlowRPCThresholdTime() { + return logSlowRPCThresholdTime; + } + public int getNumInProcessHandler() { return numInProcessHandler.get(); } @@ -543,10 +549,16 @@ public long getTotalRequestsPerSecond() { * @param logSlowRPCFlag input logSlowRPCFlag. */ @VisibleForTesting - protected void setLogSlowRPC(boolean logSlowRPCFlag) { + public void setLogSlowRPC(boolean logSlowRPCFlag) { this.logSlowRPC = logSlowRPCFlag; } + @VisibleForTesting + public void setLogSlowRPCThresholdTime(long logSlowRPCThresholdMs) { + this.logSlowRPCThresholdTime = rpcMetrics.getMetricsTimeUnit(). + convert(logSlowRPCThresholdMs, TimeUnit.MILLISECONDS); + } + private void setPurgeIntervalNanos(int purgeInterval) { int tmpPurgeInterval = CommonConfigurationKeysPublic. IPC_SERVER_PURGE_INTERVAL_MINUTES_DEFAULT; @@ -568,12 +580,15 @@ public long getPurgeIntervalNanos() { * @param methodName - RPC Request method name * @param details - Processing Detail. * - * if this request took too much time relative to other requests - * we consider that as a slow RPC. 3 is a magic number that comes - * from 3 sigma deviation. A very simple explanation can be found - * by searching for 68-95-99.7 rule. We flag an RPC as slow RPC - * if and only if it falls above 99.7% of requests. We start this logic - * only once we have enough sample size. + * If a request took significant more time than other requests, + * and its processing time is at least `logSlowRPCThresholdMs` we consider that as a slow RPC. + * + * The definition rules for calculating whether the current request took too much time + * compared to other requests are as follows: + * 3 is a magic number that comes from 3 sigma deviation. + * A very simple explanation can be found by searching for 68-95-99.7 rule. + * We flag an RPC as slow RPC if and only if it falls above 99.7% of requests. + * We start this logic only once we have enough sample size. */ void logSlowRpcCalls(String methodName, Call call, ProcessingDetails details) { @@ -587,15 +602,14 @@ void logSlowRpcCalls(String methodName, Call call, final double threeSigma = rpcMetrics.getProcessingMean() + (rpcMetrics.getProcessingStdDev() * deviation); - long processingTime = - details.get(Timing.PROCESSING, rpcMetrics.getMetricsTimeUnit()); + final TimeUnit metricsTimeUnit = rpcMetrics.getMetricsTimeUnit(); + long processingTime = details.get(Timing.PROCESSING, metricsTimeUnit); if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) && - (processingTime > threeSigma)) { - LOG.warn( - "Slow RPC : {} took {} {} to process from client {}," - + " the processing detail is {}", - methodName, processingTime, rpcMetrics.getMetricsTimeUnit(), call, - details.toString()); + (processingTime > threeSigma) && + (processingTime > getLogSlowRPCThresholdTime())) { + LOG.warn("Slow RPC : {} took {} {} to process from client {}, the processing detail is {}," + + " and the threshold time is {} {}.", methodName, processingTime, metricsTimeUnit, + call, details.toString(), getLogSlowRPCThresholdTime(), metricsTimeUnit); rpcMetrics.incrSlowRpc(); } } @@ -3359,6 +3373,10 @@ protected Server(String bindAddress, int port, CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC, CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_DEFAULT)); + this.setLogSlowRPCThresholdTime(conf.getLong( + CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY, + CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_DEFAULT)); + this.setPurgeIntervalNanos(conf.getInt( CommonConfigurationKeysPublic.IPC_SERVER_PURGE_INTERVAL_MINUTES_KEY, CommonConfigurationKeysPublic.IPC_SERVER_PURGE_INTERVAL_MINUTES_DEFAULT)); diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 6c3597a83fa69..d64abf79407ae 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2526,6 +2526,15 @@ The switch to turn S3A auditing on or off. + + ipc.server.log.slow.rpc.threshold.ms + 0 + The threshold in milliseconds for logging slow rpc when ipc.server.log.slow.rpc is enabled. + Besides of being much slower than other RPC requests, an RPC request has to take at least the threshold value + defined by this property before it can be considered as slow. By default, this threshold is set to 0 (disabled). + + + ipc.server.purge.interval 15 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java index 0740f056c8fc9..a9eaccb3bf3df 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java @@ -355,6 +355,7 @@ public void testLogSlowRPC() throws IOException, ServiceException, TimeoutException, InterruptedException { //No test with legacy assumeFalse(testWithLegacy); + server.setLogSlowRPCThresholdTime(SLEEP_DURATION); TestRpcService2 client = getClient2(); // make 10 K fast calls for (int x = 0; x < 10000; x++) { @@ -370,7 +371,13 @@ public void testLogSlowRPC() throws IOException, ServiceException, assertThat(rpcMetrics.getProcessingSampleCount()).isGreaterThan(999L); long before = rpcMetrics.getRpcSlowCalls(); - // make a really slow call. Sleep sleeps for 1000ms + // Sleep sleeps for 500ms(less than `logSlowRPCThresholdTime`), + // make sure we never called into Log slow RPC routine. + client.sleep(null, newSleepRequest(SLEEP_DURATION / 2)); + long after = rpcMetrics.getRpcSlowCalls(); + assertThat(before).isEqualTo(after); + + // Make a really slow call. Sleep sleeps for 3000ms. client.sleep(null, newSleepRequest(SLEEP_DURATION * 3)); // Ensure slow call is logged. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index bee7db315de5c..df490ea0d9fe0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -126,6 +126,10 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY; @@ -365,7 +369,9 @@ public enum OperationCategory { DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, - DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY)); + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY, + IPC_SERVER_LOG_SLOW_RPC, + IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" @@ -2369,6 +2375,9 @@ protected String reconfigurePropertyImpl(String property, String newVal) newVal); } else if (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY)) { return reconfigureMinBlocksForWrite(property, newVal); + } else if (property.equals(IPC_SERVER_LOG_SLOW_RPC) || + (property.equals(IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY))) { + return reconfigureLogSlowRPC(property, newVal); } else { throw new ReconfigurationException(property, newVal, getConf().get( property)); @@ -2511,6 +2520,43 @@ String reconfigureIPCBackoffEnabled(String newVal) { return Boolean.toString(clientBackoffEnabled); } + String reconfigureLogSlowRPC(String property, String newVal) throws ReconfigurationException { + String result = null; + try { + if (property.equals(IPC_SERVER_LOG_SLOW_RPC)) { + if (newVal != null && !newVal.equalsIgnoreCase("true") && + !newVal.equalsIgnoreCase("false")) { + throw new IllegalArgumentException(newVal + " is not boolean value"); + } + boolean logSlowRPC = (newVal == null ? IPC_SERVER_LOG_SLOW_RPC_DEFAULT : + Boolean.parseBoolean(newVal)); + rpcServer.getClientRpcServer().setLogSlowRPC(logSlowRPC); + if (rpcServer.getServiceRpcServer() != null) { + rpcServer.getServiceRpcServer().setLogSlowRPC(logSlowRPC); + } + if (rpcServer.getLifelineRpcServer() != null) { + rpcServer.getLifelineRpcServer().setLogSlowRPC(logSlowRPC); + } + result = Boolean.toString(logSlowRPC); + } else if (property.equals(IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY)) { + long logSlowRPCThresholdTime = (newVal == null ? + IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_DEFAULT : Long.parseLong(newVal)); + rpcServer.getClientRpcServer().setLogSlowRPCThresholdTime(logSlowRPCThresholdTime); + if (rpcServer.getServiceRpcServer() != null) { + rpcServer.getServiceRpcServer().setLogSlowRPCThresholdTime(logSlowRPCThresholdTime); + } + if (rpcServer.getLifelineRpcServer() != null) { + rpcServer.getLifelineRpcServer().setLogSlowRPCThresholdTime(logSlowRPCThresholdTime); + } + result = Long.toString(logSlowRPCThresholdTime); + } + LOG.info("RECONFIGURE* changed reconfigureLogSlowRPC {} to {}", property, result); + return result; + } catch (IllegalArgumentException e) { + throw new ReconfigurationException(property, newVal, getConf().get(property), e); + } + } + String reconfigureSPSModeEvent(String newVal, String property) throws ReconfigurationException { if (newVal == null diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java index 63d3a45fff81e..5a0f62a8117e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java @@ -29,6 +29,9 @@ import org.junit.Before; import org.junit.After; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY; import static org.junit.Assert.*; @@ -701,6 +704,49 @@ public void testReconfigureMinBlocksForWrite() throws Exception { assertEquals(3, bm.getMinBlocksForWrite(BlockType.STRIPED)); } + @Test + public void testReconfigureLogSlowRPC() throws ReconfigurationException { + final NameNode nameNode = cluster.getNameNode(); + final NameNodeRpcServer nnrs = (NameNodeRpcServer) nameNode.getRpcServer(); + // verify default value. + assertFalse(nnrs.getClientRpcServer().isLogSlowRPC()); + assertEquals(IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_DEFAULT, + nnrs.getClientRpcServer().getLogSlowRPCThresholdTime()); + + // try invalid logSlowRPC. + try { + nameNode.reconfigurePropertyImpl(IPC_SERVER_LOG_SLOW_RPC, "non-boolean"); + fail("should not reach here"); + } catch (ReconfigurationException e) { + assertEquals( + "Could not change property ipc.server.log.slow.rpc from 'false' to 'non-boolean'", + e.getMessage()); + } + + // try correct logSlowRPC. + nameNode.reconfigurePropertyImpl(IPC_SERVER_LOG_SLOW_RPC, "True"); + assertTrue(nnrs.getClientRpcServer().isLogSlowRPC()); + + // revert to defaults. + nameNode.reconfigurePropertyImpl(IPC_SERVER_LOG_SLOW_RPC, null); + assertFalse(nnrs.getClientRpcServer().isLogSlowRPC()); + + // try invalid logSlowRPCThresholdTime. + try { + nameNode.reconfigureProperty(IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY, + "non-numeric"); + fail("Should not reach here"); + } catch (ReconfigurationException e) { + assertEquals("Could not change property " + + "ipc.server.log.slow.rpc.threshold.ms from '0' to 'non-numeric'", e.getMessage()); + } + + // try correct logSlowRPCThresholdTime. + nameNode.reconfigureProperty(IPC_SERVER_LOG_SLOW_RPC_THRESHOLD_MS_KEY, + "20000"); + assertEquals(nnrs.getClientRpcServer().getLogSlowRPCThresholdTime(), 20000); + } + @After public void shutDown() throws IOException { if (cluster != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 70a8bab8b0905..1712c620d2c82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -442,7 +442,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("namenode", address, outs, errs); - assertEquals(23, outs.size()); + assertEquals(25, outs.size()); assertTrue(outs.get(0).contains("Reconfigurable properties:")); assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1)); assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2)); From bbf905dc99bb8939f61bcb25fe158f56cc826352 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 25 Oct 2023 14:06:13 +0100 Subject: [PATCH 121/155] HADOOP-18933. upgrade to netty 4.1.100 due to CVE (#6173) Mitigates Netty security advisory GHSA-xpw8-rcwv-8f8p "HTTP/2 Rapid Reset Attack - DDoS vector in the HTTP/2 protocol due RST frames" Contributed by PJ Fanning --- LICENSE-binary | 60 +++++++++++++++++++++--------------------- hadoop-project/pom.xml | 2 +- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index c367abdff5742..e2f61dc7cd84c 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -257,36 +257,36 @@ io.grpc:grpc-netty:1.26.0 io.grpc:grpc-protobuf:1.26.0 io.grpc:grpc-protobuf-lite:1.26.0 io.grpc:grpc-stub:1.26.0 -io.netty:netty-all:4.1.94.Final -io.netty:netty-buffer:4.1.94.Final -io.netty:netty-codec:4.1.94.Final -io.netty:netty-codec-dns:4.1.94.Final -io.netty:netty-codec-haproxy:4.1.94.Final -io.netty:netty-codec-http:4.1.94.Final -io.netty:netty-codec-http2:4.1.94.Final -io.netty:netty-codec-memcache:4.1.94.Final -io.netty:netty-codec-mqtt:4.1.94.Final -io.netty:netty-codec-redis:4.1.94.Final -io.netty:netty-codec-smtp:4.1.94.Final -io.netty:netty-codec-socks:4.1.94.Final -io.netty:netty-codec-stomp:4.1.94.Final -io.netty:netty-codec-xml:4.1.94.Final -io.netty:netty-common:4.1.94.Final -io.netty:netty-handler:4.1.94.Final -io.netty:netty-handler-proxy:4.1.94.Final -io.netty:netty-resolver:4.1.94.Final -io.netty:netty-resolver-dns:4.1.94.Final -io.netty:netty-transport:4.1.94.Final -io.netty:netty-transport-rxtx:4.1.94.Final -io.netty:netty-transport-sctp:4.1.94.Final -io.netty:netty-transport-udt:4.1.94.Final -io.netty:netty-transport-classes-epoll:4.1.94.Final -io.netty:netty-transport-native-unix-common:4.1.94.Final -io.netty:netty-transport-classes-kqueue:4.1.94.Final -io.netty:netty-resolver-dns-classes-macos:4.1.94.Final -io.netty:netty-transport-native-epoll:4.1.94.Final -io.netty:netty-transport-native-kqueue:4.1.94.Final -io.netty:netty-resolver-dns-native-macos:4.1.94.Final +io.netty:netty-all:4.1.100.Final +io.netty:netty-buffer:4.1.100.Final +io.netty:netty-codec:4.1.100.Final +io.netty:netty-codec-dns:4.1.100.Final +io.netty:netty-codec-haproxy:4.1.100.Final +io.netty:netty-codec-http:4.1.100.Final +io.netty:netty-codec-http2:4.1.100.Final +io.netty:netty-codec-memcache:4.1.100.Final +io.netty:netty-codec-mqtt:4.1.100.Final +io.netty:netty-codec-redis:4.1.100.Final +io.netty:netty-codec-smtp:4.1.100.Final +io.netty:netty-codec-socks:4.1.100.Final +io.netty:netty-codec-stomp:4.1.100.Final +io.netty:netty-codec-xml:4.1.100.Final +io.netty:netty-common:4.1.100.Final +io.netty:netty-handler:4.1.100.Final +io.netty:netty-handler-proxy:4.1.100.Final +io.netty:netty-resolver:4.1.100.Final +io.netty:netty-resolver-dns:4.1.100.Final +io.netty:netty-transport:4.1.100.Final +io.netty:netty-transport-rxtx:4.1.100.Final +io.netty:netty-transport-sctp:4.1.100.Final +io.netty:netty-transport-udt:4.1.100.Final +io.netty:netty-transport-classes-epoll:4.1.100.Final +io.netty:netty-transport-native-unix-common:4.1.100.Final +io.netty:netty-transport-classes-kqueue:4.1.100.Final +io.netty:netty-resolver-dns-classes-macos:4.1.100.Final +io.netty:netty-transport-native-epoll:4.1.100.Final +io.netty:netty-transport-native-kqueue:4.1.100.Final +io.netty:netty-resolver-dns-native-macos:4.1.100.Final io.opencensus:opencensus-api:0.12.3 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3 io.reactivex:rxjava:1.3.8 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9303d7ff4c807..25e48f293a64a 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -143,7 +143,7 @@ 5.2.0 2.9.0 3.2.4 - 4.1.94.Final + 4.1.100.Final 1.1.10.4 1.7.1 From 8bd1f65efc42d9b93568666f10653b81dd53fc01 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 25 Oct 2023 17:39:16 +0100 Subject: [PATCH 122/155] HADOOP-18948. S3A. Add option fs.s3a.directory.operations.purge.uploads to purge on rename/delete (#6218) S3A directory delete and rename will optionally abort all pending multipart uploads in their under their to-be-deleted paths when. fs.s3a.directory.operations.purge.upload is true It is off by default. The filesystems hasPathCapability("fs.s3a.directory.operations.purge.upload") probe will return true when this feature is enabled. Multipart uploads may accrue from interrupted data writes, uncommitted staging/magic committer jobs and other operations/applications. On AWS S3 lifecycle rules are the recommended way to clean these; this change improves support for stores which lack these rules. Contributed by Steve Loughran --- .../fs/statistics/StoreStatisticNames.java | 7 + .../org/apache/hadoop/fs/s3a/Constants.java | 15 ++ .../apache/hadoop/fs/s3a/MultipartUtils.java | 6 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 152 ++++++++++++---- .../org/apache/hadoop/fs/s3a/Statistic.java | 5 +- .../hadoop/fs/s3a/impl/CallableSupplier.java | 33 ++-- .../hadoop/fs/s3a/impl/DeleteOperation.java | 46 ++++- .../fs/s3a/impl/OperationCallbacks.java | 12 ++ .../hadoop/fs/s3a/impl/RenameOperation.java | 39 ++++- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 +- .../tools/hadoop-aws/third_party_stores.md | 57 +++++- .../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 5 +- .../hadoop/fs/s3a/MultipartTestUtils.java | 7 +- .../hadoop/fs/s3a/auth/ITestAssumeRole.java | 5 +- ...ITestUploadPurgeOnDirectoryOperations.java | 163 ++++++++++++++++++ .../s3a/performance/AbstractS3ACostTest.java | 7 + .../fs/s3a/s3guard/ITestS3GuardTool.java | 19 +- 17 files changed, 499 insertions(+), 83 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java index c04c1bb47fcea..19ee9d1414ecf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java @@ -244,6 +244,13 @@ public final class StoreStatisticNames { public static final String OBJECT_MULTIPART_UPLOAD_ABORTED = "object_multipart_aborted"; + /** + * Object multipart list request. + * Value :{@value}. + */ + public static final String OBJECT_MULTIPART_UPLOAD_LIST = + "object_multipart_list"; + /** * Object put/multipart upload count. * Value :{@value}. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index d69d01f99450f..8b174e92b2911 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1318,4 +1318,19 @@ private Constants() { * The bucket region header. */ public static final String BUCKET_REGION_HEADER = "x-amz-bucket-region"; + + /** + * Should directory operations purge uploads? + * This adds at least one parallelized list operation to the call, + * plus the overhead of deletions. + * Value: {@value}. + */ + public static final String DIRECTORY_OPERATIONS_PURGE_UPLOADS = + "fs.s3a.directory.operations.purge.uploads"; + + /** + * Default value of {@link #DIRECTORY_OPERATIONS_PURGE_UPLOADS}: {@value}. + */ + public static final boolean DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT = false; + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java index efca093204c25..b2057c211da7b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MultipartUtils.java @@ -36,7 +36,7 @@ import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.store.audit.AuditSpan; -import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_LIST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_LIST; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation; @@ -66,7 +66,7 @@ private MultipartUtils() { } * @param maxKeys maximum batch size to request at a time from S3. * @return an iterator of matching uploads */ - static MultipartUtils.UploadIterator listMultipartUploads( + static RemoteIterator listMultipartUploads( final StoreContext storeContext, S3Client s3, @Nullable String prefix, @@ -196,7 +196,7 @@ private void requestNextBatch() throws IOException { listing = invoker.retry("listMultipartUploads", prefix, true, trackDurationOfOperation(storeContext.getInstrumentation(), - MULTIPART_UPLOAD_LIST.getSymbol(), + OBJECT_MULTIPART_UPLOAD_LIST.getSymbol(), () -> s3.listMultipartUploads(requestBuilder.build()))); LOG.debug("Listing found {} upload(s)", listing.uploads().size()); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index d7149d7dead6b..defbcd94a5b14 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -258,6 +258,7 @@ import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.util.Preconditions.checkArgument; +import static org.apache.hadoop.util.functional.RemoteIterators.foreach; import static org.apache.hadoop.util.functional.RemoteIterators.typeCastingRemoteIterator; /** @@ -384,6 +385,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private SignerManager signerManager; private S3AInternals s3aInternals; + /** + * Do directory operations purge pending uploads? + */ + private boolean dirOperationsPurgeUploads; + /** * Page size for deletions. */ @@ -565,6 +571,9 @@ public void initialize(URI name, Configuration originalConf) //check but do not store the block size longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1); enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true); + // should the delete also purge uploads. + dirOperationsPurgeUploads = conf.getBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS, + DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT); this.prefetchEnabled = conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT); long prefetchBlockSizeLong = @@ -1230,7 +1239,7 @@ public void abortOutstandingMultipartUploads(long seconds) purgeBefore); invoker.retry("Purging multipart uploads", bucket, true, () -> { - MultipartUtils.UploadIterator uploadIterator = + RemoteIterator uploadIterator = MultipartUtils.listMultipartUploads(createStoreContext(), s3Client, null, maxKeys); while (uploadIterator.hasNext()) { @@ -2283,12 +2292,14 @@ private long innerRename(Path source, Path dest) // Initiate the rename. // this will call back into this class via the rename callbacks + final StoreContext storeContext = createStoreContext(); RenameOperation renameOperation = new RenameOperation( - createStoreContext(), + storeContext, src, srcKey, p.getLeft(), dst, dstKey, p.getRight(), - new OperationCallbacksImpl(), - pageSize); + new OperationCallbacksImpl(storeContext), + pageSize, + dirOperationsPurgeUploads); return renameOperation.execute(); } @@ -2309,8 +2320,19 @@ private final class OperationCallbacksImpl implements OperationCallbacks { /** Audit Span at time of creation. */ private final AuditSpan auditSpan; - private OperationCallbacksImpl() { - auditSpan = getActiveAuditSpan(); + private final StoreContext storeContext; + + private OperationCallbacksImpl(final StoreContext storeContext) { + this.storeContext = requireNonNull(storeContext); + this.auditSpan = storeContext.getActiveAuditSpan(); + } + + /** + * Get the audit span. + * @return the span + */ + private AuditSpan getAuditSpan() { + return auditSpan; } @Override @@ -2410,7 +2432,29 @@ public RemoteIterator listObjects( Listing.ACCEPT_ALL_BUT_S3N, auditSpan)); } - } + + /** + * Abort multipart uploads under a path. + * @param prefix prefix for uploads to abort + * @return a count of aborts + * @throws IOException trouble; FileNotFoundExceptions are swallowed. + */ + @Override + @Retries.RetryTranslated + public long abortMultipartUploadsUnderPrefix(String prefix) + throws IOException { + getAuditSpan().activate(); + // this deactivates the audit span somehow + final RemoteIterator uploads = + S3AFileSystem.this.listUploadsUnderPrefix(storeContext, prefix); + // so reactivate it. + getAuditSpan().activate(); + return foreach(uploads, upload -> + invoker.retry("Aborting multipart commit", upload.key(), true, () -> + S3AFileSystem.this.abortMultipartUpload(upload))); + } + + } // end OperationCallbacksImpl /** * Callbacks from {@link Listing}. @@ -3371,14 +3415,17 @@ protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOEx // span covers delete, getFileStatus, fake directory operations. try (AuditSpan span = createSpan(INVOCATION_DELETE.getSymbol(), path.toString(), null)) { + // SC will include active span + final StoreContext storeContext = createStoreContext(); boolean outcome = trackDuration(getDurationTrackerFactory(), INVOCATION_DELETE.getSymbol(), new DeleteOperation( - createStoreContext(), + storeContext, innerGetFileStatus(path, true, StatusProbeEnum.ALL), recursive, - new OperationCallbacksImpl(), - pageSize)); + new OperationCallbacksImpl(storeContext), + pageSize, + dirOperationsPurgeUploads)); if (outcome) { try { maybeCreateFakeParentDirectory(path); @@ -5151,13 +5198,39 @@ S3ALocatedFileStatus toLocatedFileStatus(S3AFileStatus status) @InterfaceAudience.Private @Retries.RetryTranslated @AuditEntryPoint - public MultipartUtils.UploadIterator listUploads(@Nullable String prefix) + public RemoteIterator listUploads(@Nullable String prefix) + throws IOException { + // span is picked up retained in the listing. + checkNotClosed(); + try (AuditSpan span = createSpan(MULTIPART_UPLOAD_LIST.getSymbol(), + prefix, null)) { + return listUploadsUnderPrefix(createStoreContext(), prefix); + } + } + + /** + * List any pending multipart uploads whose keys begin with prefix, using + * an iterator that can handle an unlimited number of entries. + * See {@link #listMultipartUploads(String)} for a non-iterator version of + * this. + * @param storeContext store conext. + * @param prefix optional key prefix to search + * @return Iterator over multipart uploads. + * @throws IOException on failure + */ + @InterfaceAudience.Private + @Retries.RetryTranslated + public RemoteIterator listUploadsUnderPrefix( + final StoreContext storeContext, + final @Nullable String prefix) throws IOException { // span is picked up retained in the listing. - return trackDurationAndSpan(MULTIPART_UPLOAD_LIST, prefix, null, () -> - MultipartUtils.listMultipartUploads( - createStoreContext(), s3Client, prefix, maxKeys - )); + String p = prefix; + if (prefix != null && !prefix.isEmpty() && !prefix.endsWith("/")) { + p = prefix + "/"; + } + // duration tracking is done in iterator. + return MultipartUtils.listMultipartUploads(storeContext, s3Client, p, maxKeys); } /** @@ -5179,9 +5252,10 @@ public List listMultipartUploads(String prefix) } String p = prefix; return invoker.retry("listMultipartUploads", p, true, () -> { - ListMultipartUploadsRequest.Builder requestBuilder = getRequestFactory() - .newListMultipartUploadsRequestBuilder(p); - return s3Client.listMultipartUploads(requestBuilder.build()).uploads(); + final ListMultipartUploadsRequest request = getRequestFactory() + .newListMultipartUploadsRequestBuilder(p).build(); + return trackDuration(getInstrumentation(), MULTIPART_UPLOAD_LIST.getSymbol(), () -> + s3Client.listMultipartUploads(request).uploads()); }); } @@ -5190,37 +5264,35 @@ public List listMultipartUploads(String prefix) * Retry policy: none. * @param destKey destination key * @param uploadId Upload ID + * @throws IOException IO failure, including any uprated SdkException */ - @Retries.OnceRaw - void abortMultipartUpload(String destKey, String uploadId) { - LOG.info("Aborting multipart upload {} to {}", uploadId, destKey); - s3Client.abortMultipartUpload( - getRequestFactory().newAbortMultipartUploadRequestBuilder( - destKey, - uploadId).build()); + @Retries.OnceTranslated + public void abortMultipartUpload(String destKey, String uploadId) throws IOException { + LOG.debug("Aborting multipart upload {} to {}", uploadId, destKey); + trackDuration(getInstrumentation(), OBJECT_MULTIPART_UPLOAD_ABORTED.getSymbol(), () -> + s3Client.abortMultipartUpload( + getRequestFactory().newAbortMultipartUploadRequestBuilder( + destKey, + uploadId).build())); } /** * Abort a multipart upload. * Retry policy: none. * @param upload the listed upload to abort. + * @throws IOException IO failure, including any uprated SdkException */ - @Retries.OnceRaw - void abortMultipartUpload(MultipartUpload upload) { - String destKey; - String uploadId; - destKey = upload.key(); - uploadId = upload.uploadId(); - if (LOG.isInfoEnabled()) { + @Retries.OnceTranslated + public void abortMultipartUpload(MultipartUpload upload) throws IOException { + String destKey = upload.key(); + String uploadId = upload.uploadId(); + if (LOG.isDebugEnabled()) { DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); LOG.debug("Aborting multipart upload {} to {} initiated by {} on {}", uploadId, destKey, upload.initiator(), df.format(Date.from(upload.initiated()))); } - s3Client.abortMultipartUpload( - getRequestFactory().newAbortMultipartUploadRequestBuilder( - destKey, - uploadId).build()); + abortMultipartUpload(destKey, uploadId); } /** @@ -5266,13 +5338,17 @@ public boolean hasPathCapability(final Path path, final String capability) case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE: return true; + // Do directory operations purge uploads. + case DIRECTORY_OPERATIONS_PURGE_UPLOADS: + return dirOperationsPurgeUploads; + // etags are avaialable in listings, but they // are not consistent across renames. // therefore, only availability is declared case CommonPathCapabilities.ETAGS_AVAILABLE: return true; - /* + /* * Marker policy capabilities are handed off. */ case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP: @@ -5545,7 +5621,7 @@ public MarkerToolOperations createMarkerToolOperations(final String target) throws IOException { createSpan("marker-tool-scan", target, null); - return new MarkerToolOperationsImpl(new OperationCallbacksImpl()); + return new MarkerToolOperationsImpl(new OperationCallbacksImpl(createStoreContext())); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index f4e28aa62783e..72fc75b642415 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -242,7 +242,10 @@ public enum Statistic { StoreStatisticNames.OBJECT_MULTIPART_UPLOAD_ABORTED, "Object multipart upload aborted", TYPE_DURATION), - OBJECT_PUT_REQUESTS( + OBJECT_MULTIPART_UPLOAD_LIST( + StoreStatisticNames.OBJECT_MULTIPART_UPLOAD_LIST, + "Object multipart list request issued", + TYPE_DURATION), OBJECT_PUT_REQUESTS( StoreStatisticNames.OBJECT_PUT_REQUEST, "Object put/multipart upload count", TYPE_DURATION), diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java index 0156207419210..e0580df08a76d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.List; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; @@ -155,19 +156,21 @@ public static void waitForCompletion( * Wait for a single of future to complete, extracting IOEs afterwards. * @param future future to wait for. * @param type + * @return the result * @throws IOException if one of the called futures raised an IOE. * @throws RuntimeException if one of the futures raised one. */ - public static void waitForCompletion( + public static T waitForCompletion( final CompletableFuture future) throws IOException { try (DurationInfo ignore = new DurationInfo(LOG, false, "Waiting for task completion")) { - future.join(); + return future.join(); } catch (CancellationException e) { throw new IOException(e); } catch (CompletionException e) { raiseInnerCause(e); + return null; } } @@ -175,31 +178,35 @@ public static void waitForCompletion( * Wait for a single of future to complete, ignoring exceptions raised. * @param future future to wait for. * @param type + * @return the outcome if successfully retrieved. */ - public static void waitForCompletionIgnoringExceptions( + public static Optional waitForCompletionIgnoringExceptions( @Nullable final CompletableFuture future) { - if (future != null) { - try (DurationInfo ignore = - new DurationInfo(LOG, false, "Waiting for task completion")) { - future.join(); - } catch (Exception e) { - LOG.debug("Ignoring exception raised in task completion: "); - } + + try { + return maybeAwaitCompletion(future); + } catch (Exception e) { + LOG.debug("Ignoring exception raised in task completion: ", e); + return Optional.empty(); } } /** * Block awaiting completion for any non-null future passed in; * No-op if a null arg was supplied. + * @param return type * @param future future + * @return the outcome; is empty if the future was null/had no return value * @throws IOException if one of the called futures raised an IOE. * @throws RuntimeException if one of the futures raised one. */ - public static void maybeAwaitCompletion( - @Nullable final CompletableFuture future) + public static Optional maybeAwaitCompletion( + @Nullable final CompletableFuture future) throws IOException { if (future != null) { - waitForCompletion(future); + return Optional.ofNullable(waitForCompletion(future)); + } else { + return Optional.empty(); } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java index 314d7cb82d1dd..11e73aeb750ae 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -41,6 +42,7 @@ import org.apache.hadoop.util.DurationInfo; +import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions; import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.maybeAwaitCompletion; @@ -110,6 +112,16 @@ public class DeleteOperation extends ExecutingStoreOperation { */ private long filesDeleted; + /** + * Do directory operations purge pending uploads? + */ + private final boolean dirOperationsPurgeUploads; + + /** + * Count of uploads aborted. + */ + private Optional uploadsAborted = Optional.empty(); + /** * Constructor. * @param context store context @@ -117,12 +129,14 @@ public class DeleteOperation extends ExecutingStoreOperation { * @param recursive recursive delete? * @param callbacks callback provider * @param pageSize size of delete pages + * @param dirOperationsPurgeUploads Do directory operations purge pending uploads? */ public DeleteOperation(final StoreContext context, final S3AFileStatus status, final boolean recursive, final OperationCallbacks callbacks, - final int pageSize) { + final int pageSize, + final boolean dirOperationsPurgeUploads) { super(context); this.status = status; @@ -134,12 +148,22 @@ public DeleteOperation(final StoreContext context, this.pageSize = pageSize; executor = MoreExecutors.listeningDecorator( context.createThrottledExecutor(1)); + this.dirOperationsPurgeUploads = dirOperationsPurgeUploads; } public long getFilesDeleted() { return filesDeleted; } + /** + * Get the count of uploads aborted. + * Non-empty iff enabled, and the operations completed without errors. + * @return count of aborted uploads. + */ + public Optional getUploadsAborted() { + return uploadsAborted; + } + /** * Delete a file or directory tree. *

    @@ -236,6 +260,17 @@ protected void deleteDirectoryTree(final Path path, try (DurationInfo ignored = new DurationInfo(LOG, false, "deleting %s", dirKey)) { + final CompletableFuture abortUploads; + if (dirOperationsPurgeUploads) { + final StoreContext sc = getStoreContext(); + final String key = sc.pathToKey(path) + "/"; + LOG.debug("All uploads under {} will be deleted", key); + abortUploads = submit(sc.getExecutor(), sc.getActiveAuditSpan(), () -> + callbacks.abortMultipartUploadsUnderPrefix(key)); + } else { + abortUploads = null; + } + // init the lists of keys and paths to delete resetDeleteList(); deleteFuture = null; @@ -257,10 +292,10 @@ protected void deleteDirectoryTree(final Path path, LOG.debug("Deleting final batch of listed files"); submitNextBatch(); maybeAwaitCompletion(deleteFuture); - + uploadsAborted = waitForCompletionIgnoringExceptions(abortUploads); } - LOG.debug("Delete \"{}\" completed; deleted {} objects", path, - filesDeleted); + LOG.debug("Delete \"{}\" completed; deleted {} objects and aborted {} uploads", path, + filesDeleted, uploadsAborted.orElse(0L)); } /** @@ -313,7 +348,8 @@ private void submitNextBatch() throws IOException { // delete a single page of keys and the metadata. // block for any previous batch. - maybeAwaitCompletion(deleteFuture); + maybeAwaitCompletion(deleteFuture).ifPresent(count -> + LOG.debug("Deleted {} uploads", count)); // delete the current page of keys and paths deleteFuture = submitDelete(keys); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java index e0d9c7c6aada7..9c88870633a35 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java @@ -164,4 +164,16 @@ RemoteIterator listObjects( Path path, String key) throws IOException; + + /** + * Abort multipart uploads under a path; paged. + * @param prefix prefix for uploads to abort + * @return a count of aborts + * @throws IOException trouble; FileNotFoundExceptions are swallowed. + */ + @Retries.RetryTranslated + default long abortMultipartUploadsUnderPrefix(String prefix) + throws IOException { + return 0; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java index 4bb15f74965a9..288b3c0aae585 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; @@ -44,6 +45,7 @@ import org.apache.hadoop.util.OperationDuration; import static org.apache.hadoop.fs.s3a.S3AUtils.translateException; +import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions; import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion; @@ -124,9 +126,18 @@ public class RenameOperation extends ExecutingStoreOperation { private final List keysToDelete = new ArrayList<>(); + /** + * Do directory operations purge pending uploads? + */ + private final boolean dirOperationsPurgeUploads; + + /** + * Count of uploads aborted. + */ + private Optional uploadsAborted = Optional.empty(); + /** * Initiate the rename. - * * @param storeContext store context * @param sourcePath source path * @param sourceKey key of source @@ -136,6 +147,7 @@ public class RenameOperation extends ExecutingStoreOperation { * @param destStatus destination status. * @param callbacks callback provider * @param pageSize size of delete requests + * @param dirOperationsPurgeUploads Do directory operations purge pending uploads? */ public RenameOperation( final StoreContext storeContext, @@ -146,7 +158,8 @@ public RenameOperation( final String destKey, final S3AFileStatus destStatus, final OperationCallbacks callbacks, - final int pageSize) { + final int pageSize, + final boolean dirOperationsPurgeUploads) { super(storeContext); this.sourcePath = sourcePath; this.sourceKey = sourceKey; @@ -159,6 +172,16 @@ public RenameOperation( && pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE, "page size out of range: %s", pageSize); this.pageSize = pageSize; + this.dirOperationsPurgeUploads = dirOperationsPurgeUploads; + } + + /** + * Get the count of uploads aborted. + * Non-empty iff enabled, and the operations completed without errors. + * @return count of aborted uploads. + */ + public Optional getUploadsAborted() { + return uploadsAborted; } /** @@ -341,6 +364,16 @@ protected void recursiveDirectoryRename() throws IOException { throw new RenameFailedException(srcKey, dstKey, "cannot rename a directory to a subdirectory of itself "); } + // start the async dir cleanup + final CompletableFuture abortUploads; + if (dirOperationsPurgeUploads) { + final String key = srcKey; + LOG.debug("All uploads under {} will be deleted", key); + abortUploads = submit(getStoreContext().getExecutor(), () -> + callbacks.abortMultipartUploadsUnderPrefix(key)); + } else { + abortUploads = null; + } if (destStatus != null && destStatus.isEmptyDirectory() == Tristate.TRUE) { @@ -422,6 +455,8 @@ protected void recursiveDirectoryRename() throws IOException { // have been deleted. completeActiveCopiesAndDeleteSources("final copy and delete"); + // and if uploads were being aborted, wait for that to finish + uploadsAborted = waitForCompletionIgnoringExceptions(abortUploads); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java index 22fc630dad1f5..ea1ea908486e2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java @@ -47,8 +47,8 @@ import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.Constants; -import org.apache.hadoop.fs.s3a.MultipartUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.WriteOperationHelper; import org.apache.hadoop.fs.s3a.auth.RolePolicies; @@ -683,7 +683,7 @@ private void promptBeforeAbort(PrintStream out) throws IOException { private void processUploads(PrintStream out) throws IOException { final S3AFileSystem fs = getFilesystem(); - MultipartUtils.UploadIterator uploads = fs.listUploads(prefix); + RemoteIterator uploads = fs.listUploads(prefix); // create a span so that the write operation helper // is within one AuditSpan span = diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md index a7ea7b2e59024..0216e46014c7e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md @@ -39,11 +39,12 @@ The features which may be unavailable include: * Optional Bucket Probes at startup (`fs.s3a.bucket.probe = 0`). This is now the default -do not change it. * List API to use (`fs.s3a.list.version = 1`) +* Bucket lifecycle rules to clean up pending uploads. ## Configuring s3a to connect to a third party store -### Connecting to a third party object store over HTTPS +## Connecting to a third party object store over HTTPS The core setting for a third party store is to change the endpoint in `fs.s3a.endpoint`. @@ -89,6 +90,57 @@ then these must be set, either in XML or (preferred) in a JCEKS file. If per-bucket settings are used here, then third-party stores and credentials may be used alongside an AWS store. + + +## Other issues + +### Coping without bucket lifecycle rules + +Not all third-party stores support bucket lifecycle rules to clean up buckets +of incomplete uploads. + +This can be addressed in two ways +* Command line: `hadoop s3guard uploads -abort -force \`. +* With `fs.s3a.multipart.purge` and a purge age set in `fs.s3a.multipart.purge.age` +* In rename/delete `fs.s3a.directory.operations.purge.uploads = true`. + +#### S3Guard uploads command + +This can be executed on a schedule, or manually + +``` +hadoop s3guard uploads -abort -force s3a://bucket/ +``` + +Consult the [S3Guard documentation](s3guard.html) for the full set of parameters. + +#### In startup: `fs.s3a.multipart.purge` + +This lists all uploads in a bucket when a filesystem is created and deletes +all of those above a certain age. + +This can hurt performance on a large bucket, as the purge scans the entire tree, +and is executed whenever a filesystem is created -which can happen many times during +hive, spark, distcp jobs. + +For this reason, this option may be deleted in future, however it has long been +available in the S3A client and so guaranteed to work across versions. + +#### During rename and delete: `fs.s3a.directory.operations.purge.uploads` + +When `fs.s3a.directory.operations.purge.uploads` is set, when a directory is renamed +or deleted, then in parallel with the delete an attempt is made to list +all pending uploads. +If there are any, they are aborted (sequentially). + +* This is disabled by default: it adds overhead and extra cost. +* Because it only applies to the directories being processed, directories which + are not renamed or deleted will retain all incomplete uploads. +* There is no age checking: all uploads will be aborted. +* If any other process is writing to the same directory tree, their operations +will be cancelled. + + # Troubleshooting The most common problem when talking to third-party stores are @@ -412,4 +464,5 @@ It is also a way to regression test foundational S3A third-party store compatibi ``` _Note_ If anyone is set up to test this reguarly, please let the hadoop developer team know if regressions do surface, -as it is not a common test configuration. \ No newline at end of file +as it is not a common test configuration. +[] \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java index 263a857e03300..e0559b7c49edc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMultipartUtils.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.store.audit.AuditSpan; import java.io.IOException; @@ -76,7 +77,7 @@ public void testListMultipartUploads() throws Exception { // 2. Verify all uploads are found listing by prefix describe("Verifying upload list by prefix"); - MultipartUtils.UploadIterator uploads = fs.listUploads(getPartPrefix(fs)); + RemoteIterator uploads = fs.listUploads(getPartPrefix(fs)); assertUploadsPresent(uploads, keySet); // 3. Verify all uploads are found listing without prefix @@ -97,7 +98,7 @@ public void testListMultipartUploads() throws Exception { * @param ourUploads set up uploads that should be present * @throws IOException on I/O error */ - private void assertUploadsPresent(MultipartUtils.UploadIterator list, + private void assertUploadsPresent(RemoteIterator list, Set ourUploads) throws IOException { // Don't modify passed-in set, use copy. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java index 3e343a9ea85f2..3f6870be46b2a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java @@ -23,6 +23,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.io.IOUtils; @@ -96,7 +97,7 @@ public static void clearAnyUploads(S3AFileSystem fs, Path path) { String key = fs.pathToKey(path); AuditSpan span = null; try { - MultipartUtils.UploadIterator uploads = fs.listUploads(key); + RemoteIterator uploads = fs.listUploads(key); span = fs.createSpan("multipart", path.toString(), null); final WriteOperationHelper helper = fs.getWriteOperationHelper(); @@ -118,7 +119,7 @@ public static void clearAnyUploads(S3AFileSystem fs, Path path) { public static void assertNoUploadsAt(S3AFileSystem fs, Path path) throws Exception { String key = fs.pathToKey(path); - MultipartUtils.UploadIterator uploads = fs.listUploads(key); + RemoteIterator uploads = fs.listUploads(key); while (uploads.hasNext()) { MultipartUpload upload = uploads.next(); Assert.fail("Found unexpected upload " + upload.key() + " " + @@ -130,7 +131,7 @@ public static void assertNoUploadsAt(S3AFileSystem fs, Path path) throws public static int countUploadsAt(S3AFileSystem fs, Path path) throws IOException { String key = fs.pathToKey(path); - MultipartUtils.UploadIterator uploads = fs.listUploads(key); + RemoteIterator uploads = fs.listUploads(key); int count = 0; while (uploads.hasNext()) { MultipartUpload upload = uploads.next(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 5534bb77c0ddb..12234301b50d8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -27,6 +27,7 @@ import java.util.stream.IntStream; import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.services.s3.model.MultipartUpload; import software.amazon.awssdk.services.sts.model.StsException; import com.fasterxml.jackson.core.JsonProcessingException; import org.assertj.core.api.Assertions; @@ -40,10 +41,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.AWSBadRequestException; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; -import org.apache.hadoop.fs.s3a.MultipartUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestConstants; import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; @@ -463,7 +464,7 @@ public void testReadOnlyOperations() throws Throwable { // list multipart uploads. // This is part of the read policy. int counter = 0; - MultipartUtils.UploadIterator iterator = roleFS.listUploads("/"); + RemoteIterator iterator = roleFS.listUploads("/"); while (iterator.hasNext()) { counter++; iterator.next(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java new file mode 100644 index 0000000000000..9e07027375989 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestUploadPurgeOnDirectoryOperations.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import software.amazon.awssdk.services.s3.model.MultipartUpload; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; +import org.apache.hadoop.fs.store.audit.AuditSpan; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_OPERATIONS_PURGE_UPLOADS; +import static org.apache.hadoop.fs.s3a.MultipartTestUtils.clearAnyUploads; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_LIST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_ABORTED; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_LIST; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; +import static org.apache.hadoop.util.functional.RemoteIterators.toList; + +/** + * Test behavior of purging uploads in rename and delete. + */ +public class ITestUploadPurgeOnDirectoryOperations extends AbstractS3ACostTest { + + @Override + public Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, + DIRECTORY_OPERATIONS_PURGE_UPLOADS, + MAGIC_COMMITTER_ENABLED); + conf.setBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS, true); + conf.setBoolean(MAGIC_COMMITTER_ENABLED, true); + return conf; + } + + @Override + public void setup() throws Exception { + super.setup(); + final S3AFileSystem fs = getFileSystem(); + assertHasPathCapabilities(fs, new Path("/"), + DIRECTORY_OPERATIONS_PURGE_UPLOADS); + clearAnyUploads(fs, methodPath()); + } + + @Test + public void testDeleteWithPendingUpload() throws Throwable { + + final S3AFileSystem fs = getFileSystem(); + final Path dir = methodPath(); + + // create a magic file. + createMagicFile(fs, dir); + + // and there's a pending upload + assertUploadCount(dir, 1); + + // delete the dir, with a cost of 1 abort, 1 list. + verifyMetrics(() -> fs.delete(dir, true), + with(OBJECT_MULTIPART_UPLOAD_ABORTED, 1), // abort + with(OBJECT_MULTIPART_UPLOAD_LIST, 1), // HTTP request inside iterator + with(MULTIPART_UPLOAD_LIST, 0)); // api list call + + + // and the pending upload is gone + assertUploadCount(dir, 0); + } + + @Test + public void testRenameWithPendingUpload() throws Throwable { + + final S3AFileSystem fs = getFileSystem(); + final Path base = methodPath(); + final Path dir = new Path(base, "src"); + final Path dest = new Path(base, "dest"); + + // create a magic file. + createMagicFile(fs, dir); + + // and there's a pending upload + assertUploadCount(dir, 1); + + // rename the dir, with a cost of 1 abort, 1 list. + verifyMetrics(() -> fs.rename(dir, dest), + with(OBJECT_MULTIPART_UPLOAD_ABORTED, 1), // abort + with(OBJECT_MULTIPART_UPLOAD_LIST, 1), // HTTP request inside iterator + with(MULTIPART_UPLOAD_LIST, 0)); // api list call + + // and there isn't + assertUploadCount(dir, 0); + } + + /** + * Create a magic file of "real" length more than 0 bytes long. + * @param fs filesystem + * @param dir directory + * @return the path + * @throws IOException creation failure.p + */ + private static Path createMagicFile(final S3AFileSystem fs, final Path dir) throws IOException { + Path magicFile = new Path(dir, MAGIC_PATH_PREFIX + "001/file.txt"); + createFile(fs, magicFile, true, "123".getBytes(StandardCharsets.UTF_8)); + + // the file exists but is a 0 byte marker file. + assertFileHasLength(fs, magicFile, 0); + return magicFile; + } + + /** + * Assert the upload count under a dir is the expected value. + * Failure message will include the list of entries. + * @param dir dir + * @param expected expected count + * @throws IOException listing problem + */ + private void assertUploadCount(final Path dir, final int expected) throws IOException { + Assertions.assertThat(toList(listUploads(dir))) + .describedAs("uploads under %s", dir) + .hasSize(expected); + } + + /** + * List uploads; use the same APIs that the directory operations use, + * so implicitly validating them. + * @param dir directory to list + * @return full list of entries + * @throws IOException listing problem + */ + private RemoteIterator listUploads(Path dir) throws IOException { + final S3AFileSystem fs = getFileSystem(); + try (AuditSpan ignored = span()) { + final StoreContext sc = fs.createStoreContext(); + return fs.listUploadsUnderPrefix(sc, sc.pathToKey(dir)); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java index 48378ce75dc9c..e37717bfa1e34 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java @@ -91,6 +91,13 @@ protected AbstractS3ACostTest( this.keepMarkers = keepMarkers; } + /** + * Constructor with markers kept. + */ + public AbstractS3ACostTest() { + this(true); + } + @Override public Configuration createConfiguration() { Configuration conf = super.createConfiguration(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java index 844230e8bea9f..28bc2a246af1a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java @@ -97,22 +97,22 @@ public void testStoreInfo() throws Throwable { LOG.info("Exec output=\n{}", output); } - private final static String UPLOAD_PREFIX = "test-upload-prefix"; private final static String UPLOAD_NAME = "test-upload"; @Test public void testUploads() throws Throwable { S3AFileSystem fs = getFileSystem(); - Path path = path(UPLOAD_PREFIX + "/" + UPLOAD_NAME); + Path path = methodPath(); + Path file = new Path(path, UPLOAD_NAME); describe("Cleaning up any leftover uploads from previous runs."); - final String key = fs.pathToKey(path); + final String key = fs.pathToKey(file); try { // 1. Make sure key doesn't already exist clearAnyUploads(fs, path); // 2. Confirm no uploads are listed via API - assertNoUploadsAt(fs, path.getParent()); + assertNoUploadsAt(fs, path); // 3. Confirm no uploads are listed via CLI describe("Confirming CLI lists nothing."); @@ -127,8 +127,6 @@ public void testUploads() throws Throwable { // 6. Confirm part exists via CLI, direct path and parent path describe("Confirming CLI lists one part"); assertNumUploads(path, 1); - assertNumUploads(path.getParent(), 1); - // 7. Use CLI to delete part, assert it worked describe("Deleting part via CLI"); assertNumDeleted(fs, path, 1); @@ -150,22 +148,23 @@ public void testUploads() throws Throwable { @Test public void testUploadListByAge() throws Throwable { S3AFileSystem fs = getFileSystem(); - Path path = path(UPLOAD_PREFIX + "/" + UPLOAD_NAME); + Path path = methodPath(); + Path file = new Path(path, UPLOAD_NAME); describe("Cleaning up any leftover uploads from previous runs."); + // 1. Make sure key doesn't already exist clearAnyUploads(fs, path); // 2. Create a upload part describe("Uploading single part."); - final String key = fs.pathToKey(path); + final String key = fs.pathToKey(file); createPartUpload(fs, key, 128, 1); //try (AuditSpan span = fs.startOperation("multipart", key, null)) { try { - // 3. Confirm it exists via API.. may want to wrap with - // LambdaTestUtils.eventually() ? + // 3. Confirm it exists via API assertEquals("Should be one upload", 1, countUploadsAt(fs, path)); // 4. Confirm part does appear in listing with long age filter From d18410221bd9d5357eba1af8377e82ac27a38c97 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:22:18 +0800 Subject: [PATCH 123/155] YARN-11593. [Federation] Improve command line help information. (#6199) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../hadoop/yarn/client/cli/RouterCLI.java | 294 +++++++++++++----- .../hadoop/yarn/client/cli/TestRouterCLI.java | 26 ++ .../src/site/markdown/Federation.md | 158 +++++++++- 3 files changed, 405 insertions(+), 73 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java index 0aa02c8124a3a..2da584f9d61bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -23,7 +23,10 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.ha.HAAdmin.UsageInfo; @@ -67,6 +70,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.LinkedHashMap; import java.util.stream.Collectors; import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid; @@ -74,29 +78,8 @@ public class RouterCLI extends Configured implements Tool { - private static final Logger LOG = LoggerFactory.getLogger(RouterCLI.class); - protected final static Map ADMIN_USAGE = - ImmutableMap.builder() - // Command1: deregisterSubCluster - .put("-deregisterSubCluster", new UsageInfo( - "[-sc|--subClusterId [subCluster Id]]", - "Deregister SubCluster, If the interval between the heartbeat time of the subCluster " + - "and the current time exceeds the timeout period, " + - "set the state of the subCluster to SC_LOST.")) - // Command2: policy - .put("-policy", new UsageInfo( - "[-s|--save [queue;router weight;amrm weight;headroomalpha]] " + - "[-bs|--batch-save [--format xml] [-f|--input-file fileName]]" + - "[-l|--list [--pageSize][--currentPage][--queue][--queues]]", - "We provide a set of commands for Policy:" + - " Include list policies, save policies, batch save policies. " + - " (Note: The policy type will be directly read from the" + - " yarn.federation.policy-manager in the local yarn-site.xml.)" + - " eg. (routeradmin -policy [-s|--save] root.a;SC-1:0.7,SC-2:0.3;SC-1:0.7,SC-2:0.3;1.0)")) - .build(); - // Common Constant private static final String SEMICOLON = ";"; @@ -104,6 +87,7 @@ public class RouterCLI extends Configured implements Tool { private static final String CMD_EMPTY = ""; private static final int EXIT_SUCCESS = 0; private static final int EXIT_ERROR = -1; + private static final String CMD_HELP = "-help"; // Command1: deregisterSubCluster private static final String DEREGISTER_SUBCLUSTER_TITLE = @@ -115,17 +99,46 @@ public class RouterCLI extends Configured implements Tool { private static final String OPTION_SC = "sc"; private static final String OPTION_SUBCLUSTERID = "subClusterId"; private static final String CMD_DEREGISTERSUBCLUSTER = "-deregisterSubCluster"; - private static final String CMD_HELP = "-help"; + + // DeregisterSubCluster Command Parameters + protected final static UsageInfo SUBCLUSTER_ID = new UsageInfo("<-sc|--subClusterId>", + "'-sc' option allows you to specify the sub-cluster to operate on, " + + "while the '--subClusterId' option is the long format of -sc and serves the same purpose."); + + // DeregisterSubCluster Command Examples + protected final static String DEREGISTER_SUBCLUSTER_EXAMPLE_1 = + "yarn routeradmin -deregisterSubCluster -sc SC-1"; + protected final static String DEREGISTER_SUBCLUSTER_EXAMPLE_2 = + "yarn routeradmin -deregisterSubCluster --subClusterId SC-1"; + + // DeregisterSubCluster Command Help Information + protected final static String DEREGISTER_SUBCLUSTER_HELP_INFO = + "deregister subCluster, If the interval between the heartbeat time of the subCluster and" + + "the current time exceeds the timeout period, set the state of the subCluster to SC_LOST."; + + protected final static RouterCmdUsageInfos DEREGISTER_SUBCLUSTER_USAGEINFOS = + new RouterCmdUsageInfos() + .addUsageInfo(SUBCLUSTER_ID) + .addHelpInfo(DEREGISTER_SUBCLUSTER_HELP_INFO) + .addExampleDescs(CMD_DEREGISTERSUBCLUSTER, "If we want to deregisterSubCluster SC-1") + .addExample(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_EXAMPLE_1) + .addExample(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_EXAMPLE_2); // Command2: policy + + private static final String CMD_POLICY = "-policy"; + // save policy private static final String OPTION_S = "s"; - private static final String OPTION_BATCH_S = "bs"; private static final String OPTION_SAVE = "save"; + // batch save policy + private static final String OPTION_BATCH_S = "bs"; private static final String OPTION_BATCH_SAVE = "batch-save"; private static final String OPTION_FORMAT = "format"; + private static final String FORMAT_XML = "xml"; private static final String OPTION_FILE = "f"; private static final String OPTION_INPUT_FILE = "input-file"; + // list policy private static final String OPTION_L = "l"; private static final String OPTION_LIST = "list"; private static final String OPTION_PAGE_SIZE = "pageSize"; @@ -133,9 +146,6 @@ public class RouterCLI extends Configured implements Tool { private static final String OPTION_QUEUE = "queue"; private static final String OPTION_QUEUES = "queues"; - private static final String CMD_POLICY = "-policy"; - private static final String FORMAT_XML = "xml"; - private static final String FORMAT_JSON = "json"; private static final String XML_TAG_SUBCLUSTERIDINFO = "subClusterIdInfo"; private static final String XML_TAG_AMRMPOLICYWEIGHTS = "amrmPolicyWeights"; private static final String XML_TAG_ROUTERPOLICYWEIGHTS = "routerPolicyWeights"; @@ -146,10 +156,85 @@ public class RouterCLI extends Configured implements Tool { private static final String LIST_POLICIES_TITLE = "Yarn Federation Queue Policies"; + // Columns information private static final List LIST_POLICIES_HEADER = Arrays.asList( "Queue Name", "AMRM Weight", "Router Weight"); + // Policy Commands + protected final static UsageInfo POLICY_SAVE_USAGE = new UsageInfo( + "-s|--save ()", + "This command is used to save the policy information of the queue, " + + "including queue and weight information."); + + protected final static String POLICY_SAVE_USAGE_EXAMPLE_DESC = + "We have two sub-clusters, SC-1 and SC-2. \\" + + "We want to configure a weight policy for the 'root.a' queue. \\" + + "The Router Weight is set to SC-1 with a weight of 0.7 and SC-2 with a weight of 0.3. \\" + + "The AMRM Weight is set SC-1 to 0.6 and SC-2 to 0.4. \\" + + "We are using the default value of 0.1 for headroomalpha."; + + protected final static String POLICY_SAVE_USAGE_EXAMPLE_1 = + "yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + protected final static String POLICY_SAVE_USAGE_EXAMPLE_2 = + "yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0"; + + protected final static UsageInfo POLICY_BATCH_SAVE_USAGE = new UsageInfo( + "-bs|--batch-save (--format ) (-f|--input-file )", + "This command can batch load weight information for queues " + + "based on the provided `federation-weights.xml` file."); + + protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_DESC = + "We have two sub-clusters, SC-1 and SC-2. \\" + + "We would like to configure weights for 'root.a' and 'root.b' queues. \\" + + "We can set the weights for 'root.a' and 'root.b' in the 'federation-weights.xml' file. \\" + + "and then use the batch-save command to save the configurations in bulk."; + + protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_1 = + "yarn routeradmin -policy -bs --format xml -f federation-weights.xml"; + protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_2 = + "yarn routeradmin -policy --batch-save --format xml -f federation-weights.xml"; + + protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo( + "-l|--list [--pageSize][--currentPage][--queue][--queues]", + "This command is used to display the configured queue weight information."); + + protected final static String POLICY_LIST_USAGE_EXAMPLE_DESC = + "We can display the list of already configured queue weight information. \\" + + "We can use the --queue option to query the weight information for a specific queue \\" + + " or use the --queues option to query the weight information for multiple queues. \\"; + + protected final static String POLICY_LIST_USAGE_EXAMPLE_1 = + "yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a"; + + protected final static String POLICY_LIST_USAGE_EXAMPLE_2 = + "yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b"; + + protected final static RouterCmdUsageInfos POLICY_USAGEINFOS = new RouterCmdUsageInfos() + // Policy Save + .addUsageInfo(POLICY_SAVE_USAGE) + .addExampleDescs(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_DESC) + .addExample(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_1) + .addExample(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_2) + // Policy Batch Save + .addUsageInfo(POLICY_BATCH_SAVE_USAGE) + .addExampleDescs(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_DESC) + .addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_1) + .addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_2) + // Policy List Save + .addUsageInfo(POLICY_LIST_USAGE) + .addExampleDescs(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_DESC) + .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_1) + .addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_2); + + protected final static Map ADMIN_USAGE = + ImmutableMap.builder() + // Command1: deregisterSubCluster + .put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_USAGEINFOS) + // Command2: policy + .put(CMD_POLICY, POLICY_USAGEINFOS) + .build(); + public RouterCLI() { super(); } @@ -159,43 +244,66 @@ public RouterCLI(Configuration conf) { } private static void buildHelpMsg(String cmd, StringBuilder builder) { - UsageInfo usageInfo = ADMIN_USAGE.get(cmd); - if (usageInfo == null) { + RouterCmdUsageInfos routerUsageInfo = ADMIN_USAGE.get(cmd); + + if (routerUsageInfo == null) { return; } + builder.append("[").append(cmd).append("]\n"); - if (usageInfo.args != null) { - String space = (usageInfo.args == "") ? "" : " "; - builder.append(" ") - .append(cmd) - .append(space) - .append(usageInfo.args) - .append(": ") - .append(usageInfo.help); - } else { - builder.append(" ") - .append(cmd) - .append(": ") - .append(usageInfo.help); + if (!routerUsageInfo.helpInfos.isEmpty()) { + builder.append("\t Description: \n"); + for (String helpInfo : routerUsageInfo.helpInfos) { + builder.append("\t\t").append(helpInfo).append("\n\n"); + } } - } - private static void buildIndividualUsageMsg(String cmd, StringBuilder builder) { - UsageInfo usageInfo = ADMIN_USAGE.get(cmd); - if (usageInfo == null) { - return; + if (!routerUsageInfo.usageInfos.isEmpty()) { + builder.append("\t UsageInfos: \n"); + for (UsageInfo usageInfo : routerUsageInfo.usageInfos) { + builder.append("\t\t").append(usageInfo.args) + .append(": ") + .append("\n\t\t") + .append(usageInfo.help).append("\n\n"); + } } - if (usageInfo.args == null) { - builder.append("Usage: routeradmin [") - .append(cmd) - .append("]\n"); - } else { - String space = (usageInfo.args == "") ? "" : " "; - builder.append("Usage: routeradmin [") - .append(cmd) - .append(space) - .append(usageInfo.args) - .append("]\n"); + + if (MapUtils.isNotEmpty(routerUsageInfo.examples)) { + builder.append("\t Examples: \n"); + int count = 1; + for (Map.Entry> example : routerUsageInfo.examples.entrySet()) { + + String keyCmd = example.getKey(); + builder.append("\t\t") + .append("Cmd:").append(count) + .append(". ").append(keyCmd) + .append(": \n\n"); + + // Print Command Description + List exampleDescs = routerUsageInfo.exampleDescs.get(keyCmd); + if (CollectionUtils.isNotEmpty(exampleDescs)) { + builder.append("\t\t").append("Cmd Requirement Description:\n"); + for (String value : exampleDescs) { + String[] valueDescs = StringUtils.split(value, "\\"); + for (String valueDesc : valueDescs) { + builder.append("\t\t").append(valueDesc).append("\n"); + } + } + } + + builder.append("\n"); + + // Print Command example + List valueExamples = example.getValue(); + if (CollectionUtils.isNotEmpty(valueExamples)) { + builder.append("\t\t").append("Cmd Examples:\n"); + for (String valueExample : valueExamples) { + builder.append("\t\t").append(valueExample).append("\n"); + } + } + builder.append("\n"); + count++; + } } } @@ -204,12 +312,7 @@ private static void printHelp() { summary.append("routeradmin is the command to execute ") .append("YARN Federation administrative commands.\n") .append("The full syntax is: \n\n") - .append("routeradmin\n") - .append(" [-deregisterSubCluster [-sc|--subClusterId [subCluster Id]]\n") - .append(" [-policy [-s|--save [queue;router weight;amrm weight;headroomalpha] " + - "[-bs|--batch-save [--format xml,json] [-f|--input-file fileName]]] " + - "[-l|--list [--pageSize][--currentPage][--queue][--queues]]\n") - .append(" [-help [cmd]]").append("\n"); + .append("routeradmin\n"); StringBuilder helpBuilder = new StringBuilder(); System.out.println(summary); @@ -235,13 +338,9 @@ protected ResourceManagerAdministrationProtocol createAdminProtocol() private static void buildUsageMsg(StringBuilder builder) { builder.append("routeradmin is only used in Yarn Federation Mode.\n"); builder.append("Usage: routeradmin\n"); - for (Map.Entry cmdEntry : ADMIN_USAGE.entrySet()) { - UsageInfo usageInfo = cmdEntry.getValue(); - builder.append(" ") - .append(cmdEntry.getKey()) - .append(" ") - .append(usageInfo.args) - .append("\n"); + for (String cmdKey : ADMIN_USAGE.keySet()) { + buildHelpMsg(cmdKey, builder); + builder.append("\n"); } builder.append(" -help [cmd]\n"); } @@ -249,7 +348,7 @@ private static void buildUsageMsg(StringBuilder builder) { private static void printUsage(String cmd) { StringBuilder usageBuilder = new StringBuilder(); if (ADMIN_USAGE.containsKey(cmd)) { - buildIndividualUsageMsg(cmd, usageBuilder); + buildHelpMsg(cmd, usageBuilder); } else { buildUsageMsg(usageBuilder); } @@ -353,7 +452,7 @@ private int handlePolicy(String[] args) saveOpt.setOptionalArg(true); Option batchSaveOpt = new Option(OPTION_BATCH_S, OPTION_BATCH_SAVE, false, "We will save queue policies in bulk, " + - "where users can provide XML or JSON files containing the policies. " + + "where users can provide XML files containing the policies. " + "This command will parse the file contents and store the results " + "in the FederationStateStore."); Option formatOpt = new Option(null, "format", true, @@ -748,8 +847,59 @@ public int run(String[] args) throws Exception { return EXIT_SUCCESS; } + public static UsageInfo getPolicyBatchSaveUsage() { + return POLICY_BATCH_SAVE_USAGE; + } + + static class RouterCmdUsageInfos { + private List usageInfos; + private List helpInfos; + private Map> examples; + protected Map> exampleDescs; + + RouterCmdUsageInfos() { + this.usageInfos = new ArrayList<>(); + this.helpInfos = new ArrayList<>(); + this.examples = new LinkedHashMap<>(); + this.exampleDescs = new LinkedHashMap<>(); + } + + public RouterCmdUsageInfos addUsageInfo(UsageInfo usageInfo) { + this.usageInfos.add(usageInfo); + return this; + } + + public RouterCmdUsageInfos addHelpInfo(String helpInfo) { + this.helpInfos.add(helpInfo); + return this; + } + + private RouterCmdUsageInfos addExample(String cmd, String example) { + List exampleList = this.examples.getOrDefault(cmd, new ArrayList<>()); + exampleList.add(example); + this.examples.put(cmd, exampleList); + return this; + } + + private RouterCmdUsageInfos addExampleDescs(String cmd, String exampleDesc) { + List exampleDescList = this.exampleDescs.getOrDefault(cmd, new ArrayList<>()); + exampleDescList.add(exampleDesc); + this.exampleDescs.put(cmd, exampleDescList); + return this; + } + + public Map> getExamples() { + return examples; + } + } + public static void main(String[] args) throws Exception { int result = ToolRunner.run(new RouterCLI(), args); System.exit(result); } + + @VisibleForTesting + public Map getAdminUsage(){ + return ADMIN_USAGE; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java index 6ed83826dfa58..a86878dac3f81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -266,4 +267,29 @@ public void testListPolicies() throws Exception { String[] args = {"-policy", "-l", "--queue", "root.a"}; assertEquals(0, rmAdminCLI.run(args)); } + + @Test + public void testBuildHelpMsg() throws Exception { + Map adminUsage = rmAdminCLI.getAdminUsage(); + assertEquals(2, adminUsage.size()); + + RouterCLI.RouterCmdUsageInfos deregisterSubClusterUsageInfos = + adminUsage.get("-deregisterSubCluster"); + assertNotNull(deregisterSubClusterUsageInfos); + Map> dsExamplesMap = deregisterSubClusterUsageInfos.getExamples(); + assertNotNull(dsExamplesMap); + assertEquals(1, dsExamplesMap.size()); + List dsExamples = dsExamplesMap.get("-deregisterSubCluster"); + assertNotNull(dsExamples); + assertEquals(2, dsExamples.size()); + + RouterCLI.RouterCmdUsageInfos policyUsageInfos = adminUsage.get("-policy"); + assertNotNull(policyUsageInfos); + Map> policyExamplesMap = policyUsageInfos.getExamples(); + assertNotNull(policyExamplesMap); + assertEquals(3, policyExamplesMap.size()); + policyExamplesMap.forEach((cmd, cmdExamples) -> { + assertEquals(2, cmdExamples.size()); + }); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 66c79c94cc9f9..5d5dc786e13b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -465,9 +465,165 @@ If we want to use JCache, we can configure `yarn.federation.cache.class` to `org This is a Cache implemented based on the Guava framework. If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`. +Router command line: + +- deregisterSubCluster + +This command is used to `deregister subCluster`, If the interval between the heartbeat time of the subCluster, and the current time exceeds the timeout period, set the state of the subCluster to `SC_LOST`. + +Uasge: + +`yarn routeradmin -deregisterSubCluster [-sc|--subClusterId ]` + +Options: + +| Property | Description | +|:--------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-sc, --subClusterId [subCluster Id]` | `'-sc' option allows you to specify the sub-cluster to operate on, while the '--subClusterId' option is the long format of -sc and serves the same purpose.` | + +Examples: + +If we want to deregisterSubCluster `SC-1` + +- yarn routeradmin -deregisterSubCluster -sc SC-1 +- yarn routeradmin -deregisterSubCluster --subClusterId SC-1 + +- policy + +We provide a set of commands for Policy Include list policies, save policies, batch save policies. + +Uasge: + +`yarn routeradmin -policy -s|--save (queue;router weight;amrm weight;headroomalpha)` + +`yarn routeradmin -policy -bs|--batch-save (--format xml) (-f|--input-file fileName)` + +`yarn routeradmin -policy -l|--list ([--pageSize][--currentPage][--queue][--queues])` + +- -s|--save () + +This command is used to save the policy information of the queue, including queue and weight information. + +How to configure `queue;router weight;amrm weight;headroomalpha` + +the sum of weights for all sub-clusters in routerWeight/amrmWeight should be 1. + +| Property | Description | +|:----------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `queue` | `Scheduled queue` | +| `router weight` | `Weight for routing applications to different subclusters.` | +| `amrm weight` | `Weight for resource request from ApplicationMaster (AM) to different subclusters' Resource Manager (RM).` | +| `headroomalpha` | `Used by policies that balance weight-based and load-based considerations in their decisions. It is recommended to use 1.0 because the load-base function is not yet complete.` | + +Example: + +We have two sub-clusters, `SC-1` and `SC-2`. We want to configure a weight policy for the `root.a` queue. The Router Weight is set to `SC-1` with a weight of `0.7` and `SC-2` with a weight of `0.3`. +The AMRM Weight is set `SC-1` to `0.6` and `SC-2` to `0.4`. We are using the default value of `0.1` for `headroomalpha`. + +yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 + +yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 + +- -bs|--batch-save (--format xml) (-f|--input-file fileName) + +This command can batch load weight information for queues based on the provided `federation-weights.xml` file. + +| Property | Description | +|:--------------------------|:----------------------------------------------------------------------------------------------| +| `--format [xml]` | `Configuration file format, we currently only support xml format` | +| `-f, --input-file [path]` | `The path to the configuration file. Please use the absolute path of the configuration file.` | + +How to configure `federation-weights.xml` + ```xml + + + + root.a + + + SC-1 + 0.7 + + + SC-2 + 0.3 + + + + + SC-1 + 0.6 + + + SC-2 + 0.4 + + + 1.0 + + + + + root.b + + + SC-1 + 0.8 + + + SC-2 + 0.2 + + + + + SC-1 + 0.6 + + + SC-2 + 0.4 + + + 1.0 + + + + ``` + +Example: + +We have two sub-clusters, `SC-1` and `SC-2`. We would like to configure weights for `root.a` and `root.b` queues. We can set the weights for `root.a` and `root.b` in the `federation-weights.xml` file. +and then use the batch-save command to save the configurations in bulk. + +The file name can be any file name, but it is recommended to use `federation-weights.xml` + +yarn routeradmin -policy -bs --format xml -f /path/federation-weights.xml + +yarn routeradmin -policy --batch-save --format xml -f /path/federation-weights.xml + +- -l|--list (--pageSize --currentPage --queue --queues) + +This command is used to display the configured queue weight information. + +| Property | Description | +|:----------------|:-------------------------------------------------------------| +| `--pageSize` | `The number of policies displayed per page.` | +| `--currentPage` | `This parameter represents the page number to be displayed.` | +| `--queue` | `the queue we need to filter. example: root.a` | +| `--queues` | `list of queues to filter. example: root.a,root.b,root.c` | + +Example: + +We can display the list of already configured queue weight information. We can use the `--queue` option to query the weight information for a specific queue or use the `--queues` option to query the weight information for multiple queues. + +yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a + +yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b + ### ON GPG: -GlobalPolicyGenerator, abbreviated as "GPG," is used for the automatic generation of global policies for subClusters. +GlobalPolicyGenerator, abbreviated as "GPG", is used for the automatic generation of global policies for subClusters. These are extra configurations that should appear in the **conf/yarn-site.xml** for GPG. We allow only one GPG. From 821ed83873572d408d8d923f579cc56371a33001 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 26 Oct 2023 10:35:10 -0700 Subject: [PATCH 124/155] HDFS-15273. CacheReplicationMonitor hold lock for long time and lead to NN out of service. Contributed by Xiaoqiao He. --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 12 +++++++ .../CacheReplicationMonitor.java | 31 +++++++++++++++++++ .../hdfs/server/namenode/CacheManager.java | 28 +++++++++++++++++ .../src/main/resources/hdfs-default.xml | 27 ++++++++++++++++ 4 files changed, 98 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index dd2731813bd77..88a18d9cf0763 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -192,6 +192,18 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.path.based.cache.block.map.allocation.percent"; public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f; + public static final String DFS_NAMENODE_CRM_CHECKLOCKTIME_ENABLE = + "dfs.namenode.crm.checklocktime.enable"; + public static final boolean DFS_NAMENODE_CRM_CHECKLOCKTIME_DEFAULT = false; + + public static final String DFS_NAMENODE_CRM_MAXLOCKTIME_MS = + "dfs.namenode.crm.maxlocktime.ms"; + public static final long DFS_NAMENODE_CRM_MAXLOCKTIME_MS_DEFAULT = 1000; + + public static final String DFS_NAMENODE_CRM_SLEEP_TIME_MS = + "dfs.namenode.crm.sleeptime.ms"; + public static final long DFS_NAMENODE_CRM_SLEEP_TIME_MS_DEFAULT = 300; + public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = HdfsClientConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT; public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index 1e5f952040d53..f9036c550e852 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -140,6 +140,11 @@ public class CacheReplicationMonitor extends Thread implements Closeable { */ private long scannedBlocks; + /** + * Avoid to hold global lock for long times. + */ + private long lastScanTimeMs; + public CacheReplicationMonitor(FSNamesystem namesystem, CacheManager cacheManager, long intervalMs, ReentrantLock lock) { this.namesystem = namesystem; @@ -284,6 +289,7 @@ public void close() throws IOException { private void rescan() throws InterruptedException { scannedDirectives = 0; scannedBlocks = 0; + lastScanTimeMs = Time.monotonicNow(); try { namesystem.writeLock(); try { @@ -315,6 +321,19 @@ private void resetStatistics() { } } + private void reacquireLock(long last) { + long now = Time.monotonicNow(); + if (now - last > cacheManager.getMaxLockTimeMs()) { + try { + namesystem.writeUnlock(); + Thread.sleep(cacheManager.getSleepTimeMs()); + } catch (InterruptedException e) { + } finally { + namesystem.writeLock(); + } + } + } + /** * Scan all CacheDirectives. Use the information to figure out * what cache replication factor each block should have. @@ -447,6 +466,10 @@ private void rescanFile(CacheDirective directive, INodeFile file) { if (cachedTotal == neededTotal) { directive.addFilesCached(1); } + if (cacheManager.isCheckLockTimeEnable()) { + reacquireLock(lastScanTimeMs); + lastScanTimeMs = Time.monotonicNow(); + } LOG.debug("Directive {}: caching {}: {}/{} bytes", directive.getId(), file.getFullPathName(), cachedTotal, neededTotal); } @@ -518,6 +541,10 @@ private void rescanCachedBlockMap() { } } } + if (cacheManager.isCheckLockTimeEnable()) { + reacquireLock(lastScanTimeMs); + lastScanTimeMs = Time.monotonicNow(); + } for (Iterator cbIter = cachedBlocks.iterator(); cbIter.hasNext(); ) { scannedBlocks++; @@ -603,6 +630,10 @@ private void rescanCachedBlockMap() { ); cbIter.remove(); } + if (cacheManager.isCheckLockTimeEnable()) { + reacquireLock(lastScanTimeMs); + lastScanTimeMs = Time.monotonicNow(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index e71b057595952..24ccf45b91d29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -17,6 +17,12 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CRM_CHECKLOCKTIME_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CRM_CHECKLOCKTIME_ENABLE; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CRM_MAXLOCKTIME_MS; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CRM_MAXLOCKTIME_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CRM_SLEEP_TIME_MS; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CRM_SLEEP_TIME_MS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES; @@ -194,6 +200,9 @@ public class CacheManager { * The CacheReplicationMonitor. */ private CacheReplicationMonitor monitor; + private boolean isCheckLockTimeEnable; + private long maxLockTimeMs; + private long sleepTimeMs; public static final class PersistState { public final CacheManagerSection section; @@ -235,12 +244,31 @@ public PersistState(CacheManagerSection section, this.cachedBlocks = enabled ? new LightWeightGSet( LightWeightGSet.computeCapacity(cachedBlocksPercent, "cachedBlocks")) : new LightWeightGSet<>(0); + this.isCheckLockTimeEnable = conf.getBoolean( + DFS_NAMENODE_CRM_CHECKLOCKTIME_ENABLE, + DFS_NAMENODE_CRM_CHECKLOCKTIME_DEFAULT); + this.maxLockTimeMs = conf.getLong(DFS_NAMENODE_CRM_MAXLOCKTIME_MS, + DFS_NAMENODE_CRM_MAXLOCKTIME_MS_DEFAULT); + this.sleepTimeMs = conf.getLong(DFS_NAMENODE_CRM_SLEEP_TIME_MS, + DFS_NAMENODE_CRM_SLEEP_TIME_MS_DEFAULT); } public boolean isEnabled() { return enabled; } + public boolean isCheckLockTimeEnable() { + return isCheckLockTimeEnable; + } + + public long getMaxLockTimeMs() { + return this.maxLockTimeMs; + } + + public long getSleepTimeMs() { + return this.sleepTimeMs; + } + /** * Resets all tracked directives and pools. Called during 2NN checkpointing to * reset FSNamesystem state. See {@link FSNamesystem#clear()}. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index e73fc802a0453..52075a24f1e32 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2940,6 +2940,33 @@ + + dfs.namenode.crm.checklocktime.enable + false + + Set to true to enable CacheManager to check amount of time to hold the + global rwlock. + + + + + dfs.namenode.crm.maxlocktime.ms + 1000 + + The maximum amount of time that CacheManager should hold the global rwlock. + This configuration enable when set `dfs.namenode.crm.checklocktime.enable`. + + + + + dfs.namenode.crm.sleeptime.ms + 300 + + The amount of time that CacheManager should relase the global rwlock. + This configuration enable when set `dfs.namenode.crm.checklocktime.enable`. + + + dfs.datanode.max.locked.memory 0 From 652908519eed5fe79b696e97cc62f2014387be31 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 27 Oct 2023 04:39:06 +0800 Subject: [PATCH 125/155] YARN-11588. [Federation] [Addendum] Fix uncleaned threads in yarn router thread pool executor. (#6222) --- .../hadoop/yarn/conf/YarnConfiguration.java | 16 ++++++++++++++++ .../src/main/resources/yarn-default.xml | 17 +++++++++++++++++ .../clientrm/FederationClientInterceptor.java | 8 +++++++- .../rmadmin/FederationRMAdminInterceptor.java | 14 +++++++++++++- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 90a8978a228b2..2a204519228a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4369,6 +4369,22 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME = TimeUnit.SECONDS.toMillis(0); // 0s + /** + * This method configures the policy for core threads regarding termination + * when no tasks arrive within the keep-alive time. + * When set to false, core threads are never terminated due to a lack of tasks. + * When set to true, the same keep-alive policy + * that applies to non-core threads also applies to core threads. + * To prevent constant thread replacement, + * ensure that the keep-alive time is greater than zero when setting it to true. + * It's advisable to call this method before the pool becomes actively used. + */ + public static final String ROUTER_USER_CLIENT_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT = + ROUTER_PREFIX + "interceptor.user-thread-pool.allow-core-thread-time-out"; + + public static final boolean DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT = + false; + /** The address of the Router web application. */ public static final String ROUTER_WEBAPP_ADDRESS = ROUTER_WEBAPP_PREFIX + "address"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9991e841d74b6..72e8cc70f8743 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5139,6 +5139,23 @@ + + yarn.router.interceptor.user-thread-pool.allow-core-thread-time-out + false + + This method configures the policy for core threads regarding termination + when no tasks arrive within the keep-alive time. + When set to false, core threads are never terminated due to a lack of tasks. + When set to true, the same keep-alive policy + that applies to non-core threads also applies to core threads. + To prevent constant thread replacement, + ensure that the keep-alive time is greater than zero when setting it to true. + It's advisable to call this method before the pool becomes actively used. + We need to ensure that + yarn.router.interceptor.user-thread-pool.keep-alive-time is greater than 0. + + + yarn.router.submit.interval.time 10ms diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 9c3f9971d8c77..35b3e6eeb2bd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -231,7 +231,13 @@ public void init(String userName) { keepAliveTime, TimeUnit.MILLISECONDS, workQueue, threadFactory); // Adding this line so that unused user threads will exit and be cleaned up if idle for too long - this.executorService.allowCoreThreadTimeOut(true); + boolean allowCoreThreadTimeOut = getConf().getBoolean( + YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT); + + if (keepAliveTime > 0 && allowCoreThreadTimeOut) { + this.executorService.allowCoreThreadTimeOut(allowCoreThreadTimeOut); + } final Configuration conf = this.getConf(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index b7c1462a60d56..d269cfe0971cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -130,9 +130,21 @@ public void init(String userName) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("RPC Router RMAdminClient-" + userName + "-%d ").build(); + long keepAliveTime = getConf().getTimeDuration( + YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME, + YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME, TimeUnit.SECONDS); + BlockingQueue workQueue = new LinkedBlockingQueue<>(); this.executorService = new ThreadPoolExecutor(numThreads, numThreads, - 0L, TimeUnit.MILLISECONDS, workQueue, threadFactory); + keepAliveTime, TimeUnit.MILLISECONDS, workQueue, threadFactory); + + boolean allowCoreThreadTimeOut = getConf().getBoolean( + YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT, + YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT); + + if (keepAliveTime > 0 && allowCoreThreadTimeOut) { + this.executorService.allowCoreThreadTimeOut(allowCoreThreadTimeOut); + } federationFacade = FederationStateStoreFacade.getInstance(this.getConf()); this.conf = this.getConf(); From 93a3c6e2cd4db4b395b3ec00a513dd3aceb3e306 Mon Sep 17 00:00:00 2001 From: Hiroaki Segawa Date: Fri, 27 Oct 2023 14:25:00 +0900 Subject: [PATCH 126/155] HDFS-17024. Potential data race introduced by HDFS-15865 (#6223) --- .../main/java/org/apache/hadoop/hdfs/DataStreamer.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 4fa578ab6c03f..d92f5943fd8a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -476,6 +476,7 @@ boolean doWaitForRestart() { private DataOutputStream blockStream; private DataInputStream blockReplyStream; private ResponseProcessor response = null; + private final Object nodesLock = new Object(); private volatile DatanodeInfo[] nodes = null; // list of targets for current block private volatile StorageType[] storageTypes = null; private volatile String[] storageIDs = null; @@ -619,7 +620,9 @@ private void setPipeline(LocatedBlock lb) { private void setPipeline(DatanodeInfo[] nodes, StorageType[] storageTypes, String[] storageIDs) { - this.nodes = nodes; + synchronized (nodesLock) { + this.nodes = nodes; + } this.storageTypes = storageTypes; this.storageIDs = storageIDs; } @@ -916,7 +919,10 @@ void waitForAckedSeqno(long seqno) throws IOException { try (TraceScope ignored = dfsClient.getTracer(). newScope("waitForAckedSeqno")) { LOG.debug("{} waiting for ack for: {}", this, seqno); - int dnodes = nodes != null ? nodes.length : 3; + int dnodes; + synchronized (nodesLock) { + dnodes = nodes != null ? nodes.length : 3; + } int writeTimeout = dfsClient.getDatanodeWriteTimeout(dnodes); long begin = Time.monotonicNow(); try { From 7ec636deec1751341e91453ab8051ab1fe48f37e Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 27 Oct 2023 12:23:55 +0100 Subject: [PATCH 127/155] HADOOP-18930. Make fs.s3a.create.performance a bucket-wide setting. (#6168) If fs.s3a.create.performance is set on a bucket - All file overwrite checks are skipped, even if the caller says otherwise. - All directory existence checks are skipped. - Marker deletion is *always* skipped. This eliminates a HEAD and a LIST for every creation. * New path capability "fs.s3a.create.performance.enabled" true if the option is enabled. * Parameterize ITestS3AContractCreate to expect the different outcomes * Parameterize ITestCreateFileCost similarly, with changed cost assertions there. * create(/) raises an IOE. existing bug only noticed here. Contributed by Steve Loughran --- .../filesystem/fsdataoutputstreambuilder.md | 6 + .../org/apache/hadoop/fs/s3a/Constants.java | 18 ++- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 54 ++++++-- .../hadoop/fs/s3a/impl/CreateFileBuilder.java | 7 ++ .../contract/s3a/ITestS3AContractCreate.java | 93 ++++++++++++++ .../fs/s3a/ITestS3AFSMainOperations.java | 20 +++ .../fs/s3a/ITestS3AFileOperationCost.java | 18 ++- .../fs/s3a/ITestS3AFileSystemContract.java | 20 +++ .../apache/hadoop/fs/s3a/S3ATestUtils.java | 13 ++ .../hadoop/fs/s3a/impl/ITestXAttrCost.java | 7 +- .../s3a/performance/ITestCreateFileCost.java | 118 +++++++++++++++--- .../ITestDirectoryMarkerListing.java | 8 +- .../s3a/performance/ITestS3ADeleteCost.java | 11 ++ .../fs/s3a/tools/AbstractMarkerToolTest.java | 5 +- 14 files changed, 356 insertions(+), 42 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md index ad6d107d06cbc..5f24e75569786 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md @@ -224,6 +224,12 @@ be used as evidence at the inquest as proof that they made a conscious decision to choose speed over safety and that the outcome was their own fault. +Note: the option can be set for an entire filesystem. Again, the safety checks +are there to more closely match the semantics of a classic filesystem, +and to reduce the likelihood that the object store ends up in a state which +diverges so much from the classic directory + tree structur that applications +get confused. + Accordingly: *Use if and only if you are confident that the conditions are met.* ### `fs.s3a.create.header` User-supplied header support diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 8b174e92b2911..f4aeccf1efd10 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1192,12 +1192,26 @@ private Constants() { /** * Flag for create performance. - * This is *not* a configuration option; it is for use in the - * {code createFile()} builder. + * This can be set in the {code createFile()} builder. * Value {@value}. */ public static final String FS_S3A_CREATE_PERFORMANCE = "fs.s3a.create.performance"; + /** + * Default value for create performance in an S3A FS. + * Value {@value}. + */ + public static final boolean FS_S3A_CREATE_PERFORMANCE_DEFAULT = true; + + + /** + * Capability to indicate that the FS has been instantiated with + * {@link #FS_S3A_CREATE_PERFORMANCE} set to true. + * Value {@value}. + */ + public static final String FS_S3A_CREATE_PERFORMANCE_ENABLED = + FS_S3A_CREATE_PERFORMANCE + ".enabled"; + /** * Prefix for adding a header to the object when created. * The actual value must have a "." suffix and then the actual header. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index defbcd94a5b14..f96a378b1cc92 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -235,6 +235,7 @@ import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit; import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE; import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE; +import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION; @@ -348,7 +349,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private S3AStatisticsContext statisticsContext; /** Storage Statistics Bonded to the instrumentation. */ private S3AStorageStatistics storageStatistics; - + /** Should all create files be "performance" unless unset. */ + private boolean performanceCreation; /** * Default input policy; may be overridden in * {@code openFile()}. @@ -660,6 +662,11 @@ public void initialize(URI name, Configuration originalConf) // verify there's no S3Guard in the store config. checkNoS3Guard(this.getUri(), getConf()); + // performance creation flag for code which wants performance + // at the risk of overwrites. + performanceCreation = conf.getBoolean(FS_S3A_CREATE_PERFORMANCE, + FS_S3A_CREATE_PERFORMANCE_DEFAULT); + LOG.debug("{} = {}", FS_S3A_CREATE_PERFORMANCE, performanceCreation); allowAuthoritativePaths = S3Guard.getAuthoritativePaths(this); // directory policy, which may look at authoritative paths @@ -1878,14 +1885,22 @@ public FSDataOutputStream create(Path f, FsPermission permission, Progressable progress) throws IOException { final Path path = qualify(f); + // work out the options to pass down + CreateFileBuilder.CreateFileOptions options; + if (performanceCreation) { + options = OPTIONS_CREATE_FILE_PERFORMANCE; + }else { + options = overwrite + ? OPTIONS_CREATE_FILE_OVERWRITE + : OPTIONS_CREATE_FILE_NO_OVERWRITE; + } + // the span will be picked up inside the output stream return trackDurationAndSpan(INVOCATION_CREATE, path, () -> innerCreateFile(path, progress, getActiveAuditSpan(), - overwrite - ? OPTIONS_CREATE_FILE_OVERWRITE - : OPTIONS_CREATE_FILE_NO_OVERWRITE)); + options)); } /** @@ -1912,14 +1927,19 @@ private FSDataOutputStream innerCreateFile( final CreateFileBuilder.CreateFileOptions options) throws IOException { auditSpan.activate(); String key = pathToKey(path); + if (key.isEmpty()) { + // no matter the creation options, root cannot be written to. + throw new PathIOException("/", "Can't create root path"); + } EnumSet flags = options.getFlags(); - boolean overwrite = flags.contains(CreateFlag.OVERWRITE); - boolean performance = options.isPerformance(); - boolean skipProbes = performance || isUnderMagicCommitPath(path); + + boolean skipProbes = options.isPerformance() || isUnderMagicCommitPath(path); if (skipProbes) { LOG.debug("Skipping existence/overwrite checks"); } else { try { + boolean overwrite = flags.contains(CreateFlag.OVERWRITE); + // get the status or throw an FNFE. // when overwriting, there is no need to look for any existing file, // just a directory (for safety) @@ -1951,7 +1971,7 @@ private FSDataOutputStream innerCreateFile( // put options are derived from the path and the // option builder. - boolean keep = performance || keepDirectoryMarkers(path); + boolean keep = options.isPerformance() || keepDirectoryMarkers(path); final PutObjectOptions putOptions = new PutObjectOptions(keep, null, options.getHeaders()); @@ -2034,11 +2054,14 @@ public FSDataOutputStreamBuilder createFile(final Path path) { final AuditSpan span = entryPoint(INVOCATION_CREATE_FILE, pathToKey(qualified), null); - return new CreateFileBuilder(this, + final CreateFileBuilder builder = new CreateFileBuilder(this, qualified, - new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_FILE, span)) - .create() - .overwrite(true); + new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_FILE, span)); + builder + .create() + .overwrite(true) + .must(FS_S3A_CREATE_PERFORMANCE, performanceCreation); + return builder; } catch (IOException e) { // catch any IOEs raised in span creation and convert to // an UncheckedIOException @@ -2101,7 +2124,8 @@ public FSDataOutputStream createNonRecursive(Path p, .create() .withFlags(flags) .blockSize(blockSize) - .bufferSize(bufferSize); + .bufferSize(bufferSize) + .must(FS_S3A_CREATE_PERFORMANCE, performanceCreation); if (progress != null) { builder.progress(progress); } @@ -5371,6 +5395,10 @@ public boolean hasPathCapability(final Path path, final String capability) case FS_S3A_CREATE_HEADER: return true; + // is the FS configured for create file performance + case FS_S3A_CREATE_PERFORMANCE_ENABLED: + return performanceCreation; + default: return super.hasPathCapability(p, cap); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java index 0392afac59d91..ae2945989ddd3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java @@ -71,6 +71,12 @@ public class CreateFileBuilder extends public static final CreateFileOptions OPTIONS_CREATE_FILE_NO_OVERWRITE = new CreateFileOptions(CREATE_NO_OVERWRITE_FLAGS, true, false, null); + /** + * Performance create options. + */ + public static final CreateFileOptions OPTIONS_CREATE_FILE_PERFORMANCE = + new CreateFileOptions(CREATE_OVERWRITE_FLAGS, true, true, null); + /** * Callback interface. */ @@ -129,6 +135,7 @@ public FSDataOutputStream build() throws IOException { if (flags.contains(CreateFlag.APPEND)) { throw new UnsupportedOperationException("Append is not supported"); } + if (!flags.contains(CreateFlag.CREATE) && !flags.contains(CreateFlag.OVERWRITE)) { throw new PathIOException(path.toString(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java index d2a858f615ef6..7a2a10879dd8e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -18,18 +18,111 @@ package org.apache.hadoop.fs.contract.s3a; +import java.util.Arrays; +import java.util.Collection; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractCreateTest; import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.s3a.S3ATestUtils; + +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; /** * S3A contract tests creating files. + * Parameterized on the create performance flag as all overwrite + * tests are required to fail in create performance mode. */ +@RunWith(Parameterized.class) public class ITestS3AContractCreate extends AbstractContractCreateTest { + /** + * This test suite is parameterized for the different create file + * options. + * @return a list of test parameters. + */ + @Parameterized.Parameters + public static Collection params() { + return Arrays.asList(new Object[][]{ + {false}, + {true} + }); + } + + /** + * Is this test run in create performance mode? + */ + private final boolean createPerformance; + + public ITestS3AContractCreate(final boolean createPerformance) { + this.createPerformance = createPerformance; + } + @Override protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); } + @Override + protected Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, + FS_S3A_CREATE_PERFORMANCE); + conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance); + S3ATestUtils.disableFilesystemCaching(conf); + return conf; + } + + @Override + public void testOverwriteNonEmptyDirectory() throws Throwable { + try { + super.testOverwriteNonEmptyDirectory(); + failWithCreatePerformance(); + } catch (AssertionError e) { + swallowWithCreatePerformance(e); + } + } + + @Override + public void testOverwriteEmptyDirectory() throws Throwable { + try { + super.testOverwriteEmptyDirectory(); + failWithCreatePerformance(); + } catch (AssertionError e) { + swallowWithCreatePerformance(e); + } + } + + @Override + public void testCreateFileOverExistingFileNoOverwrite() throws Throwable { + try { + super.testCreateFileOverExistingFileNoOverwrite(); + failWithCreatePerformance(); + } catch (AssertionError e) { + swallowWithCreatePerformance(e); + } + } + + private void failWithCreatePerformance() { + if (createPerformance) { + fail("expected an assertion error in create performance mode"); + } + } + + /** + * Swallow an assertion error if the create performance flag is set. + * @param e assertion error + */ + private void swallowWithCreatePerformance(final AssertionError e) { + // this is expected in create performance modea + if (!createPerformance) { + // but if the create performance flag is set, then it is supported + // and the assertion error is unexpected + throw e; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java index 6669e8426af0a..013ec901d0a77 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java @@ -18,6 +18,9 @@ package org.apache.hadoop.fs.s3a; +import java.io.IOException; + +import org.assertj.core.api.Assertions; import org.junit.Ignore; import org.apache.hadoop.conf.Configuration; @@ -27,6 +30,7 @@ import org.apache.hadoop.fs.contract.s3a.S3AContract; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; /** * S3A Test suite for the FSMainOperationsBaseTest tests. @@ -70,4 +74,20 @@ public void testCopyToLocalWithUseRawLocalFileSystemOption() throws Exception { } + @Override + public void testOverwrite() throws IOException { + boolean createPerformance = isCreatePerformanceEnabled(fSys); + try { + super.testOverwrite(); + Assertions.assertThat(createPerformance) + .describedAs("create performance enabled") + .isFalse(); + } catch (AssertionError e) { + // swallow the exception if create performance is enabled, + // else rethrow + if (!createPerformance) { + throw e; + } + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index dae6312d48098..0e4a8eda5b297 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.s3a; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; @@ -39,6 +40,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.test.GenericTestUtils.getTestDir; @@ -47,6 +50,9 @@ /** * Use metrics to assert about the cost of file API calls. * Parameterized on directory marker keep vs delete. + * When the FS is instantiated with creation performance, things + * behave differently...its value is that of the marker keep flag, + * so deletion costs are the same. */ @RunWith(Parameterized.class) public class ITestS3AFileOperationCost extends AbstractS3ACostTest { @@ -71,6 +77,14 @@ public ITestS3AFileOperationCost( super(keepMarkers); } + @Override + public Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, FS_S3A_CREATE_PERFORMANCE); + conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers()); + return conf; + } + /** * Test the cost of {@code listLocatedStatus(file)}. */ @@ -377,7 +391,7 @@ public void testCostOfGlobStatus() throws Throwable { // create a bunch of files int filesToCreate = 10; for (int i = 0; i < filesToCreate; i++) { - create(basePath.suffix("/" + i)); + file(basePath.suffix("/" + i)); } fs.globStatus(basePath.suffix("/*")); @@ -396,7 +410,7 @@ public void testCostOfGlobStatusNoSymlinkResolution() throws Throwable { // create a single file, globStatus returning a single file on a pattern // triggers attempts at symlinks resolution if configured String fileName = "/notASymlinkDOntResolveMeLikeOne"; - create(basePath.suffix(fileName)); + file(basePath.suffix(fileName)); // unguarded: 2 head + 1 list from getFileStatus on path, // plus 1 list to match the glob pattern // no additional operations from symlink resolution diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 7ce7b8385cec4..56827043c9b82 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -19,7 +19,9 @@ package org.apache.hadoop.fs.s3a; import java.io.FileNotFoundException; +import java.io.IOException; +import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -32,6 +34,7 @@ import org.apache.hadoop.fs.Path; import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.*; import static org.junit.Assert.*; @@ -137,4 +140,21 @@ public void testRenameNonExistentPath() throws Exception { () -> super.testRenameNonExistentPath()); } + + @Override + public void testOverwrite() throws IOException { + boolean createPerformance = isCreatePerformanceEnabled(fs); + try { + super.testOverwrite(); + Assertions.assertThat(createPerformance) + .describedAs("create performance enabled") + .isFalse(); + } catch (AssertionError e) { + // swallow the exception if create performance is enabled, + // else rethrow + if (!createPerformance) { + throw e; + } + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 3382c300b9315..aa38186c65032 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -1554,4 +1554,17 @@ public static boolean isBulkDeleteEnabled(FileSystem fs) { return fs.getConf().getBoolean(Constants.ENABLE_MULTI_DELETE, true); } + + /** + * Does this FS have create performance enabled? + * @param fs filesystem + * @return true if create performance is enabled + * @throws IOException IO problems + */ + public static boolean isCreatePerformanceEnabled(FileSystem fs) + throws IOException { + return fs.hasPathCapability(new Path("/"), FS_S3A_CREATE_PERFORMANCE_ENABLED); + } + + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java index f6fe68b8e6d33..71fdb7aeaa62c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_OP_XATTR_LIST; import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_MAP; import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_XATTR_GET_NAMED; @@ -43,6 +44,7 @@ import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_STANDARD_HEADERS; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.decodeBytes; import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_OVERWRITE; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST; /** * Invoke XAttr API calls against objects in S3 and validate header @@ -95,8 +97,11 @@ private void logXAttrs(final Map xAttrs) { public void testXAttrFile() throws Throwable { describe("Test xattr on a file"); Path testFile = methodPath(); - create(testFile, true, CREATE_FILE_OVERWRITE); S3AFileSystem fs = getFileSystem(); + boolean createPerformance = isCreatePerformanceEnabled(fs); + + create(testFile, true, + createPerformance ? NO_HEAD_OR_LIST : CREATE_FILE_OVERWRITE); Map xAttrs = verifyMetrics(() -> fs.getXAttrs(testFile), with(INVOCATION_XATTR_GET_MAP, GET_METADATA_ON_OBJECT)); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java index 39530d97bf794..2d128cffc5af0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java @@ -19,16 +19,22 @@ package org.apache.hadoop.fs.s3a.performance; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import org.assertj.core.api.Assertions; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStreamBuilder; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; import static java.util.Objects.requireNonNull; @@ -36,6 +42,7 @@ import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE; @@ -45,6 +52,7 @@ import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_DIR_MARKER; import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_FILE; import static org.apache.hadoop.fs.s3a.performance.OperationCost.HEAD_OPERATION; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_OPERATION; import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST; /** @@ -52,13 +60,55 @@ * with the FS_S3A_CREATE_PERFORMANCE option. */ @SuppressWarnings("resource") +@RunWith(Parameterized.class) public class ITestCreateFileCost extends AbstractS3ACostTest { + /** + * This test suite is parameterized for the different create file + * options. + * @return a list of test parameters. + */ + @Parameterized.Parameters + public static Collection params() { + return Arrays.asList(new Object[][]{ + {false}, + {true} + }); + } + + /** + * Flag for performance creation; all cost asserts need changing. + */ + private final boolean createPerformance; + /** * Create with markers kept, always. */ - public ITestCreateFileCost() { + public ITestCreateFileCost(final boolean createPerformance) { + // keep markers to permit assertions that create performance + // always skips marker deletion. super(false); + this.createPerformance = createPerformance; + } + + /** + * Determine the expected cost of a create operation; + * if {@link #createPerformance} is true, then the cost is always "no IO". + * @param source source cost + * @return cost to assert + */ + private OperationCost expected(OperationCost source) { + return createPerformance ? NO_HEAD_OR_LIST : source; + } + + @Override + public Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, + FS_S3A_CREATE_PERFORMANCE); + conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance); + S3ATestUtils.disableFilesystemCaching(conf); + return conf; } @Test @@ -67,7 +117,7 @@ public void testCreateNoOverwrite() throws Throwable { Path testFile = methodPath(); // when overwrite is false, the path is checked for existence. create(testFile, false, - CREATE_FILE_NO_OVERWRITE); + expected(CREATE_FILE_NO_OVERWRITE)); } @Test @@ -75,7 +125,7 @@ public void testCreateOverwrite() throws Throwable { describe("Test file creation with overwrite"); Path testFile = methodPath(); // when overwrite is true: only the directory checks take place. - create(testFile, true, CREATE_FILE_OVERWRITE); + create(testFile, true, expected(CREATE_FILE_OVERWRITE)); } @Test @@ -85,21 +135,45 @@ public void testCreateNoOverwriteFileExists() throws Throwable { // now there is a file there, an attempt with overwrite == false will // fail on the first HEAD. - interceptOperation(FileAlreadyExistsException.class, "", - FILE_STATUS_FILE_PROBE, - () -> file(testFile, false)); + if (!createPerformance) { + interceptOperation(FileAlreadyExistsException.class, "", + FILE_STATUS_FILE_PROBE, + () -> file(testFile, false)); + } else { + create(testFile, false, NO_HEAD_OR_LIST); + } } @Test - public void testCreateFileOverDir() throws Throwable { - describe("Test cost of create file failing with existing dir"); + public void testCreateFileOverDirNoOverwrite() throws Throwable { + describe("Test cost of create file overwrite=false failing with existing dir"); Path testFile = dir(methodPath()); - // now there is a file there, an attempt with overwrite == false will + // now there is a dir marker there, an attempt with overwrite == true will // fail on the first HEAD. - interceptOperation(FileAlreadyExistsException.class, "", - GET_FILE_STATUS_ON_DIR_MARKER, - () -> file(testFile, false)); + if (!createPerformance) { + interceptOperation(FileAlreadyExistsException.class, "", + GET_FILE_STATUS_ON_DIR_MARKER, + () -> file(testFile, false)); + } else { + create(testFile, false, NO_HEAD_OR_LIST); + } + } + + @Test + public void testCreateFileOverDirWithOverwrite() throws Throwable { + describe("Test cost of create file overwrite=false failing with existing dir"); + Path testFile = dir(methodPath()); + + // now there is a dir marker there, an attempt with overwrite == true will + // fail on the LIST; no HEAD is issued. + if (!createPerformance) { + interceptOperation(FileAlreadyExistsException.class, "", + LIST_OPERATION, + () -> file(testFile, true)); + } else { + create(testFile, true, NO_HEAD_OR_LIST); + } } /** @@ -117,14 +191,18 @@ public void testCreateBuilderSequence() throws Throwable { // files and so briefly the path not being present // only make sure the dest path isn't a directory. buildFile(testFile, true, false, - FILE_STATUS_DIR_PROBE); + expected(FILE_STATUS_DIR_PROBE)); // now there is a file there, an attempt with overwrite == false will // fail on the first HEAD. - interceptOperation(FileAlreadyExistsException.class, "", - GET_FILE_STATUS_ON_FILE, - () -> buildFile(testFile, false, true, - GET_FILE_STATUS_ON_FILE)); + if (!createPerformance) { + interceptOperation(FileAlreadyExistsException.class, "", + GET_FILE_STATUS_ON_FILE, + () -> buildFile(testFile, false, true, + GET_FILE_STATUS_ON_FILE)); + } else { + buildFile(testFile, false, true, NO_HEAD_OR_LIST); + } } @Test @@ -162,7 +240,7 @@ public void testCreateFileRecursive() throws Throwable { builder.must(FS_S3A_CREATE_HEADER + ".h1", custom); verifyMetrics(() -> build(builder), - always(CREATE_FILE_NO_OVERWRITE)); + always(expected(CREATE_FILE_NO_OVERWRITE))); // the header is there and the probe should be a single HEAD call. String header = verifyMetrics(() -> @@ -181,7 +259,7 @@ public void testCreateFileNonRecursive() throws Throwable { verifyMetrics(() -> build(fs.createFile(methodPath()).overwrite(true)), - always(CREATE_FILE_OVERWRITE)); + always(expected(CREATE_FILE_OVERWRITE))); } @@ -196,7 +274,7 @@ public void testCreateNonRecursive() throws Throwable { .close(); return ""; }, - always(CREATE_FILE_OVERWRITE)); + always(expected(CREATE_FILE_OVERWRITE))); } private FSDataOutputStream build(final FSDataOutputStreamBuilder builder) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java index cd4ee44406676..8dbf5baaa6206 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java @@ -54,6 +54,7 @@ import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY; import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE; import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP; +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -80,8 +81,7 @@ *

    * Similarly: JUnit assertions over AssertJ. *

    - * The tests work with unguarded buckets only -the bucket settings are changed - * appropriately. + * s3a create performance is disabled for consistent assertions. */ @RunWith(Parameterized.class) public class ITestDirectoryMarkerListing extends AbstractS3ATestBase { @@ -199,11 +199,13 @@ protected Configuration createConfiguration() { // directory marker options removeBaseAndBucketOverrides(bucketName, conf, - DIRECTORY_MARKER_POLICY); + DIRECTORY_MARKER_POLICY, + FS_S3A_CREATE_PERFORMANCE); conf.set(DIRECTORY_MARKER_POLICY, keepMarkers ? DIRECTORY_MARKER_POLICY_KEEP : DIRECTORY_MARKER_POLICY_DELETE); + conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index fae2df973af7f..97f51fe2c8dcd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; @@ -38,6 +39,8 @@ import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe; @@ -74,6 +77,14 @@ public ITestS3ADeleteCost(final String name, super(keepMarkers); } + @Override + public Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, FS_S3A_CREATE_PERFORMANCE); + conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false); + return conf; + } + @Override public void teardown() throws Exception { if (isKeepingMarkers()) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java index 0c2473a5f61da..759a3bf129eef 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java @@ -73,12 +73,15 @@ protected Configuration createConfiguration() { removeBaseAndBucketOverrides(bucketName, conf, S3A_BUCKET_PROBE, DIRECTORY_MARKER_POLICY, - AUTHORITATIVE_PATH); + AUTHORITATIVE_PATH, + FS_S3A_CREATE_PERFORMANCE); // base FS is legacy conf.set(DIRECTORY_MARKER_POLICY, DIRECTORY_MARKER_POLICY_DELETE); + conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false); // turn off bucket probes for a bit of speedup in the connectors we create. conf.setInt(S3A_BUCKET_PROBE, 0); + return conf; } From e4eda40ac9966a7d8488bbb0630eaa02c3f2b0e0 Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Fri, 27 Oct 2023 22:11:01 +0800 Subject: [PATCH 128/155] YARN-11597. Fix NPE when loading static files in SLSWebApp (#6216) Contributed by Junfan Zhang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java index a2974615c4c67..ce36854bca2fd 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java @@ -136,6 +136,7 @@ public void start() throws Exception { String webRootDir = getClass().getClassLoader().getResource("html"). toExternalForm(); staticHandler.setResourceBase(webRootDir); + staticHandler.start(); Handler handler = new AbstractHandler() { @Override From 40e8300719124b3e1e08d5ddbeb2270ac83b892c Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 28 Oct 2023 07:09:09 +0800 Subject: [PATCH 129/155] YARN-11592. Add timeout to GPGUtils#invokeRMWebService. (#6189) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../hadoop/yarn/conf/YarnConfiguration.java | 5 ++++ .../src/main/resources/yarn-default.xml | 23 +++++++++++++++++++ .../globalpolicygenerator/GPGUtils.java | 21 ++++++++++++++++- .../TestGPGPolicyFacade.java | 3 +-- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2a204519228a4..d195c8cdae681 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4565,6 +4565,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT; + public static final String GPG_WEBAPP_CONNECT_TIMEOUT = GPG_WEBAPP_PREFIX + "connect-timeout"; + public static final long DEFAULT_GPG_WEBAPP_CONNECT_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + public static final String GPG_WEBAPP_READ_TIMEOUT = GPG_WEBAPP_PREFIX + "read-timeout"; + public static final long DEFAULT_GPG_WEBAPP_READ_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + /** * Connection and Read timeout from the Router to RM. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 72e8cc70f8743..c2be5d420bfcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5723,4 +5723,27 @@ 50000     + + + Set the connect timeout interval, in milliseconds. + A value of 0 means no timeout, otherwise values must be between 1 and Integer#MAX_VALUE. + This ensures that the connection is established within a specified time, + or it triggers a connection timeout exception. + + yarn.federation.gpg.webapp.connect-timeout + 30s + + + + + Set the read timeout interval, in milliseconds. + A value of 0 means no timeout, otherwise values must be between 1 and Integer#MAX_VALUE. + This timeout specifies the maximum time a client should wait for data to be read from a server + once the connection has been established. + If data is not received within the specified time, a read timeout exception is raised. + + yarn.federation.gpg.webapp.read-timeout + 30s + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java index 02344a51493c6..13e1a8a01273f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.ws.rs.core.MediaType; @@ -65,7 +66,7 @@ private GPGUtils() { */ public static T invokeRMWebService(String webAddr, String path, final Class returnType, Configuration conf, String selectParam) { - Client client = Client.create(); + Client client = createJerseyClient(conf); T obj; // webAddr stores the form of host:port in subClusterInfo @@ -128,4 +129,22 @@ public static Map createUniformWeights( } return weights; } + + /** + * Create JerseyClient based on configuration file. + * We will set the timeout when creating JerseyClient. + * + * @param conf Configuration. + * @return JerseyClient. + */ + public static Client createJerseyClient(Configuration conf) { + Client client = Client.create(); + int connectTimeOut = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_CONNECT_TIMEOUT, + YarnConfiguration.DEFAULT_GPG_WEBAPP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS); + client.setConnectTimeout(connectTimeOut); + int readTimeout = (int) conf.getTimeDuration(YarnConfiguration.GPG_WEBAPP_READ_TIMEOUT, + YarnConfiguration.DEFAULT_GPG_WEBAPP_READ_TIMEOUT, TimeUnit.MILLISECONDS); + client.setReadTimeout(readTimeout); + return client; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java index d20d6c0485f19..d5408dfdafbc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java @@ -42,7 +42,6 @@ import org.junit.Test; import org.mockito.Matchers; -import java.io.IOException; import java.util.HashSet; import java.util.Set; @@ -73,7 +72,7 @@ public TestGPGPolicyFacade() { } @Before - public void setUp() throws IOException, YarnException { + public void setUp() throws YarnException { stateStore = new MemoryFederationStateStore(); stateStore.init(conf); facade.reinitialize(stateStore, conf); From b9c9c42b291187cdf7a4f1d2ae959e31367ec95b Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Sun, 29 Oct 2023 07:39:12 +0000 Subject: [PATCH 130/155] HADOOP-18936. Upgrade to jetty 9.4.53 (#6181). Contributed by PJ Fanning. Signed-off-by: Ayush Saxena --- LICENSE-binary | 28 ++++++++++++++-------------- hadoop-project/pom.xml | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index e2f61dc7cd84c..3a0e19c5824de 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -339,20 +339,20 @@ org.apache.solr:solr-solrj:8.11.2 org.apache.yetus:audience-annotations:0.5.0 org.apache.zookeeper:zookeeper:3.7.2 org.codehaus.jettison:jettison:1.5.4 -org.eclipse.jetty:jetty-annotations:9.4.51.v20230217 -org.eclipse.jetty:jetty-http:9.4.51.v20230217 -org.eclipse.jetty:jetty-io:9.4.51.v20230217 -org.eclipse.jetty:jetty-jndi:9.4.51.v20230217 -org.eclipse.jetty:jetty-plus:9.4.51.v20230217 -org.eclipse.jetty:jetty-security:9.4.51.v20230217 -org.eclipse.jetty:jetty-server:9.4.51.v20230217 -org.eclipse.jetty:jetty-servlet:9.4.51.v20230217 -org.eclipse.jetty:jetty-util:9.4.51.v20230217 -org.eclipse.jetty:jetty-util-ajax:9.4.51.v20230217 -org.eclipse.jetty:jetty-webapp:9.4.51.v20230217 -org.eclipse.jetty:jetty-xml:9.4.51.v20230217 -org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217 -org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217 +org.eclipse.jetty:jetty-annotations:9.4.53.v20231009 +org.eclipse.jetty:jetty-http:9.4.53.v20231009 +org.eclipse.jetty:jetty-io:9.4.53.v20231009 +org.eclipse.jetty:jetty-jndi:9.4.53.v20231009 +org.eclipse.jetty:jetty-plus:9.4.53.v20231009 +org.eclipse.jetty:jetty-security:9.4.53.v20231009 +org.eclipse.jetty:jetty-server:9.4.53.v20231009 +org.eclipse.jetty:jetty-servlet:9.4.53.v20231009 +org.eclipse.jetty:jetty-util:9.4.53.v20231009 +org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009 +org.eclipse.jetty:jetty-webapp:9.4.53.v20231009 +org.eclipse.jetty:jetty-xml:9.4.53.v20231009 +org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.53.v20231009 +org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.53.v20231009 org.ehcache:ehcache:3.3.1 org.ini4j:ini4j:0.5.4 org.lz4:lz4-java:1.7.1 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 25e48f293a64a..5b1c569d21de6 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.51.v20230217 + 9.4.53.v20231009 _ _ From 7c6af6a5f626d18d68b656d085cc23e4c1f7a1ef Mon Sep 17 00:00:00 2001 From: ConfX <114765570+teamconfx@users.noreply.github.com> Date: Sun, 29 Oct 2023 16:00:28 +0800 Subject: [PATCH 131/155] HADOOP-18905. Negative timeout in ZKFailovercontroller due to overflow. (#6092). Contributed by ConfX. Reviewed-by: Inigo Goiri Signed-off-by: Ayush Saxena --- .../src/main/java/org/apache/hadoop/ha/ZKFailoverController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java index 487d7b9409159..91f720a49eed9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java @@ -660,6 +660,7 @@ public Void run() throws Exception { private void doGracefulFailover() throws ServiceFailedException, IOException, InterruptedException { int timeout = FailoverController.getGracefulFenceTimeout(conf) * 2; + Preconditions.checkArgument(timeout >= 0, "timeout should be non-negative."); // Phase 1: pre-flight checks checkEligibleForFailover(); From a079f6261d77512a4eeb9a1d10e667caaecde29c Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Mon, 30 Oct 2023 04:05:02 +0000 Subject: [PATCH 132/155] HADOOP-18917. Addendum. Fix deprecation issues after commons-io upgrade. (#6228). Contributed by PJ Fanning. --- .../hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java | 2 +- .../tools/offlineImageViewer/TestOfflineImageViewer.java | 2 +- .../java/org/apache/hadoop/tools/HadoopArchiveLogs.java | 6 +++++- .../hadoop/yarn/logaggregation/AggregatedLogFormat.java | 6 +++++- .../yarn/server/timeline/TestLeveldbTimelineStore.java | 5 +++-- .../server/timeline/TestRollingLevelDBTimelineStore.java | 5 +++-- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java index 107333c5a63c3..3a3898727fc10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java @@ -257,7 +257,7 @@ public void testOfflineImageViewer() throws Exception { FSImageTestUtil.getFSImage( cluster.getNameNode()).getStorage().getStorageDir(0)); assertNotNull("Didn't generate or can't find fsimage", originalFsimage); - PrintStream o = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM); + PrintStream o = new PrintStream(NullOutputStream.INSTANCE); PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); v.visit(new RandomAccessFile(originalFsimage, "r")); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index c24c9132cbcd5..b2112a74e8553 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -405,7 +405,7 @@ private static FileStatus pathToFileEntry(FileSystem hdfs, String file) @Test(expected = IOException.class) public void testTruncatedFSImage() throws IOException { File truncatedFile = new File(tempDir, "truncatedFsImage"); - PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM); + PrintStream output = new PrintStream(NullOutputStream.INSTANCE); copyPartOfFile(originalFsimage, truncatedFile); try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) { new FileDistributionCalculator(new Configuration(), 0, 0, false, output) diff --git a/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java index f745a2e519e3c..9b28ca406d693 100644 --- a/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java +++ b/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java @@ -54,6 +54,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -505,7 +506,10 @@ void generateScript(File localScript) throws IOException { String classpath = halrJarPath + File.pathSeparator + harJarPath; FileWriterWithEncoding fw = null; try { - fw = new FileWriterWithEncoding(localScript, "UTF-8"); + fw = FileWriterWithEncoding.builder() + .setFile(localScript) + .setCharset(StandardCharsets.UTF_8) + .get(); fw.write("#!/bin/bash\nset -e\nset -x\n"); int containerCount = 1; for (AppInfo context : eligibleApplications) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 477a8a293cebf..26c3e01a45d03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -32,6 +32,7 @@ import java.io.PrintStream; import java.io.Writer; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; @@ -783,7 +784,10 @@ public static void readAcontainerLogs(DataInputStream valueStream, OutputStream os = null; PrintStream ps = null; try { - os = new WriterOutputStream(writer, Charset.forName("UTF-8")); + os = WriterOutputStream.builder() + .setWriter(writer) + .setCharset(StandardCharsets.UTF_8) + .get(); ps = new PrintStream(os); while (true) { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java index c7166a1268c55..1e550890fda8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java @@ -496,8 +496,9 @@ void testLevelDbRepair() throws IOException { store.init(conf); Mockito.verify(factory, Mockito.times(1)) .repair(Mockito.any(File.class), Mockito.any(Options.class)); - FileFilter fileFilter = new WildcardFileFilter( - "*" + LeveldbTimelineStore.BACKUP_EXT + "*"); + FileFilter fileFilter = WildcardFileFilter.builder() + .setWildcards("*" + LeveldbTimelineStore.BACKUP_EXT +"*") + .get(); assertTrue(path.listFiles(fileFilter).length > 0); } finally { store.close(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java index b20497728bab1..6d37a9730e1d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java @@ -444,8 +444,9 @@ void testLevelDbRepair() throws IOException { store.init(conf); Mockito.verify(factory, Mockito.times(1)) .repair(Mockito.any(File.class), Mockito.any(Options.class)); - FilenameFilter fileFilter = - new WildcardFileFilter("*" + RollingLevelDBTimelineStore.BACKUP_EXT + "*"); + FilenameFilter fileFilter = WildcardFileFilter.builder() + .setWildcards("*" + RollingLevelDBTimelineStore.BACKUP_EXT + "*") + .get(); assertTrue(new File(path.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME) .list(fileFilter).length > 0); } finally { From 254dbab5a323b29897a4e90f3502dc32ccc30688 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 31 Oct 2023 06:56:00 +0800 Subject: [PATCH 133/155] YARN-9013. [BackPort] [GPG] fix order of steps cleaning Registry entries in ApplicationCleaner. (#6147) Contributed by Botong Huang, Shilun Fan. Co-authored-by: Botong Huang Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../src/main/resources/yarn-default.xml | 4 +- .../ApplicationCleaner.java | 17 ++------ .../DefaultApplicationCleaner.java | 41 +++++++++++++------ .../TestDefaultApplicationCleaner.java | 39 ++++++++++++++++++ 4 files changed, 74 insertions(+), 27 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c2be5d420bfcf..0e317712f826c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5132,10 +5132,10 @@ yarn.router.interceptor.user-thread-pool.keep-alive-time - 0s + 30s This configurable is used to set the keepAliveTime of the thread pool of the interceptor. - Default is 0s. + Default is 30s. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java index af0bd6184b797..76380af8c986e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/ApplicationCleaner.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.apache.commons.lang3.time.DurationFormatUtils; @@ -95,6 +94,10 @@ public GPGContext getGPGContext() { return this.gpgContext; } + public FederationRegistryClient getRegistryClient() { + return this.registryClient; + } + /** * Query router for applications. * @@ -152,18 +155,6 @@ public Set getRouterKnownApplications() throws YarnException { + " success Router queries after " + totalAttemptCount + " retries"); } - protected void cleanupAppRecordInRegistry(Set knownApps) { - List allApps = this.registryClient.getAllApplications(); - LOG.info("Got {} existing apps in registry.", allApps.size()); - for (String app : allApps) { - ApplicationId appId = ApplicationId.fromString(app); - if (!knownApps.contains(appId)) { - LOG.info("removing finished application entry for {}", app); - this.registryClient.removeAppFromRegistry(appId, true); - } - } - } - @Override public abstract void run(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java index 5b2ff26fcfb4d..c3f79d0284c5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/DefaultApplicationCleaner.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; @@ -45,33 +46,49 @@ public void run() { LOG.info("Application cleaner run at time {}", now); FederationStateStoreFacade facade = getGPGContext().getStateStoreFacade(); - Set candidates = new HashSet<>(); try { + // Get the candidate list from StateStore before calling router + Set allStateStoreApps = new HashSet<>(); List response = facade.getApplicationsHomeSubCluster(); for (ApplicationHomeSubCluster app : response) { - candidates.add(app.getApplicationId()); + allStateStoreApps.add(app.getApplicationId()); } - LOG.info("{} app entries in FederationStateStore", candidates.size()); + LOG.info("{} app entries in FederationStateStore", allStateStoreApps.size()); + // Get the candidate list from Registry before calling router + List allRegistryApps = getRegistryClient().getAllApplications(); + LOG.info("{} app entries in FederationRegistry", allStateStoreApps.size()); + + // Get the list of known apps from Router Set routerApps = getRouterKnownApplications(); LOG.info("{} known applications from Router", routerApps.size()); - candidates.removeAll(routerApps); - LOG.info("Deleting {} applications from statestore", candidates.size()); - if (LOG.isDebugEnabled()) { - LOG.debug("Apps to delete: {}.", candidates.stream().map(Object::toString) - .collect(Collectors.joining(","))); - } - for (ApplicationId appId : candidates) { + // Clean up StateStore entries + Set toDelete = + Sets.difference(allStateStoreApps, routerApps); + + LOG.info("Deleting {} applications from statestore", toDelete.size()); + LOG.debug("Apps to delete: {}.", + toDelete.stream().map(Object::toString).collect(Collectors.joining(","))); + + for (ApplicationId appId : toDelete) { try { + LOG.debug("Deleting {} from statestore ", appId); facade.deleteApplicationHomeSubCluster(appId); } catch (Exception e) { LOG.error("deleteApplicationHomeSubCluster failed at application {}.", appId, e); } } - // Clean up registry entries - cleanupAppRecordInRegistry(routerApps); + + // Clean up Registry entries + for (String app : allRegistryApps) { + ApplicationId appId = ApplicationId.fromString(app); + if (!routerApps.contains(appId)) { + LOG.debug("removing finished application entry for {}", app); + getRegistryClient().removeAppFromRegistry(appId, true); + } + } } catch (Throwable e) { LOG.error("Application cleaner started at time {} fails. ", now, e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java index 1e703b51960e2..c028bbdbe2c17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/applicationcleaner/TestDefaultApplicationCleaner.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext; @@ -63,6 +64,8 @@ public class TestDefaultApplicationCleaner { // The list of applications returned by mocked router private Set routerAppIds; + private ApplicationId appIdToAddConcurrently; + @Before public void setup() throws Exception { conf = new YarnConfiguration(); @@ -111,6 +114,7 @@ public void setup() throws Exception { new Token()); } Assert.assertEquals(3, registryClient.getAllApplications().size()); + appIdToAddConcurrently = null; } @After @@ -159,7 +163,42 @@ public class TestableDefaultApplicationCleaner extends DefaultApplicationCleaner { @Override public Set getAppsFromRouter() throws YarnRuntimeException { + if (appIdToAddConcurrently != null) { + SubClusterId scId = SubClusterId.newInstance("MySubClusterId"); + try { + ApplicationHomeSubCluster appHomeSubCluster = + ApplicationHomeSubCluster.newInstance(appIdToAddConcurrently, scId); + AddApplicationHomeSubClusterRequest request = + AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster); + stateStore.addApplicationHomeSubCluster(request); + } catch (YarnException e) { + throw new YarnRuntimeException(e); + } + registryClient.writeAMRMTokenForUAM(appIdToAddConcurrently, scId.toString(), + new Token<>()); + } return routerAppIds; } } + + @Test + public void testConcurrentNewApp() throws YarnException { + appIdToAddConcurrently = ApplicationId.newInstance(1, 1); + + appCleaner.run(); + + // The concurrently added app should be still there + GetApplicationsHomeSubClusterRequest appHomeSubClusterRequest = + GetApplicationsHomeSubClusterRequest.newInstance(); + GetApplicationsHomeSubClusterResponse applicationsHomeSubCluster = + stateStore.getApplicationsHomeSubCluster(appHomeSubClusterRequest); + Assert.assertNotNull(applicationsHomeSubCluster); + List appsHomeSubClusters = + applicationsHomeSubCluster.getAppsHomeSubClusters(); + Assert.assertNotNull(appsHomeSubClusters); + Assert.assertEquals(1, appsHomeSubClusters.size()); + + // The concurrently added app should be still there + Assert.assertEquals(1, registryClient.getAllApplications().size()); + } } From f1ce273150c0282c39b74df4e2a0c067df5a5e5c Mon Sep 17 00:00:00 2001 From: mudit1289 <50325057+mudit1289@users.noreply.github.com> Date: Tue, 31 Oct 2023 04:28:22 +0530 Subject: [PATCH 134/155] MAPREDUCE-7457: Added support to limit count of spill files (#6155) Contributed by Mudit Sharma. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../org/apache/hadoop/mapred/MapTask.java | 31 +++++++- .../apache/hadoop/mapreduce/MRJobConfig.java | 1 + .../src/main/resources/mapred-default.xml | 9 +++ .../org/apache/hadoop/mapred/TestMapTask.java | 78 +++++++++++++++++++ 4 files changed, 116 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index 06d9fbbe7a323..4f86f912838fa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -955,7 +955,10 @@ public static class MapOutputBuffer new ArrayList(); private int totalIndexCacheMemory; private int indexCacheMemoryLimit; + private int spillFilesCountLimit; private static final int INDEX_CACHE_MEMORY_LIMIT_DEFAULT = 1024 * 1024; + private static final int SPILL_FILES_COUNT_LIMIT_DEFAULT = -1; + private static final int SPILL_FILES_COUNT_UNBOUNDED_LIMIT_VALUE = -1; private MapTask mapTask; private MapOutputFile mapOutputFile; @@ -984,10 +987,17 @@ public void init(MapOutputCollector.Context context MRJobConfig.DEFAULT_IO_SORT_MB); indexCacheMemoryLimit = job.getInt(JobContext.INDEX_CACHE_MEMORY_LIMIT, INDEX_CACHE_MEMORY_LIMIT_DEFAULT); + spillFilesCountLimit = job.getInt(JobContext.SPILL_FILES_COUNT_LIMIT, + SPILL_FILES_COUNT_LIMIT_DEFAULT); if (spillper > (float)1.0 || spillper <= (float)0.0) { throw new IOException("Invalid \"" + JobContext.MAP_SORT_SPILL_PERCENT + "\": " + spillper); } + if(spillFilesCountLimit != SPILL_FILES_COUNT_UNBOUNDED_LIMIT_VALUE + && spillFilesCountLimit < 0) { + throw new IOException("Invalid value for \"" + JobContext.SPILL_FILES_COUNT_LIMIT + "\", " + + "current value: " + spillFilesCountLimit); + } if ((sortmb & 0x7FF) != sortmb) { throw new IOException( "Invalid \"" + JobContext.IO_SORT_MB + "\": " + sortmb); @@ -1698,7 +1708,7 @@ private void sortAndSpill() throws IOException, ClassNotFoundException, spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; } LOG.info("Finished spill " + numSpills); - ++numSpills; + incrementNumSpills(); } finally { if (out != null) out.close(); if (partitionOut != null) { @@ -1774,7 +1784,7 @@ private void spillSingleRecord(final K key, final V value, totalIndexCacheMemory += spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; } - ++numSpills; + incrementNumSpills(); } finally { if (out != null) out.close(); if (partitionOut != null) { @@ -2022,7 +2032,7 @@ private void sameVolRename(Path srcPath, if (!dst.getParentFile().exists()) { if (!dst.getParentFile().mkdirs()) { throw new IOException("Unable to rename " + src + " to " - + dst + ": couldn't create parent directory"); + + dst + ": couldn't create parent directory"); } } @@ -2030,6 +2040,21 @@ private void sameVolRename(Path srcPath, throw new IOException("Unable to rename " + src + " to " + dst); } } + + /** + * Increments numSpills local counter by taking into consideration + * the max limit on spill files being generated by the job. + * If limit is reached, this function throws an IOException + */ + private void incrementNumSpills() throws IOException { + ++numSpills; + if(spillFilesCountLimit != SPILL_FILES_COUNT_UNBOUNDED_LIMIT_VALUE + && numSpills > spillFilesCountLimit) { + throw new IOException("Too many spill files got created, control it with " + + "mapreduce.task.spill.files.count.limit, current value: " + spillFilesCountLimit + + ", current spill count: " + numSpills); + } + } } // MapOutputBuffer /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 8ec984e777bb6..289159ad922a7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -323,6 +323,7 @@ public interface MRJobConfig { public static final int DEFAULT_IO_SORT_MB = 100; public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes"; + String SPILL_FILES_COUNT_LIMIT = "mapreduce.task.spill.files.count.limit"; public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 9b0d8b563d7bd..ca144a7b156cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -62,6 +62,15 @@ set to less than .5 + + mapreduce.task.spill.files.count.limit + -1 + Number of spill files that can be created by a MapTask. + After breaching this, task will fail. Default value for this config is -1 + which indicates that there is no limit on number of spill files being + created + + mapreduce.job.local-fs.single-disk-limit.bytes -1 diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java index d5164de46d12a..fef179994f09a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java @@ -27,14 +27,20 @@ import org.apache.hadoop.mapred.MapTask.MapOutputBuffer; import org.apache.hadoop.mapred.Task.TaskReporter; import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.util.Progress; import org.junit.After; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; @@ -51,6 +57,9 @@ public void cleanup() throws Exception { FileUtil.fullyDelete(TEST_ROOT_DIR); } + @Rule + public ExpectedException exception = ExpectedException.none(); + // Verify output files for shuffle have group read permission even when // the configured umask normally would prevent it. @Test @@ -84,4 +93,73 @@ public void testShufflePermissions() throws Exception { Assert.assertEquals("Incorrect index file perms", (short)0640, perms.toShort()); } + + @Test + public void testSpillFilesCountLimitInvalidValue() throws Exception { + JobConf conf = new JobConf(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); + conf.setInt(MRJobConfig.SPILL_FILES_COUNT_LIMIT, -2); + MapOutputFile mof = new MROutputFiles(); + mof.setConf(conf); + TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); + MapTask mockTask = mock(MapTask.class); + doReturn(mof).when(mockTask).getMapOutputFile(); + doReturn(attemptId).when(mockTask).getTaskID(); + doReturn(new Progress()).when(mockTask).getSortPhase(); + TaskReporter mockReporter = mock(TaskReporter.class); + doReturn(new Counter()).when(mockReporter).getCounter(any(TaskCounter.class)); + MapOutputCollector.Context ctx = new MapOutputCollector.Context(mockTask, conf, mockReporter); + MapOutputBuffer mob = new MapOutputBuffer<>(); + + exception.expect(IOException.class); + exception.expectMessage("Invalid value for \"mapreduce.task.spill.files.count.limit\", " + + "current value: -2"); + + mob.init(ctx); + mob.close(); + } + + @Test + public void testSpillFilesCountBreach() throws Exception { + JobConf conf = new JobConf(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); + conf.setInt(MRJobConfig.SPILL_FILES_COUNT_LIMIT, 2); + MapOutputFile mof = new MROutputFiles(); + mof.setConf(conf); + TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); + MapTask mockTask = mock(MapTask.class); + doReturn(mof).when(mockTask).getMapOutputFile(); + doReturn(attemptId).when(mockTask).getTaskID(); + doReturn(new Progress()).when(mockTask).getSortPhase(); + TaskReporter mockReporter = mock(TaskReporter.class); + doReturn(new Counter()).when(mockReporter).getCounter(any(TaskCounter.class)); + MapOutputCollector.Context ctx = new MapOutputCollector.Context(mockTask, conf, mockReporter); + MapOutputBuffer mob = new MapOutputBuffer<>(); + mob.numSpills = 2; + mob.init(ctx); + + Method method = mob.getClass().getDeclaredMethod("incrementNumSpills"); + method.setAccessible(true); + boolean gotExceptionWithMessage = false; + try { + method.invoke(mob); + } catch(InvocationTargetException e) { + Throwable targetException = e.getTargetException(); + if (targetException != null) { + String errorMessage = targetException.getMessage(); + if (errorMessage != null) { + if(errorMessage.equals("Too many spill files got created, control it with " + + "mapreduce.task.spill.files.count.limit, current value: 2, current spill count: 3")) { + gotExceptionWithMessage = true; + } + } + } + } + + mob.close(); + + Assert.assertTrue(gotExceptionWithMessage); + } } From 4c04a6768c0cb3d5081cfa5d84ffb389d92f5805 Mon Sep 17 00:00:00 2001 From: Brian Goerlitz <2602635+bgoerlitz@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:25:41 -0500 Subject: [PATCH 135/155] YARN-11584. Safely fail on leaf queue with empty name (#6148) --- .../scheduler/capacity/QueuePath.java | 5 +++ ...estCapacitySchedulerAutoQueueCreation.java | 38 +++++++++++++++++++ .../scheduler/capacity/TestQueuePath.java | 2 + 3 files changed, 45 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 440742b908927..692be1132eac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -96,6 +96,11 @@ private void setFromFullPath(String fullPath) { * @return true if there is at least one empty part of the path */ public boolean hasEmptyPart() { + // iterator will not contain an empty leaf queue, so check directly + if (leaf.isEmpty()) { + return true; + } + for (String part : this) { if (part.isEmpty()) { return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index ca5069d22133f..590b0a4b7668d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -109,6 +109,8 @@ public class TestCapacitySchedulerAutoQueueCreation private static final Logger LOG = LoggerFactory.getLogger( TestCapacitySchedulerAutoQueueCreation.class); + private static final String SPECIFIED_QUEUE_MAPPING = "%specified"; + private static final String CURRENT_USER_MAPPING = "%user"; private static final Resource TEMPLATE_MAX_RES = Resource.newInstance(16 * @@ -1054,4 +1056,40 @@ public RMNodeLabelsManager createNodeLabelManager() { } } } + + @Test(timeout = 10000) + public void testAutoCreateLeafQueueFailsWithSpecifiedEmptyStringLeafQueue() + throws Exception { + + final String invalidQueue = ""; + + MockRM newMockRM = setupSchedulerInstance(); + CapacityScheduler newCS = + (CapacityScheduler) newMockRM.getResourceScheduler(); + + //queue mapping to place app in queue specified by user + setupQueueMapping(newCS, "app_user", "root.c", SPECIFIED_QUEUE_MAPPING); + newCS.updatePlacementRules(); + + try { + //submitting to root.c. should fail WITHOUT crashing the RM + submitApp(newCS, "app_user", invalidQueue, "root.c"); + + RMContext rmContext = mock(RMContext.class); + when(rmContext.getDispatcher()).thenReturn(dispatcher); + newCS.setRMContext(rmContext); + + ApplicationId appId = BuilderUtils.newApplicationId(1, 1); + SchedulerEvent addAppEvent = new AppAddedSchedulerEvent( + appId, "root.c." + invalidQueue, "app_user"); + newCS.handle(addAppEvent); + + RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED, + "error"); + dispatcher.spyOnNextEvent(event, 10000); + } finally { + ((CapacityScheduler) newMockRM.getResourceScheduler()).stop(); + newMockRM.stop(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java index 7eb577d9c1271..91171966c119c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java @@ -48,9 +48,11 @@ public void testCreation() { @Test public void testEmptyPart() { QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2"); + QueuePath queuePathWithEmptyLeaf = new QueuePath("root.level_1."); QueuePath queuePathWithoutEmptyPart = new QueuePath(TEST_QUEUE); Assert.assertTrue(queuePathWithEmptyPart.hasEmptyPart()); + Assert.assertTrue(queuePathWithEmptyLeaf.hasEmptyPart()); Assert.assertFalse(queuePathWithoutEmptyPart.hasEmptyPart()); } From cf3a4b3bb77c605f82aae062cea8600b980c12a5 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 1 Nov 2023 05:30:35 -0800 Subject: [PATCH 136/155] HADOOP-18850. S3A: Enable dual-layer server-side encryption with AWS KMS keys (#6140) Contributed by Viraj Jasani --- .../src/main/resources/core-default.xml | 4 +- .../hadoop/fs/s3a/S3AEncryptionMethods.java | 5 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 5 + .../EncryptionSecretOperations.java | 3 +- .../fs/s3a/impl/RequestFactoryImpl.java | 107 +++++++---- .../markdown/tools/hadoop-aws/encryption.md | 86 ++++++++- .../markdown/tools/hadoop-aws/performance.md | 2 +- .../site/markdown/tools/hadoop-aws/testing.md | 2 +- .../hadoop/fs/s3a/EncryptionTestUtils.java | 44 +++-- ...3ADSSEEncryptionWithDefaultS3Settings.java | 167 ++++++++++++++++++ ...estS3AEncryptionDSSEKMSUserDefinedKey.java | 61 +++++++ ...estS3AEncryptionWithDefaultS3Settings.java | 26 ++- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 21 ++- .../scale/ITestS3AHugeFilesEncryption.java | 13 +- 14 files changed, 473 insertions(+), 73 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index d64abf79407ae..31d980353bf26 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1724,14 +1724,14 @@ fs.s3a.encryption.algorithm Specify a server-side encryption or client-side encryption algorithm for s3a: file system. Unset by default. It supports the - following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS' + following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'DSSE-KMS', 'SSE-C', and 'CSE-KMS' fs.s3a.encryption.key Specific encryption key to use if fs.s3a.encryption.algorithm - has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C + has been set to 'SSE-KMS', 'DSSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C , the value of this property should be the Base64 encoded key. If you are using SSE-KMS and leave this property empty, you'll be using your default's S3 KMS key, otherwise you should set this property to the specific KMS key diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java index b599790a1cf76..6cacdff6c42d8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java @@ -33,13 +33,14 @@ public enum S3AEncryptionMethods { SSE_KMS("SSE-KMS", true, false), SSE_C("SSE-C", true, true), CSE_KMS("CSE-KMS", false, true), - CSE_CUSTOM("CSE-CUSTOM", false, true); + CSE_CUSTOM("CSE-CUSTOM", false, true), + DSSE_KMS("DSSE-KMS", true, false); /** * Error string when {@link #getMethod(String)} fails. * Used in tests. */ - static final String UNKNOWN_ALGORITHM + public static final String UNKNOWN_ALGORITHM = "Unknown encryption algorithm "; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 7466690744e82..f6bcd3dbdc460 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -1440,6 +1440,11 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket, diagnostics); break; + case DSSE_KMS: + LOG.debug("Using DSSE-KMS with {}", + diagnostics); + break; + case NONE: default: LOG.debug("Data is unencrypted"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java index bcd358e2d1672..8a55a970134f3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java @@ -53,7 +53,8 @@ public static Optional getSSECustomerKey(final EncryptionSecrets secrets * @return an optional key to attach to a request. */ public static Optional getSSEAwsKMSKey(final EncryptionSecrets secrets) { - if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS + if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS + || secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS) && secrets.hasEncryptionKey()) { return Optional.of(secrets.getEncryptionKey()); } else { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index ca36b658d70f3..b441bda521a95 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -60,6 +60,7 @@ import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import static org.apache.commons.lang3.StringUtils.isNotEmpty; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.UNKNOWN_ALGORITHM; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.util.Preconditions.checkNotNull; @@ -273,24 +274,38 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom, return; } - if (S3AEncryptionMethods.SSE_S3 == algorithm) { + switch (algorithm) { + case SSE_S3: copyObjectRequestBuilder.serverSideEncryption(algorithm.getMethod()); - } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + break; + case SSE_KMS: copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) - .ifPresent(kmsKey -> copyObjectRequestBuilder.ssekmsKeyId(kmsKey)); - } else if (S3AEncryptionMethods.SSE_C == algorithm) { + .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); + break; + case DSSE_KMS: + copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); + break; + case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) - .ifPresent(base64customerKey -> { - copyObjectRequestBuilder.copySourceSSECustomerAlgorithm( - ServerSideEncryption.AES256.name()).copySourceSSECustomerKey(base64customerKey) - .copySourceSSECustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))) - .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) - .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); - }); + .ifPresent(base64customerKey -> copyObjectRequestBuilder + .copySourceSSECustomerAlgorithm(ServerSideEncryption.AES256.name()) + .copySourceSSECustomerKey(base64customerKey) + .copySourceSSECustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))) + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey)))); + break; + case CSE_KMS: + case CSE_CUSTOM: + case NONE: + break; + default: + LOG.warn(UNKNOWN_ALGORITHM + ": " + algorithm); } } /** @@ -348,20 +363,35 @@ private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu final S3AEncryptionMethods algorithm = getServerSideEncryptionAlgorithm(); - if (S3AEncryptionMethods.SSE_S3 == algorithm) { + switch (algorithm) { + case SSE_S3: putObjectRequestBuilder.serverSideEncryption(algorithm.getMethod()); - } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + break; + case SSE_KMS: putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) - .ifPresent(kmsKey -> putObjectRequestBuilder.ssekmsKeyId(kmsKey)); - } else if (S3AEncryptionMethods.SSE_C == algorithm) { + .ifPresent(putObjectRequestBuilder::ssekmsKeyId); + break; + case DSSE_KMS: + putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(putObjectRequestBuilder::ssekmsKeyId); + break; + case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) - .ifPresent(base64customerKey -> { - putObjectRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) - .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); - }); + .ifPresent(base64customerKey -> putObjectRequestBuilder + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5(Md5Utils.md5AsBase64( + Base64.getDecoder().decode(base64customerKey)))); + break; + case CSE_KMS: + case CSE_CUSTOM: + case NONE: + break; + default: + LOG.warn(UNKNOWN_ALGORITHM + ": " + algorithm); } } @@ -409,20 +439,35 @@ private void multipartUploadEncryptionParameters( CreateMultipartUploadRequest.Builder mpuRequestBuilder) { final S3AEncryptionMethods algorithm = getServerSideEncryptionAlgorithm(); - if (S3AEncryptionMethods.SSE_S3 == algorithm) { + switch (algorithm) { + case SSE_S3: mpuRequestBuilder.serverSideEncryption(algorithm.getMethod()); - } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + break; + case SSE_KMS: mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) - .ifPresent(kmsKey -> mpuRequestBuilder.ssekmsKeyId(kmsKey)); - } else if (S3AEncryptionMethods.SSE_C == algorithm) { + .ifPresent(mpuRequestBuilder::ssekmsKeyId); + break; + case DSSE_KMS: + mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(mpuRequestBuilder::ssekmsKeyId); + break; + case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) - .ifPresent(base64customerKey -> { - mpuRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) - .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); - }); + .ifPresent(base64customerKey -> mpuRequestBuilder + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey)))); + break; + case CSE_KMS: + case CSE_CUSTOM: + case NONE: + break; + default: + LOG.warn(UNKNOWN_ALGORITHM + ": " + algorithm); } } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index ce1286c414a60..11bb2937db994 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -66,7 +66,7 @@ The server-side "SSE" encryption is performed with symmetric AES256 encryption; S3 offers different mechanisms for actually defining the key to use. -There are four key management mechanisms, which in order of simplicity of use, +There are five key management mechanisms, which in order of simplicity of use, are: * S3 Default Encryption @@ -75,6 +75,9 @@ are: by Amazon's Key Management Service, a key referenced by name in the uploading client. * SSE-C : the client specifies an actual base64 encoded AES-256 key to be used to encrypt and decrypt the data. +* DSSE-KMS: Two independent layers of encryption at server side. An AES256 key is +generated in S3, and encrypted with a secret key provided by Amazon's Key Management +Service. Encryption options @@ -84,6 +87,7 @@ Encryption options | `SSE-KMS` | server side, KMS key | key used to encrypt/decrypt | none | | `SSE-C` | server side, custom key | encryption algorithm and secret | encryption algorithm and secret | | `CSE-KMS` | client side, KMS key | encryption algorithm and key ID | encryption algorithm | +| `DSSE-KMS` | server side, KMS key | key used to encrypt/decrypt | none | With server-side encryption, the data is uploaded to S3 unencrypted (but wrapped by the HTTPS encryption channel). @@ -91,7 +95,7 @@ The data is encrypted in the S3 store and decrypted when it's being retrieved. A server side algorithm can be enabled by default for a bucket, so that whenever data is uploaded unencrypted a default encryption algorithm is added. -When data is encrypted with S3-SSE or SSE-KMS it is transparent to all clients +When data is encrypted with S3-SSE, SSE-KMS or DSSE-KMS it is transparent to all clients downloading the data. SSE-C is different in that every client must know the secret key needed to decypt the data. @@ -132,7 +136,7 @@ not explicitly declare an encryption algorithm. [S3 Default Encryption for S3 Buckets](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) -This supports SSE-S3 and SSE-KMS. +This supports SSE-S3, SSE-KMS and DSSE-KMS. There is no need to set anything up in the client: do it in the AWS console. @@ -316,6 +320,82 @@ metadata. Since only one encryption key can be provided at a time, S3A will not pass the correct encryption key to decrypt the data. +### DSSE-KMS: Dual-layer Server-Encryption with KMS Managed Encryption Keys + +By providing a dual-layer server-side encryption mechanism using AWS Key Management Service +(AWS KMS) keys, known as DSSE-KMS, two layers of encryption are applied to objects upon their +upload to Amazon S3. DSSE-KMS simplifies the process of meeting compliance requirements that +mandate the implementation of multiple layers of encryption for data while maintaining complete +control over the encryption keys. + + +When uploading data encrypted with SSE-KMS, the sequence is as follows: + +1. The S3A client must declare a specific CMK in the property `fs.s3a.encryption.key`, or leave + it blank to use the default configured for that region. + +2. The S3A client uploads all the data as normal, now including encryption information. + +3. The S3 service encrypts the data with a symmetric key unique to the new object. + +4. The S3 service retrieves the chosen CMK key from the KMS service, and, if the user has + the right to use it, uses it to provide dual-layer encryption for the data. + + +When downloading DSSE-KMS encrypted data, the sequence is as follows + +1. The S3A client issues an HTTP GET request to read the data. + +2. S3 sees that the data was encrypted with DSSE-KMS, and looks up the specific key in the + KMS service. + +3. If and only if the requesting user has been granted permission to use the CMS key does + the KMS service provide S3 with the key. + +4. As a result, S3 will only decode the data if the user has been granted access to the key. + +Further reading on DSSE-KMS [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingDSSEncryption.html) + +AWS Blog post [here](https://aws.amazon.com/blogs/aws/new-amazon-s3-dual-layer-server-side-encryption-with-keys-stored-in-aws-key-management-service-dsse-kms/) + +### Enabling DSSE-KMS + +To enable DSSE-KMS, the property `fs.s3a.encryption.algorithm` must be set to `DSSE-KMS` in `core-site`: + +```xml + + fs.s3a.encryption.algorithm + DSSE-KMS + +``` + +The ID of the specific key used to encrypt the data should also be set in the property `fs.s3a.encryption.key`: + +```xml + + fs.s3a.encryption.key + arn:aws:kms:us-west-2:360379543683:key/071a86ff-8881-4ba0-9230-95af6d01ca01 + +``` + +Organizations may define a default key in the Amazon KMS; if a default key is set, +then it will be used whenever SSE-KMS encryption is chosen and the value of `fs.s3a.encryption.key` is empty. + +### the S3A `fs.s3a.encryption.key` key only affects created files + +With SSE-KMS, the S3A client option `fs.s3a.encryption.key` sets the +key to be used when new files are created. When reading files, this key, +and indeed the value of `fs.s3a.encryption.algorithm` is ignored: +S3 will attempt to retrieve the key and decrypt the file based on the create-time settings. + +This means that + +* There's no need to configure any client simply reading data. +* It is possible for a client to read data encrypted with one KMS key, and + write it with another. + + + ## Encryption best practises diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index e3ab79d92e5dc..2c83b063b1efc 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -447,7 +447,7 @@ and rate of requests. Spreading data across different buckets, and/or using a more balanced directory structure may be beneficial. Consult [the AWS documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html). -Reading or writing data encrypted with SSE-KMS forces S3 to make calls of +Reading or writing data encrypted with SSE-KMS or DSSE-KMS forces S3 to make calls of the AWS KMS Key Management Service, which comes with its own [Request Rate Limits](http://docs.aws.amazon.com/kms/latest/developerguide/limits.html). These default to 1200/second for an account, across all keys and all uses of diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 6100bc0ae5c95..c2eafbcb8de28 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -1087,7 +1087,7 @@ The specific tests an Assumed Role ARN is required for are To run these tests you need: 1. A role in your AWS account will full read and write access rights to -the S3 bucket used in the tests, and KMS for any SSE-KMS tests. +the S3 bucket used in the tests, and KMS for any SSE-KMS or DSSE-KMS tests. 1. Your IAM User to have the permissions to "assume" that role. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java index 8d927dc957b16..7b2b1c639e3cc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java @@ -28,8 +28,7 @@ import org.apache.hadoop.fs.Path; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.assertj.core.api.Assertions.assertThat; public final class EncryptionTestUtils { @@ -39,6 +38,8 @@ private EncryptionTestUtils() { public static final String AWS_KMS_SSE_ALGORITHM = "aws:kms"; + public static final String AWS_KMS_DSSE_ALGORITHM = "aws:kms:dsse"; + public static final String SSE_C_ALGORITHM = "AES256"; /** @@ -77,25 +78,36 @@ public static void assertEncrypted(S3AFileSystem fs, md.ssekmsKeyId()); switch(algorithm) { case SSE_C: - assertNull("Metadata algorithm should have been null in " - + details, - md.serverSideEncryptionAsString()); - assertEquals("Wrong SSE-C algorithm in " - + details, - SSE_C_ALGORITHM, md.sseCustomerAlgorithm()); + assertThat(md.serverSideEncryptionAsString()) + .describedAs("Details of the server-side encryption algorithm used: %s", details) + .isNull(); + assertThat(md.sseCustomerAlgorithm()) + .describedAs("Details of SSE-C algorithm: %s", details) + .isEqualTo(SSE_C_ALGORITHM); String md5Key = convertKeyToMd5(fs); - assertEquals("getSSECustomerKeyMd5() wrong in " + details, - md5Key, md.sseCustomerKeyMD5()); + assertThat(md.sseCustomerKeyMD5()) + .describedAs("Details of the customer provided encryption key: %s", details) + .isEqualTo(md5Key); break; case SSE_KMS: - assertEquals("Wrong algorithm in " + details, - AWS_KMS_SSE_ALGORITHM, md.serverSideEncryptionAsString()); - assertEquals("Wrong KMS key in " + details, - kmsKeyArn, - md.ssekmsKeyId()); + assertThat(md.serverSideEncryptionAsString()) + .describedAs("Details of the server-side encryption algorithm used: %s", details) + .isEqualTo(AWS_KMS_SSE_ALGORITHM); + assertThat(md.ssekmsKeyId()) + .describedAs("Details of the KMS key: %s", details) + .isEqualTo(kmsKeyArn); + break; + case DSSE_KMS: + assertThat(md.serverSideEncryptionAsString()) + .describedAs("Details of the server-side encryption algorithm used: %s", details) + .isEqualTo(AWS_KMS_DSSE_ALGORITHM); + assertThat(md.ssekmsKeyId()) + .describedAs("Details of the KMS key: %s", details) + .isEqualTo(kmsKeyArn); break; default: - assertEquals("AES256", md.serverSideEncryptionAsString()); + assertThat(md.serverSideEncryptionAsString()) + .isEqualTo("AES256"); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java new file mode 100644 index 0000000000000..a39490174424c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; + +import org.junit.Ignore; +import org.junit.Test; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.EncryptionTestUtils.AWS_KMS_DSSE_ALGORITHM; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; +import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; + +/** + * Concrete class that extends {@link AbstractTestS3AEncryption} + * and tests already configured bucket level DSSE encryption using s3 console. + */ +public class ITestS3ADSSEEncryptionWithDefaultS3Settings extends + AbstractTestS3AEncryption { + + @Override + public void setup() throws Exception { + super.setup(); + // get the KMS key for this test. + S3AFileSystem fs = getFileSystem(); + Configuration c = fs.getConf(); + skipIfEncryptionNotSet(c, getSSEAlgorithm()); + } + + @SuppressWarnings("deprecation") + @Override + protected void patchConfigurationEncryptionSettings( + final Configuration conf) { + removeBaseAndBucketOverrides(conf, + S3_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_ALGORITHM); + conf.set(S3_ENCRYPTION_ALGORITHM, + getSSEAlgorithm().getMethod()); + } + + /** + * Setting this to NONE as we don't want to overwrite + * already configured encryption settings. + * @return the algorithm + */ + @Override + protected S3AEncryptionMethods getSSEAlgorithm() { + return S3AEncryptionMethods.NONE; + } + + /** + * The check here is that the object is encrypted + * and that the encryption key is the KMS key + * provided, not any default key. + * @param path path + */ + @Override + protected void assertEncrypted(Path path) throws IOException { + S3AFileSystem fs = getFileSystem(); + Configuration c = fs.getConf(); + String kmsKey = getS3EncryptionKey(getTestBucketName(c), c); + EncryptionTestUtils.assertEncrypted(fs, path, DSSE_KMS, kmsKey); + } + + @Override + @Ignore + @Test + public void testEncryptionSettingPropagation() throws Throwable { + } + + @Override + @Ignore + @Test + public void testEncryption() throws Throwable { + } + + /** + * Skipping if the test bucket is not configured with + * aws:kms encryption algorithm. + */ + @Override + public void testEncryptionOverRename() throws Throwable { + skipIfBucketNotKmsEncrypted(); + super.testEncryptionOverRename(); + } + + /** + * If the test bucket is not configured with aws:kms encryption algorithm, + * skip the test. + * + * @throws IOException If the object creation/deletion/access fails. + */ + private void skipIfBucketNotKmsEncrypted() throws IOException { + S3AFileSystem fs = getFileSystem(); + Path path = methodPath(); + ContractTestUtils.touch(fs, path); + try { + String sseAlgorithm = + getS3AInternals().getObjectMetadata(path).serverSideEncryptionAsString(); + if (StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_DSSE_ALGORITHM)) { + skip("Test bucket is not configured with " + AWS_KMS_DSSE_ALGORITHM); + } + } finally { + ContractTestUtils.assertDeleted(fs, path, false); + } + } + + @Test + public void testEncryptionOverRename2() throws Throwable { + skipIfBucketNotKmsEncrypted(); + S3AFileSystem fs = getFileSystem(); + + // write the file with the unencrypted FS. + // this will pick up whatever defaults we have. + Path src = path(createFilename(1024)); + byte[] data = dataset(1024, 'a', 'z'); + EncryptionSecrets secrets = fs.getEncryptionSecrets(); + validateEncryptionSecrets(secrets); + writeDataset(fs, src, data, data.length, 1024 * 1024, true); + ContractTestUtils.verifyFileContents(fs, src, data); + + Configuration fs2Conf = new Configuration(fs.getConf()); + fs2Conf.set(S3_ENCRYPTION_ALGORITHM, + DSSE_KMS.getMethod()); + try (FileSystem kmsFS = FileSystem.newInstance(fs.getUri(), fs2Conf)) { + Path targetDir = path("target"); + kmsFS.mkdirs(targetDir); + ContractTestUtils.rename(kmsFS, src, targetDir); + Path renamedFile = new Path(targetDir, src.getName()); + ContractTestUtils.verifyFileContents(fs, renamedFile, data); + String kmsKey = getS3EncryptionKey(getTestBucketName(fs2Conf), fs2Conf); + // we assert that the renamed file has picked up the KMS key of our FS + EncryptionTestUtils.assertEncrypted(fs, renamedFile, DSSE_KMS, kmsKey); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java new file mode 100644 index 0000000000000..028ba7f6e1fe0 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; + +/** + * Concrete class that extends {@link AbstractTestS3AEncryption} + * and tests DSSE-KMS encryption. + */ +public class ITestS3AEncryptionDSSEKMSUserDefinedKey + extends AbstractTestS3AEncryption { + + @Override + protected Configuration createConfiguration() { + // get the KMS key for this test. + Configuration c = new Configuration(); + String kmsKey = S3AUtils.getS3EncryptionKey(getTestBucketName(c), c); + // skip the test if DSSE-KMS or KMS key not set. + try { + skipIfEncryptionNotSet(c, DSSE_KMS); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + assume("KMS key is expected to be present", StringUtils.isNotBlank(kmsKey)); + Configuration conf = super.createConfiguration(); + conf.set(S3_ENCRYPTION_KEY, kmsKey); + return conf; + } + + @Override + protected S3AEncryptionMethods getSSEAlgorithm() { + return DSSE_KMS; + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java index 1b25846fafddf..c246161a938dd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java @@ -115,20 +115,34 @@ public void testEncryption() throws Throwable { */ @Override public void testEncryptionOverRename() throws Throwable { + skipIfBucketNotKmsEncrypted(); + super.testEncryptionOverRename(); + } + + /** + * If the test bucket is not configured with aws:kms encryption algorithm, + * skip the test. + * + * @throws IOException If the object creation/deletion/access fails. + */ + private void skipIfBucketNotKmsEncrypted() throws IOException { S3AFileSystem fs = getFileSystem(); Path path = path(getMethodName() + "find-encryption-algo"); ContractTestUtils.touch(fs, path); - String sseAlgorithm = getS3AInternals().getObjectMetadata(path) - .serverSideEncryptionAsString(); - if(StringUtils.isBlank(sseAlgorithm) || - !sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) { - skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM); + try { + String sseAlgorithm = + getS3AInternals().getObjectMetadata(path).serverSideEncryptionAsString(); + if (StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) { + skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM); + } + } finally { + ContractTestUtils.assertDeleted(fs, path, false); } - super.testEncryptionOverRename(); } @Test public void testEncryptionOverRename2() throws Throwable { + skipIfBucketNotKmsEncrypted(); S3AFileSystem fs = getFileSystem(); // write the file with the unencrypted FS. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index aa38186c65032..eaa84c6086bd9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -78,6 +78,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -1483,19 +1484,25 @@ public static S3AFileStatus innerGetFileStatus( * Skip a test if encryption algorithm or encryption key is not set. * * @param configuration configuration to probe. + * @param s3AEncryptionMethods list of encryption algorithms to probe. + * @throws IOException if the secret lookup fails. */ public static void skipIfEncryptionNotSet(Configuration configuration, - S3AEncryptionMethods s3AEncryptionMethod) throws IOException { + S3AEncryptionMethods... s3AEncryptionMethods) throws IOException { + if (s3AEncryptionMethods == null || s3AEncryptionMethods.length == 0) { + throw new IllegalArgumentException("Specify at least one encryption method"); + } // if S3 encryption algorithm is not set to desired method or AWS encryption // key is not set, then skip. String bucket = getTestBucketName(configuration); final EncryptionSecrets secrets = buildEncryptionSecrets(bucket, configuration); - if (!s3AEncryptionMethod.getMethod().equals(secrets.getEncryptionMethod().getMethod()) - || StringUtils.isBlank(secrets.getEncryptionKey())) { - skip(S3_ENCRYPTION_KEY + " is not set for " + s3AEncryptionMethod - .getMethod() + " or " + S3_ENCRYPTION_ALGORITHM + " is not set to " - + s3AEncryptionMethod.getMethod() - + " in " + secrets); + boolean encryptionMethodMatching = Arrays.stream(s3AEncryptionMethods).anyMatch( + s3AEncryptionMethod -> s3AEncryptionMethod.getMethod() + .equals(secrets.getEncryptionMethod().getMethod())); + if (!encryptionMethodMatching || StringUtils.isBlank(secrets.getEncryptionKey())) { + skip(S3_ENCRYPTION_KEY + " is not set or " + S3_ENCRYPTION_ALGORITHM + " is not set to " + + Arrays.stream(s3AEncryptionMethods).map(S3AEncryptionMethods::getMethod) + .collect(Collectors.toList()) + " in " + secrets); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java index 93242155c678a..404a9684f42ec 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java @@ -27,6 +27,8 @@ import org.apache.hadoop.fs.s3a.EncryptionTestUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_KMS; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; @@ -43,7 +45,7 @@ public class ITestS3AHugeFilesEncryption extends AbstractSTestS3AHugeFiles { @Override public void setup() throws Exception { Configuration c = new Configuration(); - skipIfEncryptionNotSet(c, SSE_KMS); + skipIfEncryptionNotSet(c, SSE_KMS, DSSE_KMS); super.setup(); } @@ -67,7 +69,12 @@ protected boolean isEncrypted(S3AFileSystem fileSystem) { protected void assertEncrypted(Path hugeFile) throws IOException { Configuration c = new Configuration(); String kmsKey = getS3EncryptionKey(getTestBucketName(c), c); - EncryptionTestUtils.assertEncrypted(getFileSystem(), hugeFile, - SSE_KMS, kmsKey); + if (SSE_KMS.getMethod().equals(c.get(S3_ENCRYPTION_ALGORITHM))) { + EncryptionTestUtils.assertEncrypted(getFileSystem(), hugeFile, SSE_KMS, kmsKey); + } else if (DSSE_KMS.getMethod().equals(c.get(S3_ENCRYPTION_ALGORITHM))) { + EncryptionTestUtils.assertEncrypted(getFileSystem(), hugeFile, DSSE_KMS, kmsKey); + } else { + throw new AssertionError("Invalid encryption configured"); + } } } From cbb153b69ec2afc69247f1680be237b0e2be1e2f Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Wed, 1 Nov 2023 21:40:15 +0530 Subject: [PATCH 137/155] HDFS-17246. Fix shaded client for building Hadoop on Windows (#5943) --- dev-support/bin/hadoop.sh | 67 +++++++++++-------- dev-support/docker/Dockerfile_windows_10 | 32 ++++++--- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 4 +- 3 files changed, 67 insertions(+), 36 deletions(-) diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh index b51a9525bd89e..fd9363e66bbd7 100755 --- a/dev-support/bin/hadoop.sh +++ b/dev-support/bin/hadoop.sh @@ -21,18 +21,20 @@ personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" # These flags are needed to run Yetus against Hadoop on Windows. -WINDOWS_FLAGS="-Pnative-win - -Dhttps.protocols=TLSv1.2 - -Drequire.openssl - -Drequire.test.libhadoop - -Dshell-executable=${BASH_EXECUTABLE} - -Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES} - -Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES} - -Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE} - -Dwindows.cmake.build.type=RelWithDebInfo - -Dwindows.build.hdfspp.dll=off - -Dwindows.no.sasl=on - -Duse.platformToolsetVersion=v142" +WINDOWS_FLAGS=( + "-Pnative-win" + "-Dhttps.protocols=TLSv1.2" + "-Drequire.openssl" + "-Drequire.test.libhadoop" + "-Dshell-executable=${BASH_EXECUTABLE}" + "-Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}" + "-Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}" + "-Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}" + "-Dwindows.cmake.build.type=RelWithDebInfo" + "-Dwindows.build.hdfspp.dll=off" + "-Dwindows.no.sasl=on" + "-Duse.platformToolsetVersion=v142" +) ## @description Globals specific to this personality ## @audience private @@ -292,7 +294,7 @@ function hadoop_native_flags -Drequire.snappy \ -Pdist \ -Dtar \ - "${WINDOWS_FLAGS}" + "${WINDOWS_FLAGS[@]}" ;; *) echo \ @@ -436,7 +438,7 @@ function personality_modules fi if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then - extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}" + extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS[*]} ${extra}" fi for module in $(hadoop_order ${ordering}); do @@ -557,14 +559,6 @@ function shadedclient_rebuild declare module declare -a modules=() - if [[ ${OSTYPE} = Windows_NT || - ${OSTYPE} =~ ^CYGWIN.* || - ${OSTYPE} =~ ^MINGW32.* || - ${OSTYPE} =~ ^MSYS.* ]]; then - echo "hadoop personality: building on windows, skipping check of client artifacts." - return 0 - fi - yetus_debug "hadoop personality: seeing if we need the test of client artifacts." for module in hadoop-client-modules/hadoop-client-check-invariants \ hadoop-client-modules/hadoop-client-check-test-invariants \ @@ -581,28 +575,47 @@ function shadedclient_rebuild big_console_header "Checking client artifacts on ${repostatus} with shaded clients" - extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true" + extra=( + "-Dtest=NoUnitTests" + "-Dmaven.javadoc.skip=true" + "-Dcheckstyle.skip=true" + "-Dspotbugs.skip=true" + ) if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then + # shellcheck disable=SC2206 + extra+=(${WINDOWS_FLAGS[*]}) + + # The shaded client integration tests require the Hadoop jars that were just built to be + # installed in the local maven repository. + # shellcheck disable=SC2086 + echo_and_redirect "${logfile}" \ + "${MAVEN}" "${MAVEN_ARGS[@]}" install -fae --batch-mode \ + -DskipTests -DskipDocs -Pdist -Dtar ${extra[*]} + + # The shaded client integration tests spawn a MiniDFS and MiniYARN cluster for testing. Both of + # them require winutils.exe to be found in the PATH and HADOOP_HOME to be set. if load_hadoop_version; then export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT" + WIN_HADOOP_HOME=$(cygpath -w -a "${HADOOP_HOME}") + export PATH="${PATH};${WIN_HADOOP_HOME}\bin" else yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail." fi - - extra="${WINDOWS_FLAGS} ${extra}" fi + # shellcheck disable=SC2086 echo_and_redirect "${logfile}" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}" + "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" ${extra[*]} big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients" + # shellcheck disable=SC2086 echo_and_redirect "${logfile}" \ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ "${modules[@]}" \ -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \ - -Dspotbugs.skip=true "${extra}" + -Dspotbugs.skip=true ${extra[*]} count=$("${GREP}" -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10 index 20cad3f56d6e3..105529c5d65b2 100644 --- a/dev-support/docker/Dockerfile_windows_10 +++ b/dev-support/docker/Dockerfile_windows_10 @@ -38,8 +38,8 @@ RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtoo && del /q vs_buildtools.exe # Install Chocolatey. +ENV chocolateyVersion=1.4.0 RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" -RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" # Install git. RUN choco install git.install -y @@ -55,24 +55,18 @@ RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows -ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows" # Install Azul Java 8 JDK. RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java" -ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64" -RUN setx PATH "%PATH%;%JAVA_HOME%\bin" # Install Apache Maven. RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven" -RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin" -ENV MAVEN_OPTS '-Xmx2048M -Xss128M' # Install CMake 3.19.0. RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake" -RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin" # Install zstd 1.5.4. RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip @@ -112,13 +106,35 @@ RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Fi RUN powershell Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip -OutFile $Env:TEMP\python-3.10.11-embed-amd64.zip RUN powershell Expand-Archive -Path $Env:TEMP\python-3.10.11-embed-amd64.zip -DestinationPath "C:\Python3" RUN powershell New-Item -ItemType HardLink -Value "C:\Python3\python.exe" -Path "C:\Python3\python3.exe" + +# Create a user HadoopBuilder with basic privileges and use it for building Hadoop on Windows. +RUN powershell New-LocalUser -Name 'HadoopBuilder' -Description 'User account for building Apache Hadoop' -Password ([securestring]::new()) -AccountNeverExpires -PasswordNeverExpires + +# Grant the privilege to create symbolic links to HadoopBuilder. +RUN powershell secedit /export /cfg "C:\secpol.cfg" +RUN powershell "(Get-Content C:\secpol.cfg).Replace('SeCreateSymbolicLinkPrivilege = ', 'SeCreateSymbolicLinkPrivilege = HadoopBuilder,') | Out-File C:\secpol.cfg" +RUN powershell secedit /configure /db "C:\windows\security\local.sdb" /cfg "C:\secpol.cfg" +RUN powershell Remove-Item -Force "C:\secpol.cfg" -Confirm:$false + +# Login as HadoopBuilder and set the necessary environment and PATH variables. +USER HadoopBuilder +ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows" +ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64" +ENV MAVEN_OPTS '-Xmx2048M -Xss128M' +RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +RUN setx PATH "%PATH%;%JAVA_HOME%\bin" +RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin" +RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin" +RUN setx PATH "%PATH%;C:\ZStd" RUN setx path "%PATH%;C:\Python3" +RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin" # We get strange Javadoc errors without this. RUN setx classpath "" +# Setting Git configurations. +RUN git config --global core.autocrlf true RUN git config --global core.longpaths true -RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin" # Define the entry point for the docker container. ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"] diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index dbcee7492f06d..71cff2e3915b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -63,6 +63,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -661,7 +662,8 @@ public static boolean isValidName(String src) { for (int i = 0; i < components.length; i++) { String element = components[i]; if (element.equals(".") || - (element.contains(":")) || + // For Windows, we must allow the : in the drive letter. + (!Shell.WINDOWS && i == 1 && element.contains(":")) || (element.contains("/"))) { return false; } From 2b1378c99e78e5cc13c03f80acea5a501cbad683 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 2 Nov 2023 02:51:38 +0800 Subject: [PATCH 138/155] YARN-11583. [Addendum] Improve Node Link for YARN Federation Web Page. (#6232) --- .../apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java index 2ee813934f751..28bc49ed8cca1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java @@ -167,7 +167,6 @@ private void initYarnFederationNodeLabelsOfCluster(NodeLabelsInfo nodeLabelsInfo row = row.td(String.valueOf(nActiveNMs)); } - row = row.td(String.valueOf(nActiveNMs)); PartitionInfo partitionInfo = info.getPartitionInfo(); ResourceInfo available = partitionInfo.getResourceAvailable(); row.td(available.toFormattedString()).__(); From 5b215f23d562e63c8b522d39852ef8d51f0ebdeb Mon Sep 17 00:00:00 2001 From: YuanHanzhong Date: Sat, 4 Nov 2023 07:42:39 +0800 Subject: [PATCH 139/155] HADOOP-18963. Fix typos in .gitignore (#6243) --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 166954793dd0c..84d9572cbb5e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -.DS_Store +*.DS_Store *.iml *.ipr *.iws From d6bb47e5db2fac8236b23ad30389b5ae69a5c2bd Mon Sep 17 00:00:00 2001 From: rRajivramachandran Date: Fri, 3 Nov 2023 20:53:50 -0500 Subject: [PATCH 140/155] MAPREDUCE-7459. Fixed TestHistoryViewerPrinter flakiness during string comparison (#6215). Contributed by Rajiv Ramachandran. Reviewed-by: Inigo Goiri Signed-off-by: Ayush Saxena --- .../jobhistory/TestHistoryViewerPrinter.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java index cb508f6d29b9d..3a1408bdff9d4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java @@ -24,6 +24,7 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskType; +import org.assertj.core.api.Assertions; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -35,7 +36,9 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.TimeZone; import java.util.Locale; @@ -160,6 +163,13 @@ public void testHumanPrinter() throws Exception { LINE_SEPARATOR, outStr); } + private static void assertEqualLines(String str1, String str2) { + final List linesFromStr1 = Arrays.asList(str1.trim().split("\n")); + final List linesFromStr2 = Arrays.asList(str2.trim().split("\n")); + + Assertions.assertThat(linesFromStr1).containsExactlyInAnyOrderElementsOf(linesFromStr2); + } + @Test public void testHumanPrinterAll() throws Exception { JobHistoryParser.JobInfo job = createJobInfo(); @@ -168,7 +178,7 @@ public void testHumanPrinterAll() throws Exception { TimeZone.getTimeZone("GMT")); String outStr = run(printer); if (System.getProperty("java.version").startsWith("1.7")) { - Assert.assertEquals("\n" + + assertEqualLines("\n" + "Hadoop job: job_1317928501754_0001\n" + "=====================================\n" + "User: rkanter\n" + @@ -356,7 +366,7 @@ public void testHumanPrinterAll() throws Exception { "localhost\ttask_1317928501754_0001_m_000002, " + LINE_SEPARATOR, outStr); } else { - Assert.assertEquals("\n" + + assertEqualLines("\n" + "Hadoop job: job_1317928501754_0001\n" + "=====================================\n" + "User: rkanter\n" + From 0a867b9c39012bde4461ff45306c99bada74e5e8 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sat, 4 Nov 2023 19:09:48 +0800 Subject: [PATCH 141/155] YARN-11594. [Federation] Improve Yarn Federation documentation. (#6209) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../src/site/markdown/Federation.md | 424 ++++++++++++------ 1 file changed, 297 insertions(+), 127 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 5d5dc786e13b4..631a62896a790 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -24,7 +24,7 @@ This document described a federation-based approach to scale a single YARN clust The applications running in this federated environment will see a single massive YARN cluster and will be able to schedule tasks on any node of the federated cluster. Under the hood, the federation system will negotiate with sub-clusters resource managers and provide resources to the application. The goal is to allow an individual job to “span” sub-clusters seamlessly. This design is structurally scalable, as we bound the number of nodes each RM is responsible for, and appropriate policies, will try to ensure that the majority of applications will reside within a single sub-cluster, thus the number of applications each RM will see is also bounded. This means we could almost linearly scale, by simply adding sub-clusters (as very little coordination is needed across them). -This architecture can provide very tight enforcement of scheduling invariants within each sub-cluster (simply inherits from YARN), while continuous rebalancing across subcluster will enforce (less strictly) that these properties are also respected at a global level (e.g., if a sub-cluster loses a large number of nodes, we could re-map queues to other sub-clusters to ensure users running on the impaired sub-cluster are not unfairly affected). +This architecture can provide very tight enforcement of scheduling invariants within each sub-cluster (simply inherits from YARN), while continuous re-balancing across sub-cluster will enforce (less strictly) that these properties are also respected at a global level (e.g., if a sub-cluster loses a large number of nodes, we could re-map queues to other sub-clusters to ensure users running on the impaired sub-cluster are not unfairly affected). Federation is designed as a “layer” atop of existing YARN codebase, with limited changes in the core YARN mechanisms. @@ -78,7 +78,7 @@ to minimize overhead on the scheduling infrastructure (more in section on scalab 3. Intercepts all the requests, thus it can enforce application quotas, which would not be enforceable by sub-cluster RM (as each only see a fraction of the AM requests). 4. The AMRMProxy can enforce load-balancing / overflow policies. -###Global Policy Generator +### Global Policy Generator Global Policy Generator overlooks the entire federation and ensures that the system is configured and tuned properly all the time. A key design point is that the cluster availability does not depend on an always-on GPG. The GPG operates continuously but out-of-band from all cluster operations, and provide us with a unique vantage point, that allows to enforce global invariants, affect load balancing, trigger draining of sub-clusters that will undergo maintenance, etc. @@ -94,19 +94,19 @@ This part of the federation system is part of future work in [YARN-5597](https:/ ### Federation State-Store The Federation State defines the additional state that needs to be maintained to loosely couple multiple individual sub-clusters into a single large federated cluster. This includes the following information: -####Sub-cluster Membership +#### Sub-cluster Membership The member YARN RMs continuously heartbeat to the state store to keep alive and publish their current capability/load information. This information is used by the -Global Policy Generator (GPG) to make proper policy decisions. Also this information can be used by routers to select the best home sub-cluster. This mechanism allows +Global Policy Generator (GPG) to make proper policy decisions. Also, this information can be used by routers to select the best home sub-cluster. This mechanism allows us to dynamically grow/shrink the “cluster fleet” by adding or removing sub-clusters. This also allows for easy maintenance of each sub-cluster. This is new functionality that needs to be added to the YARN RM but the mechanisms are well understood as it’s similar to individual YARN RM HA. -####Application’s Home Sub-cluster +#### Application’s Home Sub-cluster The sub-cluster on which the Application Master (AM) runs is called the Application’s “home sub-cluster”. The AM is not limited to resources from the home sub-cluster but can also request resources from other sub-clusters, referred to as secondary sub-clusters. The federated environment will be configured and tuned periodically such that when an AM is placed on a sub-cluster, it should be able to find most of the resources on the home sub-cluster. Only in certain cases it should need to ask for resources from other sub-clusters. -###Federation Policy Store +### Federation Policy Store The federation Policy Store is a logically separate store (while it might be backed by the same physical component), which contains information about how applications and resource requests are routed to different sub-clusters. The current implementation provides @@ -161,13 +161,12 @@ Configuration These are common configurations that should appear in the **conf/yarn-site.xml** at each machine in the federation. +| Property | Example | Description | +|:----------------------------------|:-------------------------|:----------------------------------------------------------------------------| +| `yarn.federation.enabled` | `true` | Whether federation is enabled or not | +| `yarn.resourcemanager.cluster-id` | `` | The unique subcluster identifier for this RM (same as the one used for HA). | -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.federation.enabled` | `true` | Whether federation is enabled or not | -|`yarn.resourcemanager.cluster-id` | `` | The unique subcluster identifier for this RM (same as the one used for HA). | - -#### State-Store: +#### How to configure State-Store Currently, we support ZooKeeper and SQL based implementations of the state-store. @@ -175,20 +174,26 @@ Currently, we support ZooKeeper and SQL based implementations of the state-store ZooKeeper: one must set the ZooKeeper settings for Hadoop: -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore` | The type of state-store to use. | -|`hadoop.zk.address` | `host:port` | The address for the ZooKeeper ensemble. | +| Property | Example | Description | +|:------------------------------------|:------------------------------------------------------------------------------------|:----------------------------------------| +| `yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore` | The type of state-store to use. | +| `hadoop.zk.address` | `host:port` | The address for the ZooKeeper ensemble. | SQL: one must setup the following parameters: -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. | -|`yarn.federation.state-store.sql.url` | `jdbc:mysql://:/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. | -|`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. | -|`yarn.federation.state-store.sql.username` | `` | For SQLFederationStateStore the username for the DB connection. | -|`yarn.federation.state-store.sql.password` | `` | For SQLFederationStateStore the password for the DB connection. | +| Property | Example | Description | +|:--------------------------------------------------|:------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| `yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. | +| `yarn.federation.state-store.sql.url` | `jdbc:mysql://:/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. | +| `yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. | +| `yarn.federation.state-store.sql.username` | `` | For SQLFederationStateStore the username for the DB connection. | +| `yarn.federation.state-store.sql.password` | `` | For SQLFederationStateStore the password for the DB connection. | +| `yarn.federation.state-store.sql.max-connections` | `1` | This is the maximum number of parallel connections each Router makes to the state-store. | +| `yarn.federation.state-store.sql.minimum-idle` | `1` | The property controls the minimum number of idle connections that HikariCP trie to maintain in the pool. | +| `yarn.federation.state-store.sql.pool-name` | `YARN-Federation-DataBasePool` | Specifies the name of the connection pool used by the FederationSQLStateStore. | +| `yarn.federation.state-store.sql.max-life-time` | `30m` | This property controls the maximum lifetime of a connection in the pool. | +| `yarn.federation.state-store.sql.idle-time-out` | `10m` | This property controls the maximum amount of time that a connection is allowed to sit idle in the pool. | +| `yarn.federation.state-store.sql.conn-time-out` | `10s` | Set the maximum amount of time that a client will wait for a connection from the pool. | We provide scripts for **MySQL** and **Microsoft SQL Server**. @@ -224,30 +229,32 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. 4. SQL Server 2017 Enterprise 5. SQL Server 2019 Enterprise -#### Optional: +#### How to configure Optional -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.federation.failover.enabled` | `true` | Whether should retry considering RM failover within each subcluster. | -|`yarn.federation.blacklist-subclusters` | `` | A list of black-listed sub-clusters, useful to disable a sub-cluster | -|`yarn.federation.policy-manager` | `org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager` | The choice of policy manager determines how Applications and ResourceRequests are routed through the system. | -|`yarn.federation.policy-manager-params` | `` | The payload that configures the policy. In our example a set of weights for router and amrmproxy policies. This is typically generated by serializing a policymanager that has been configured programmatically, or by populating the state-store with the .json serialized form of it. | -|`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. | -|`yarn.federation.machine-list` | `` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example:

    node1, subcluster1, rack1
    node2, subcluster2, rack1
    node3, subcluster3, rack2
    node4, subcluster3, rack2 | +| Property | Example | Description | +|:--------------------------------------------|:------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.federation.failover.enabled` | `true` | Whether should retry considering RM failover within each sub-cluster. | +| `yarn.federation.non-ha.enabled` | `false` | If our subCluster's ResourceManager (RM) does not have High Availability (HA) enabled, we can configure this parameter as true. However, it is recommended to use RM HA in a production environment. | +| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | A FailoverProxyProvider implementation that uses the FederationStateStore to determine the ResourceManager to connect to. This supports both HA and regular mode which is controlled by configuration. | +| `yarn.federation.blacklist-subclusters` | `` | A list of black-listed sub-clusters, useful to disable a sub-cluster | +| `yarn.federation.policy-manager` | `org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager` | The choice of policy manager determines how Applications and ResourceRequests are routed through the system. | +| `yarn.federation.policy-manager-params` | `` | The payload that configures the policy. In our example a set of weights for router and amrmproxy policies. This is typically generated by serializing a policymanager that has been configured programmatically, or by populating the state-store with the .json serialized form of it. | +| `yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which sub-cluster a node belongs to, and which subcluster(s) a rack belongs to. | +| `yarn.federation.machine-list` | `` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example:

    node1, subcluster1, rack1
    node2, subcluster2, rack1
    node3, subcluster3, rack2
    node4, subcluster3, rack2 | -- yarn.federation.policy-manager-params +##### How to configure yarn.federation.policy-manager-params To configure the `yarn.federation.policy-manager-params` parameter, which represents the weight policy for the default queue, and where the relevant information will be parsed as `WeightedPolicyInfo`. We can use the following JSON format for configuration: - ```xml - - yarn.federation.policy-manager-params - {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} - - ``` +```xml + + yarn.federation.policy-manager-params + {"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"} + +``` This JSON configuration allows you to define the weight policy for default queue, where: @@ -255,10 +262,9 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. - The `amrmPolicyWeights` represents the allocation ratios for Application Master when request containers from different subclusters' RM. For instance, when an AM requests containers, it will request `40%` of the containers from `SC-2` and `60%` of the containers from `SC-1`. - The `headroomAlpha` used by policies that balance weight-based and load-based considerations in their decisions. For policies that use this parameter, values close to 1 indicate that most of the decision should be based on currently observed headroom from various sub-clusters, values close to zero, indicate that the decision should be mostly based on weights and practically ignore current load. -How to configure the policy-manager --------------------- +##### How to configure the policy-manager -Router Policy +**Router Policy** Router Policy defines the logic for determining the routing of an application submission and determines the HomeSubCluster for the application. @@ -284,7 +290,7 @@ Router Policy - WeightedRandomRouterPolicy - This policy implements a weighted random sample among currently active sub-clusters. -AMRM Policy +**AMRM Policy** AMRM Proxy defines the logic to split the resource request list received by AM among RMs. @@ -303,26 +309,26 @@ AMRM Policy - RejectAMRMProxyPolicy - This policy simply rejects all requests. Useful to prevent apps from accessing any sub-cluster. -Policy Manager +**Policy Manager** The PolicyManager is providing a combination of RouterPolicy and AMRMPolicy. We can set policy-manager like this: - ```xml - - - yarn.federation.policy-manager - org.apache.hadoop.yarn.server.federation.policies.manager.HashBroadcastPolicyManager - - ``` +```xml + + + yarn.federation.policy-manager + org.apache.hadoop.yarn.server.federation.policies.manager.HashBroadcastPolicyManager + +``` - HashBroadcastPolicyManager - Policy that routes applications via hashing of their queuename, and broadcast resource requests. This picks a HashBasedRouterPolicy for the router and a BroadcastAMRMProxyPolicy for the amrmproxy as they are designed to work together. @@ -337,8 +343,7 @@ Policy Manager - WeightedLocalityPolicyManager - Policy that allows operator to configure "weights" for routing. This picks a LocalityRouterPolicy for the router and a LocalityMulticastAMRMProxyPolicy for the amrmproxy as they are designed to work together. -How to configure the queue policy --------------------- +##### How to configure the queue policy We will provide a set of commands to view and save queue policies. @@ -369,38 +374,120 @@ WeightedPolicyInfo include the following: used by policies that balance weight-based and load-based considerations in their decisions. For policies that use this parameter, values close to 1 indicate that most of the decision should be based on currently observed headroom from various sub-clusters, values close to zero, indicate that the decision should be mostly based on weights and practically ignore current load. +##### How to Config ZookeeperFederationStateStore Hierarchies + +Similar to YARN-2962, We have implemented hierarchical storage for applications in the ZooKeeper federation store to manage the number of nodes under a specific Znode. + +We can configure `yarn.resourcemanager.zk-appid-node.split-index`, default is 0, Index at which last section of application id (with each section separated by _ in application id) will be split so that application znode stored in zookeeper RM state store will be stored as two different znodes (parent-child). Split is done from the end. +For instance, with no split, appid znode will be of the form application_1352994193343_0001. If the value of this config is 1, the appid znode will be broken into two parts application_1352994193343_000 and 1 respectively with former being the parent node. +application_1352994193343_0002 will then be stored as 2 under the parent node application_1352994193343_000. This config can take values from 0 to 4. 0 means there will be no split. If configuration value is outside this range, it will be treated as config value of 0(i.e. no split). A value +larger than 0 (up to 4) should be configured if you are storing a large number of apps in ZK based RM state store and state store operations are failing due to LenError in Zookeeper. + ### ON RMs: These are extra configurations that should appear in the **conf/yarn-site.xml** at each ResourceManager. -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.resourcemanager.epoch` | `` | The seed value for the epoch. This is used to guarantee uniqueness of container-IDs generate by different RMs. It must therefore be unique among sub-clusters and `well-spaced` to allow for failures which increment epoch. Increments of 1000 allow for a large number of sub-clusters and practically ensure near-zero chance of collisions (a clash will only happen if a container is still alive for 1000 restarts of one RM, while the next RM never restarted, and an app requests more containers). | +| Property | Example | Description | +|:-----------------------------|:-----------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.resourcemanager.epoch` | `` | The seed value for the epoch. This is used to guarantee uniqueness of container-IDs generate by different RMs. It must therefore be unique among sub-clusters and `well-spaced` to allow for failures which increment epoch. Increments of 1000 allow for a large number of sub-clusters and practically ensure near-zero chance of collisions (a clash will only happen if a container is still alive for 1000 restarts of one RM, while the next RM never restarted, and an app requests more containers). | Optional: -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.federation.state-store.heartbeat-interval-secs` | `60` | The rate at which RMs report their membership to the federation to the central state-store. | +| Property | Example | Description | +|:------------------------------------------------------|:--------|:--------------------------------------------------------------------------------------------| +| `yarn.federation.state-store.heartbeat-interval-secs` | `60` | The rate at which RMs report their membership to the federation to the central state-store. | +**How to configure the cleanup of applications** + +The Router supports storing scheduled applications in the StateStore. However, as the number of applications increases, it's essential to provide a mechanism for application cleanup. +We have implemented an automatic cleanup method in the ResourceManager (RM), which attempts to clean up the data in the StateStore after an application has completed its execution and has been removed from RM's memory. +Additionally, to account for certain exceptional cases where some applications may not be cleaned up properly, when the RM starts, we utilize a separate thread to attempt cleanup of completed applications. +We can refer to [YARN-11323](https://issues.apache.org/jira/browse/YARN-11323). + +| Property | Example | Description | +|:--------------------------------------------------------|:--------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.federation.state-store.clean-up-retry-count` | `1` | The number of retries to clear the app in the FederationStateStore, the default value is 1, that is, after the app fails to clean up, it will retry the cleanup again. | +| `yarn.federation.state-store.clean-up-retry-sleep-time` | `1s` | Clear the sleep time of App retry in FederationStateStore. When the app fails to clean up, it will sleep for a period of time and then try to clean up. The default value is 1s. | ### ON ROUTER: +#### How to select Router Mode + +Router supports YARN `Federation mode` and `Non-Federation` mode. + +- Non-YARN Federation mode + +the Router's role is to straightforwardly forward client requests to the cluster resourcemanager. In this mode, all we need to do is configure the cluster's ResourceManager addresses in the Router's **conf/yarn-site.xml**. + +- YARN Federation mode + +the Router distributes client requests to different sub-cluster resourcemanager based on user-configured queue policy. In this mode, we need to configure items such as interceptors, state store, and other settings. + These are extra configurations that should appear in the **conf/yarn-site.xml** at each Router. | Property | Example | Description | |:--------------------------------------------------|:----------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `yarn.router.bind-host` | `0.0.0.0` | Host IP to bind the router to. The actual address the server will bind to. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in yarn.router.*.address respectively. This is most useful for making Router listen to all interfaces by setting to 0.0.0.0. | | `yarn.router.clientrm.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client. The last step of this pipeline must be the Federation Client Interceptor. | +| `yarn.router.rmadmin.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.rmadmin.FederationRMAdminInterceptor` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client via Admin interface. The last step of this pipeline must be the Federation Admin Interceptor. | +| `yarn.router.webapp.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client via REST interface. The last step of this pipeline must be the Federation Interceptor REST. | + +#### How to configure Router interceptor -> Enable ApplicationSubmissionContextInterceptor +- yarn.router.clientrm.interceptor-class.pipeline + +The role of the interceptor is to forward client requests to RM. + +| Property | Mode | Description | +|:----------------------------------------------------------------------------------------|:-----------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor` | `Non-Federation` | That simply forwards the client requests to the cluster resource manager. | +| `org.apache.hadoop.yarn.server.router.clientrm.PassThroughClientRequestInterceptor` | `Federation` | Interceptor that does not do anything other than forwarding it to the next Interceptor in the chain. | +| `org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor` | `Federation` | This Class provides an implementation for federation of YARN RM and scaling an application across multiple YARN SubClusters. All the federation specific implementation is encapsulated in this class. This is always the last interceptor in the chain. | +| `org.apache.hadoop.yarn.server.router.clientrm.ApplicationSubmissionContextInterceptor` | `Federation` | It prevents DoS attack over the ApplicationClientProtocol. Currently, it checks the size of the ApplicationSubmissionContext. If it exceeds the limit it can cause Zookeeper failures. | + +**How to configure the thread pool of FederationClientInterceptor** + +The FederationClientInterceptor retrieves data from multiple subClusters. To improve performance, we utilize a thread pool for concurrent access to multiple subClusters. +Below is the configuration for the thread pool, which can be adjusted based on the requirements of the production environment. + +| Property | Mode | Description | +|:----------------------------------------------------------------------|:--------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.router.interceptor.user-thread-pool.minimum-pool-size` | `5` | This configurable is used to set the corePoolSize(minimumPoolSize) of the thread pool of the interceptor. Default is 5. | +| `yarn.router.interceptor.user-thread-pool.maximum-pool-size` | `5` | This configuration is used to set the default value of maximumPoolSize of the thread pool of the interceptor. Default is 5. | +| `yarn.router.interceptor.user-thread-pool.keep-alive-time` | `0s` | This configurable is used to set the keepAliveTime of the thread pool of the interceptor. Default is 0s. | +| `yarn.router.interceptor.user-thread-pool.allow-core-thread-time-out` | `false` | This method configures the policy for core threads regarding termination when no tasks arrive within the keep-alive time. If set to true, We need to ensure that yarn.router.interceptor.user-thread-pool.keep-alive-time is greater than 0. | + +- yarn.router.rmadmin.interceptor-class.pipeline + +The role of the interceptor is to forward client's administrator requests to RM. + +| Property | Mode | Description | +|:--------------------------------------------------------------------------------|:-----------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor` | `Non-Federation` | That simply forwards the client requests to the cluster resource manager. | +| `org.apache.hadoop.yarn.server.router.rmadmin.FederationRMAdminInterceptor` | `Federation` | Intercept the client's administrator request and forward it to the ResourceManager of Yarn SubClusters. This is always the last interceptor in the chain. | + +**How to configure the thread pool of FederationRMAdminInterceptor** + +The thread pool configuration used by the `FederationRMAdminInterceptor` is consistent with that of the `FederationClientInterceptor` and can be directly referenced for configuration. + +- yarn.router.webapp.interceptor-class.pipeline + +The role of the interceptor is to forward client Rest requests to RM. + +| Property | Mode | Description | +|:----------------------------------------------------------------------------|:-----------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| +| `org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST` | `Non-Federation` | That simply forwards the client requests to the cluster resource manager. | +| `org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST` | `Federation` | Intercept the client's Rest request and forward it to the ResourceManager of Yarn SubClusters. This is always the last interceptor in the chain. | + +How to enable ApplicationSubmissionContextInterceptor - If the `FederationStateStore` is configured with `Zookpeer` storage, the app information will be stored in `Zookpeer`. If the size of the app information exceeds `1MB`, `Zookpeer` may fail. `ApplicationSubmissionContextInterceptor` will check the size of `ApplicationSubmissionContext`, if the size exceeds the limit(default 1MB), an exception will be thrown. - - The size of the ApplicationSubmissionContext of the application application_123456789_0001 is above the limit. Size = 1.02 MB. + +- The size of the ApplicationSubmissionContext of the application `application_123456789_0001` is above the limit. Size = 1.02 MB. - The required configuration is as follows: -``` +```xml yarn.router.clientrm.interceptor-class.pipeline org.apache.hadoop.yarn.server.router.clientrm.PassThroughClientRequestInterceptor, @@ -415,42 +502,40 @@ These are extra configurations that should appear in the **conf/yarn-site.xml** Optional: -| Property | Example | Description | -|:---- |:---- |:---- | -|`yarn.router.hostname` | `0.0.0.0` | Router host name. -|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. | -|`yarn.router.webapp.address` | `0.0.0.0:8089` | Webapp address at the router. | -|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. | -|`yarn.router.webapp.https.address` | `0.0.0.0:8091` | Secure webapp address at the router. | -|`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. | -|`yarn.router.submit.interval.time` | `10ms` | The interval between two retry, the default value is 10ms. | -|`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. | -|`yarn.federation.cache-ttl.secs` | `60` | The Router caches informations, and this is the time to leave before the cache is invalidated. | -|`yarn.federation.cache.class` | `org.apache.hadoop.yarn.server.federation.cache.FederationJCache` | The Router caches informations, We can configure the Cache implementation and the default implementation is FederationJCache.| -|`yarn.router.webapp.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client via REST interface. The last step of this pipeline must be the Federation Interceptor REST. | - -Security: +| Property | Example | Description | +|:------------------------------------------------|:------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.router.hostname` | `0.0.0.0` | Router host name. | +| `yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. | +| `yarn.router.webapp.address` | `0.0.0.0:8089` | Webapp address at the router. | +| `yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. | +| `yarn.router.webapp.https.address` | `0.0.0.0:8091` | Secure webapp address at the router. | +| `yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. | +| `yarn.router.submit.interval.time` | `10ms` | The interval between two retry, the default value is 10ms. | +| `yarn.federation.cache-ttl.secs` | `300s` | The Router caches informations, and this is the time to leave before the cache is invalidated. | +| `yarn.federation.cache.class` | `org.apache.hadoop.yarn.server.federation.cache.FederationJCache` | The Router caches informations, We can configure the Cache implementation and the default implementation is FederationJCache. | + +#### How to configure Router security Kerberos supported in federation. -| Property | Example | Description | -|:---- |:---- |:---- | -| `yarn.router.keytab.file` | | The keytab file used by router to login as its service principal. The principal name is configured with 'yarn.router.kerberos.principal'.| -| `yarn.router.kerberos.principal` | | The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all Routers in setup. | -| `yarn.router.kerberos.principal.hostname` | | Optional. The hostname for the Router containing this configuration file. Will be different for each machine. Defaults to current hostname. | +| Property | Example | Description | +|:------------------------------------------|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.router.keytab.file` | | The keytab file used by router to login as its service principal. The principal name is configured with 'yarn.router.kerberos.principal'. | +| `yarn.router.kerberos.principal` | | The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all Routers in setup. | +| `yarn.router.kerberos.principal.hostname` | | Optional. The hostname for the Router containing this configuration file. Will be different for each machine. Defaults to current hostname. | -Enabling CORS support: +#### How to configure Router Cors support To enable cross-origin support (CORS) for the Yarn Router, please set the following configuration parameters: -| Property | Example | Description | -| ----------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | +| Property | Example | Description | +|-------------------------------------------|---------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| | `hadoop.http.filter.initializers` | `org.apache.hadoop.security.HttpCrossOriginFilterInitializer` | Optional. Set the filter to HttpCrossOriginFilterInitializer, Configure this parameter in core-site.xml. | -| `yarn.router.webapp.cross-origin.enabled` | `true` | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. | +| `yarn.router.webapp.cross-origin.enabled` | `true` | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. | -Cache: +#### How to configure Router Cache -Cache is not enabled by default. When we set the `yarn.federation.cache-ttl.secs` parameter and its value is greater than 0, Cache will be enabled. +Cache is enabled by default. When we set the `yarn.federation.cache-ttl.secs` parameter and its value is greater than 0, Cache will be enabled. We currently provide two Cache implementations: `JCache` and `GuavaCache`. - JCache @@ -465,13 +550,69 @@ If we want to use JCache, we can configure `yarn.federation.cache.class` to `org This is a Cache implemented based on the Guava framework. If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`. -Router command line: +#### How to configure Router AuditLog + +We can enable the AuditLog configuration for the Router and collect the AuditLog in a separate log file. We need to modify the configuration related to RouterAuditLog in the **conf/log4j.properties** file. -- deregisterSubCluster +The configuration is as follows: + +``` +router.audit.logger=INFO,ROUTERAUDIT +router.audit.log.maxfilesize=256MB +router.audit.log.maxbackupindex=20 +log4j.logger.org.apache.hadoop.yarn.server.router.RouterAuditLogger=${router.audit.logger} +log4j.additivity.org.apache.hadoop.yarn.server.router.RouterAuditLogger=false +log4j.appender.ROUTERAUDIT=org.apache.log4j.RollingFileAppender +log4j.appender.ROUTERAUDIT.File=${hadoop.log.dir}/router-audit.log +log4j.appender.ROUTERAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.ROUTERAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.ROUTERAUDIT.MaxFileSize=${router.audit.log.maxfilesize} +log4j.appender.ROUTERAUDIT.MaxBackupIndex=${router.audit.log.maxbackupindex} +``` + +#### How to configure Router Opts + +If we need to modify the `HEAPSIZE` or `OPTS` for the Router, we can make changes in the `yarn-env.sh` file. + +- YARN_ROUTER_HEAPSIZE + +``` +# Specify the max heapsize for the Router. If no units are given, it will be assumed to be in MB. +# Default is the same as HADOOP_HEAPSIZE_MAX +export YARN_ROUTER_HEAPSIZE= +``` + +- YARN_ROUTER_OPTS +``` +# Specify the JVM options to be used when starting the Router. These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +export YARN_ROUTER_OPTS="-Drouter.audit.logger=INFO,ROUTERAUDIT" +``` + +#### How to configure the client to randomly Select a Router + +By default, the client will try from the first router in the configured router list. If the connection is successful, the router will not be replaced. We can set `yarn.federation.failover.random.order` to true to allow clients to randomly select Routers. + +#### How to configure the cleanup of expired subClusters + +We allow the Router to initiate a separate thread for periodically monitoring the status of all subClusters. If a subCluster's heartbeat exceeds a certain time threshold, we categorize that subCluster as "LOST". + +| Property | Example | Description | +|:---------------------------------------------------|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.router.deregister.subcluster.enable` | `true` | Whether to enable the capability for automatic subCluster offline. Default is true. | +| `yarn.router.subcluster.heartbeat.expiration.time` | `30m` | The default subCluster heartbeat timeout time is 30 minutes. | +| `yarn.router.scheduled.executor.threads` | `1` | The number of threads started to check for subCluster timeouts has a default value of 1, and using the default value is sufficient for meeting the checking requirements. | +| `yarn.router.subcluster.cleaner.interval.time` | `60s` | The check thread's checking interval. Default 60s. | + +**Note** We don't need to configure the subCluster deregister checking threads for all Routers; using 1-2 Routers for checking is sufficient. + +#### How to use Router Command Line + +##### Cmd1: deregisterSubCluster This command is used to `deregister subCluster`, If the interval between the heartbeat time of the subCluster, and the current time exceeds the timeout period, set the state of the subCluster to `SC_LOST`. -Uasge: +Usage: `yarn routeradmin -deregisterSubCluster [-sc|--subClusterId ]` @@ -485,14 +626,16 @@ Examples: If we want to deregisterSubCluster `SC-1` -- yarn routeradmin -deregisterSubCluster -sc SC-1 -- yarn routeradmin -deregisterSubCluster --subClusterId SC-1 +``` + yarn routeradmin -deregisterSubCluster -sc SC-1 + yarn routeradmin -deregisterSubCluster --subClusterId SC-1 +``` -- policy +##### Cmd2: policy We provide a set of commands for Policy Include list policies, save policies, batch save policies. -Uasge: +Usage: `yarn routeradmin -policy -s|--save (queue;router weight;amrm weight;headroomalpha)` @@ -500,7 +643,7 @@ Uasge: `yarn routeradmin -policy -l|--list ([--pageSize][--currentPage][--queue][--queues])` -- -s|--save () +###### SubCmd1: -s|--save () This command is used to save the policy information of the queue, including queue and weight information. @@ -520,11 +663,13 @@ Example: We have two sub-clusters, `SC-1` and `SC-2`. We want to configure a weight policy for the `root.a` queue. The Router Weight is set to `SC-1` with a weight of `0.7` and `SC-2` with a weight of `0.3`. The AMRM Weight is set `SC-1` to `0.6` and `SC-2` to `0.4`. We are using the default value of `0.1` for `headroomalpha`. +``` yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0 +``` -- -bs|--batch-save (--format xml) (-f|--input-file fileName) +###### SubCmd2: -bs|--batch-save (--format xml) (-f|--input-file fileName) This command can batch load weight information for queues based on the provided `federation-weights.xml` file. @@ -534,7 +679,7 @@ This command can batch load weight information for queues based on the provided | `-f, --input-file [path]` | `The path to the configuration file. Please use the absolute path of the configuration file.` | How to configure `federation-weights.xml` - ```xml +```xml @@ -589,7 +734,7 @@ How to configure `federation-weights.xml` - ``` +``` Example: @@ -598,11 +743,13 @@ and then use the batch-save command to save the configurations in bulk. The file name can be any file name, but it is recommended to use `federation-weights.xml` +``` yarn routeradmin -policy -bs --format xml -f /path/federation-weights.xml yarn routeradmin -policy --batch-save --format xml -f /path/federation-weights.xml +``` -- -l|--list (--pageSize --currentPage --queue --queues) +###### SubCmd3: -l|--list (--pageSize --currentPage --queue --queues) This command is used to display the configured queue weight information. @@ -617,13 +764,15 @@ Example: We can display the list of already configured queue weight information. We can use the `--queue` option to query the weight information for a specific queue or use the `--queues` option to query the weight information for multiple queues. +``` yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b +``` ### ON GPG: -GlobalPolicyGenerator, abbreviated as "GPG", is used for the automatic generation of global policies for subClusters. +GlobalPolicyGenerator, abbreviated as “GPG”, is used for the automatic generation of global policies for subClusters. The functionality of GPG is still under development and not yet complete. It is not recommended for use in a production environment. These are extra configurations that should appear in the **conf/yarn-site.xml** for GPG. We allow only one GPG. @@ -643,7 +792,7 @@ Optional: | `yarn.federation.gpg.policy.generator.blacklist` | | Which sub-clusters the policy generator should blacklist. | | `yarn.federation.gpg.policy.generator.load-based.pending.minimum` | `100` | The minimum number of pending applications in the subCluster. | | `yarn.federation.gpg.policy.generator.load-based.pending.maximum` | `1000` | The maximum number of pending applications in the subCluster. | -| `yarn.federation.gpg.policy.generator.load-based.weight.minimum` | `0` | If a subCluster has a very high load, we will assign this value to the subCluster. The default value is 0, which means that we no longer assign appliaction to this subCluster. | +| `yarn.federation.gpg.policy.generator.load-based.weight.minimum` | `0` | If a subCluster has a very high load, we will assign this value to the subCluster. The default value is 0, which means that we no longer assign application to this subCluster. | | `yarn.federation.gpg.policy.generator.load-based.edit.maximum` | `3` | This value represents the number of subClusters we want to calculate. default is 3. | | `yarn.federation.gpg.policy.generator.load-based.scaling` | `LINEAR` | We provide 4 calculation methods: NONE, LINEAR, QUADRATIC, LOG. | | `yarn.federation.gpg.webapp.address` | `0.0.0.0:8069` | The address of the GPG web application. | @@ -683,9 +832,10 @@ No calculation is required, and the weight is 1 at this time. LINEAR is used by default. -Security: +**Note** +It is not recommended to use GPG's capability to clean up expired applications in a production environment as this feature is still undergoing further development. -Kerberos supported in GPG. +#### How to configure GPG security | Property | Example | Description | |:--------------------------------------------------|:--------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -693,7 +843,7 @@ Kerberos supported in GPG. | `yarn.federation.gpg.kerberos.principal` | | The GPG service principal. This is typically set to GPG/_HOST@REALM.TLD. GPG will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on GPG in setup. | | `yarn.federation.gpg.kerberos.principal.hostname` | | Optional. The hostname for the GPG containing this configuration file. Will be different for each machine. Defaults to current hostname. | -Enabling CORS support: +#### How to configure GPG Cors support To enable cross-origin support (CORS) for the Yarn Router, please set the following configuration parameters: @@ -702,32 +852,52 @@ To enable cross-origin support (CORS) for the Yarn Router, please set the follow | `hadoop.http.filter.initializers` | `org.apache.hadoop.security.HttpCrossOriginFilterInitializer` | Optional. Set the filter to HttpCrossOriginFilterInitializer, Configure this parameter in core-site.xml. | | `yarn.federation.gpg.webapp.cross-origin.enabled` | `true` | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. | +#### How to configure GPG Opts + +If we need to modify the `HEAPSIZE` or `OPTS` for the GPG, we can make changes in the `yarn-env.sh` file. + +- YARN_GLOBALPOLICYGENERATOR_HEAPSIZE +``` +# Specify the max heapsize for the Global Policy Generator. +# If no units are given, it will be assumed to be in MB. Default is the same as HADOOP_HEAPSIZE_MAX +# export YARN_GLOBALPOLICYGENERATOR_HEAPSIZE= +``` + +- YARN_GLOBALPOLICYGENERATOR_OPTS +``` +# Specify the JVM options to be used when starting the GPG. +# These options will be appended to the options specified as HADOOP_OPTS and therefore may override any similar flags set in HADOOP_OPTS +# +# See ResourceManager for some examples +# export YARN_GLOBALPOLICYGENERATOR_OPTS= +``` + ### ON NMs: These are extra configurations that should appear in the **conf/yarn-site.xml** at each NodeManager. -| Property | Example | Description | -|:---- |:---- |:---- | -| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. | +| Property | Example | Description | +|:--------------------------------------------------------|:----------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. | | `yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor. | Optional: -| Property | Example | Description | -|:---- |:---- |:---- | -| `yarn.nodemanager.amrmproxy.ha.enable` | `true` | Whether or not the AMRMProxy HA is enabled for multiple application attempt support. | -| `yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. | -| `yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. | +| Property | Example | Description | +|:---------------------------------------------|:--------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `yarn.nodemanager.amrmproxy.ha.enable` | `true` | Whether or not the AMRMProxy HA is enabled for multiple application attempt support. | +| `yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. | +| `yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. | Running a Sample Job -------------------- In order to submit jobs to a Federation cluster one must create a separate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations: -| Property | Example | Description | -|:--- |:--- |:---- | -| `yarn.resourcemanager.address` | `:8050` | Redirects jobs launched at the client to the router's client RM port. | -| `yarn.resourcemanager.scheduler.address` | `localhost:8049` | Redirects jobs to the federation AMRMProxy port.| +| Property | Example | Description | +|:-----------------------------------------|:---------------------|:----------------------------------------------------------------------| +| `yarn.resourcemanager.address` | `:8050` | Redirects jobs launched at the client to the router's client RM port. | +| `yarn.resourcemanager.scheduler.address` | `localhost:8049` | Redirects jobs to the federation AMRMProxy port. | Any YARN jobs for the cluster can be submitted from the client configurations described above. In order to launch a job through federation, first start up all the clusters involved in the federation as described [here](../../hadoop-project-dist/hadoop-common/ClusterSetup.html). Next, start up the router on the router machine with the following command: @@ -865,7 +1035,7 @@ $HADOOP_HOME/bin/yarn --daemon start resourcemanager true - + yarn.federation.failover.enabled false @@ -994,7 +1164,7 @@ $HADOOP_HOME/bin/yarn --daemon start router true - + yarn.federation.failover.enabled false From a79cd110a45a28094d8e8c351b2f67b34dc5ca68 Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Sun, 5 Nov 2023 07:50:41 +0800 Subject: [PATCH 142/155] YARN-11609. [Addendum] [Federation] Router Supports DeregisterSubCluster. (#6254) YuanBen Wang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../server/router/rmadmin/FederationRMAdminInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java index d269cfe0971cf..30b36909fb6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java @@ -153,7 +153,7 @@ public void init(String userName) { this.heartbeatExpirationMillis = this.conf.getTimeDuration( YarnConfiguration.ROUTER_SUBCLUSTER_EXPIRATION_TIME, - YarnConfiguration.DEFAULT_ROUTER_SUBCLUSTER_EXPIRATION_TIME, TimeUnit.MINUTES); + YarnConfiguration.DEFAULT_ROUTER_SUBCLUSTER_EXPIRATION_TIME, TimeUnit.MILLISECONDS); } @VisibleForTesting From c15fd3b2c0375ae9865210fae7eab480ef3d4fcc Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Sun, 5 Nov 2023 13:57:48 +0800 Subject: [PATCH 143/155] YARN-11599. Incorrect log4j properties file in SLS sample conf (#6220) Contributed by Junfan Zhang. Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- hadoop-tools/hadoop-sls/src/main/sample-conf/log4j.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-sls/src/main/sample-conf/log4j.properties b/hadoop-tools/hadoop-sls/src/main/sample-conf/log4j.properties index cfd405b16e7ce..b2a63bc0a9634 100644 --- a/hadoop-tools/hadoop-sls/src/main/sample-conf/log4j.properties +++ b/hadoop-tools/hadoop-sls/src/main/sample-conf/log4j.properties @@ -16,4 +16,4 @@ log4j.appender.test.Target=System.out log4j.appender.test.layout=org.apache.log4j.PatternLayout log4j.appender.test.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n -log4j.logger=NONE, test +log4j.rootLogger=WARN, test From 4ef2322b6d7c1d1ae1cd9e62042f2db0ae42fc7c Mon Sep 17 00:00:00 2001 From: huhaiyang Date: Mon, 6 Nov 2023 11:20:25 +0800 Subject: [PATCH 144/155] HDFS-17243. Add the parameter storage type for getBlocks method (#6238). Contributed by Haiyang Hu. Reviewed-by: He Xiaoqiao Reviewed-by: Tao Li Signed-off-by: Shuyan Zhang --- .../router/RouterNamenodeProtocol.java | 7 +- .../federation/router/RouterRpcServer.java | 4 +- .../federation/router/TestRouterRpc.java | 6 +- ...amenodeProtocolServerSideTranslatorPB.java | 4 +- .../NamenodeProtocolTranslatorPB.java | 11 +- .../hdfs/server/balancer/Dispatcher.java | 2 +- .../server/balancer/NameNodeConnector.java | 5 +- .../server/blockmanagement/BlockManager.java | 7 +- .../hdfs/server/namenode/FSNamesystem.java | 9 +- .../server/namenode/NameNodeRpcServer.java | 4 +- .../server/protocol/NamenodeProtocol.java | 4 +- .../src/main/proto/NamenodeProtocol.proto | 1 + .../org/apache/hadoop/hdfs/TestGetBlocks.java | 110 ++++++++++++++++-- .../hdfs/server/balancer/TestBalancer.java | 2 +- .../balancer/TestBalancerWithHANameNodes.java | 2 +- 15 files changed, 142 insertions(+), 36 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java index 278d282fd7e6f..a5a047d115cd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Map.Entry; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; @@ -53,7 +54,7 @@ public RouterNamenodeProtocol(RouterRpcServer server) { @Override public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, - long minBlockSize, long hotBlockTimeInterval) throws IOException { + long minBlockSize, long hotBlockTimeInterval, StorageType storageType) throws IOException { rpcServer.checkOperation(OperationCategory.READ); // Get the namespace where the datanode is located @@ -79,8 +80,8 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, if (nsId != null) { RemoteMethod method = new RemoteMethod( NamenodeProtocol.class, "getBlocks", new Class[] - {DatanodeInfo.class, long.class, long.class, long.class}, - datanode, size, minBlockSize, hotBlockTimeInterval); + {DatanodeInfo.class, long.class, long.class, long.class, StorageType.class}, + datanode, size, minBlockSize, hotBlockTimeInterval, storageType); return rpcClient.invokeSingle(nsId, method, BlocksWithLocations.class); } return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index cae61b7d927dd..2aa2eae5305d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -1612,9 +1612,9 @@ public DatanodeInfo[] getSlowDatanodeReport() throws IOException { @Override // NamenodeProtocol public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, - long minBlockSize, long hotBlockTimeInterval) throws IOException { + long minBlockSize, long hotBlockTimeInterval, StorageType storageType) throws IOException { return nnProto.getBlocks(datanode, size, minBlockSize, - hotBlockTimeInterval); + hotBlockTimeInterval, storageType); } @Override // NamenodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index d44b40b052385..93e905b4eafff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -1385,9 +1385,11 @@ public void testProxyGetBlocks() throws Exception { // Verify that checking that datanode works BlocksWithLocations routerBlockLocations = - routerNamenodeProtocol.getBlocks(dn0, 1024, 0, 0); + routerNamenodeProtocol.getBlocks(dn0, 1024, 0, 0, + null); BlocksWithLocations nnBlockLocations = - nnNamenodeProtocol.getBlocks(dn0, 1024, 0, 0); + nnNamenodeProtocol.getBlocks(dn0, 1024, 0, 0, + null); BlockWithLocations[] routerBlocks = routerBlockLocations.getBlocks(); BlockWithLocations[] nnBlocks = nnBlockLocations.getBlocks(); assertEquals(nnBlocks.length, routerBlocks.length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java index e89a6b62b507d..f4025366391c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java @@ -89,7 +89,9 @@ public GetBlocksResponseProto getBlocks(RpcController unused, BlocksWithLocations blocks; try { blocks = impl.getBlocks(dnInfo, request.getSize(), - request.getMinBlockSize(), request.getTimeInterval()); + request.getMinBlockSize(), request.getTimeInterval(), + request.hasStorageType() ? + PBHelperClient.convertStorageType(request.getStorageType()): null); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java index fd40e0ecef34c..87518aa1e231a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.NamenodeCommandProto; @@ -101,11 +102,15 @@ public Object getUnderlyingProxyObject() { @Override public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long - minBlockSize, long timeInterval) + minBlockSize, long timeInterval, StorageType storageType) throws IOException { - GetBlocksRequestProto req = GetBlocksRequestProto.newBuilder() + GetBlocksRequestProto.Builder builder = GetBlocksRequestProto.newBuilder() .setDatanode(PBHelperClient.convert((DatanodeID)datanode)).setSize(size) - .setMinBlockSize(minBlockSize).setTimeInterval(timeInterval).build(); + .setMinBlockSize(minBlockSize).setTimeInterval(timeInterval); + if (storageType != null) { + builder.setStorageType(PBHelperClient.convertStorageType(storageType)); + } + GetBlocksRequestProto req = builder.build(); return PBHelper.convert(ipc(() -> rpcProxy.getBlocks(NULL_CONTROLLER, req) .getBlocks())); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index 98a6d8449b629..6ad0e4d22a854 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -839,7 +839,7 @@ private long getBlockList() throws IOException, IllegalArgumentException { final long size = Math.min(getBlocksSize, blocksToReceive); final BlocksWithLocations newBlksLocs = nnc.getBlocks(getDatanodeInfo(), size, getBlocksMinBlockSize, - hotBlockTimeInterval); + hotBlockTimeInterval, storageType); if (LOG.isTraceEnabled()) { LOG.trace("getBlocks(" + getDatanodeInfo() + ", " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index 34be025203d47..e8274fdbe779b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter; import org.apache.hadoop.ha.HAServiceProtocol; @@ -255,7 +256,7 @@ public URI getNameNodeUri() { /** @return blocks with locations. */ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long - minBlockSize, long timeInterval) throws IOException { + minBlockSize, long timeInterval, StorageType storageType) throws IOException { if (getBlocksRateLimiter != null) { getBlocksRateLimiter.acquire(); } @@ -274,7 +275,7 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long } else { nnProxy = namenode; } - return nnProxy.getBlocks(datanode, size, minBlockSize, timeInterval); + return nnProxy.getBlocks(datanode, size, minBlockSize, timeInterval, storageType); } finally { if (isRequestStandby) { LOG.info("Request #getBlocks to Standby NameNode success. " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2351bb4782873..2d216be945772 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1720,8 +1720,8 @@ private boolean isHotBlock(BlockInfo blockInfo, long time) { /** Get all blocks with location information from a datanode. */ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode, - final long size, final long minBlockSize, final long timeInterval) throws - UnregisteredNodeException { + final long size, final long minBlockSize, final long timeInterval, + final StorageType storageType) throws UnregisteredNodeException { final DatanodeDescriptor node = getDatanodeManager().getDatanode(datanode); if (node == null) { blockLog.warn("BLOCK* getBlocks: Asking for blocks from an" + @@ -1735,10 +1735,11 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode, return new BlocksWithLocations(new BlockWithLocations[0]); } - // skip stale storage + // skip stale storage, then choose specific storage type. DatanodeStorageInfo[] storageInfos = Arrays .stream(node.getStorageInfos()) .filter(s -> !s.areBlockContentsStale()) + .filter(s -> storageType == null || s.getStorageType().equals(storageType)) .toArray(DatanodeStorageInfo[]::new); // starting from a random block diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3d360c6d0dd2a..7918daf6b9db8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1946,10 +1946,13 @@ public boolean isInStandbyState() { * * @param datanode on which blocks are located * @param size total size of blocks - * @param minimumBlockSize + * @param minimumBlockSize each block should be of this minimum Block Size + * @param timeInterval prefer to get blocks which are belong to + * the cold files accessed before the time interval + * @param storageType the given storage type {@link StorageType} */ public BlocksWithLocations getBlocks(DatanodeID datanode, long size, long - minimumBlockSize, long timeInterval) throws IOException { + minimumBlockSize, long timeInterval, StorageType storageType) throws IOException { OperationCategory checkOp = isGetBlocksCheckOperationEnabled ? OperationCategory.READ : OperationCategory.UNCHECKED; @@ -1958,7 +1961,7 @@ public BlocksWithLocations getBlocks(DatanodeID datanode, long size, long try { checkOperation(checkOp); return getBlockManager().getBlocksWithLocations(datanode, size, - minimumBlockSize, timeInterval); + minimumBlockSize, timeInterval, storageType); } finally { readUnlock("getBlocks"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 4a041dbec2758..f02688d1629f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -649,7 +649,7 @@ private static UserGroupInformation getRemoteUser() throws IOException { ///////////////////////////////////////////////////// @Override // NamenodeProtocol public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long - minBlockSize, long timeInterval) + minBlockSize, long timeInterval, StorageType storageType) throws IOException { String operationName = "getBlocks"; if(size <= 0) { @@ -663,7 +663,7 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long checkNNStartup(); namesystem.checkSuperuserPrivilege(operationName); namesystem.checkNameNodeSafeMode("Cannot execute getBlocks"); - return namesystem.getBlocks(datanode, size, minBlockSize, timeInterval); + return namesystem.getBlocks(datanode, size, minBlockSize, timeInterval, storageType); } @Override // NamenodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java index 44ffb85f79ece..03ddc5ef8b1e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; @@ -76,6 +77,7 @@ public interface NamenodeProtocol { * @param minBlockSize each block should be of this minimum Block Size * @param hotBlockTimeInterval prefer to get blocks which are belong to * the cold files accessed before the time interval + * @param storageType the given storage type {@link StorageType} * @return BlocksWithLocations a list of blocks & their locations * @throws IOException if size is less than or equal to 0 or datanode does not exist @@ -83,7 +85,7 @@ public interface NamenodeProtocol { @Idempotent @ReadOnly BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long - minBlockSize, long hotBlockTimeInterval) throws IOException; + minBlockSize, long hotBlockTimeInterval, StorageType storageType) throws IOException; /** * Get the current block keys diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto index 32cdade055eee..29a9aa01b68d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto @@ -48,6 +48,7 @@ message GetBlocksRequestProto { // For more info refer HDFS-13356 optional uint64 minBlockSize = 3 [default = 10485760]; optional uint64 timeInterval = 4 [default = 0]; + optional StorageTypeProto storageType = 5; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java index cad7d0fb5026a..5abb8adc14e8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java @@ -20,6 +20,7 @@ import static org.junit.Assert.*; import java.net.InetSocketAddress; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -36,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -239,26 +241,29 @@ public void testGetBlocks() throws Exception { DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy(); // Should return all 13 blocks, as minBlockSize is not passed - locs = namenode.getBlocks(dataNodes[0], fileLen, 0, 0).getBlocks(); + locs = namenode.getBlocks(dataNodes[0], fileLen, 0, 0, + null).getBlocks(); assertEquals(blkLocsSize, locs.length); assertEquals(locs[0].getStorageIDs().length, replicationFactor); assertEquals(locs[1].getStorageIDs().length, replicationFactor); // Should return 12 blocks, as minBlockSize is blkSize - locs = namenode.getBlocks(dataNodes[0], fileLen, blkSize, 0).getBlocks(); + locs = namenode.getBlocks(dataNodes[0], fileLen, blkSize, 0, + null).getBlocks(); assertEquals(blkLocsSize - 1, locs.length); assertEquals(locs[0].getStorageIDs().length, replicationFactor); assertEquals(locs[1].getStorageIDs().length, replicationFactor); // get blocks of size BlockSize from dataNodes[0] locs = namenode.getBlocks(dataNodes[0], blkSize, - blkSize, 0).getBlocks(); + blkSize, 0, null).getBlocks(); assertEquals(locs.length, 1); assertEquals(locs[0].getStorageIDs().length, replicationFactor); // get blocks of size 1 from dataNodes[0] - locs = namenode.getBlocks(dataNodes[0], 1, 1, 0).getBlocks(); + locs = namenode.getBlocks(dataNodes[0], 1, 1, 0, + null).getBlocks(); assertEquals(locs.length, 1); assertEquals(locs[0].getStorageIDs().length, replicationFactor); @@ -283,7 +288,8 @@ public void testGetBlocks() throws Exception { // Namenode should refuse to provide block locations to the balancer // while in safemode. - locs = namenode.getBlocks(dataNodes[0], fileLen, 0, 0).getBlocks(); + locs = namenode.getBlocks(dataNodes[0], fileLen, 0, 0, + null).getBlocks(); assertEquals(blkLocsSize, locs.length); assertFalse(fs.isInSafeMode()); LOG.info("Entering safe mode"); @@ -310,7 +316,8 @@ private void getBlocksWithException(NamenodeProtocol namenode, // Namenode should refuse should fail LambdaTestUtils.intercept(exClass, - msg, () -> namenode.getBlocks(datanode, size, minBlkSize, 0)); + msg, () -> namenode.getBlocks(datanode, size, minBlkSize, 0, + null)); } /** @@ -450,7 +457,7 @@ public void testGetBlocksWithHotBlockTimeInterval() throws Exception { .getBlockLocations(fileNew, 0, fileLen).getLocatedBlocks(); BlockWithLocations[] locsAll = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, hotInterval).getBlocks(); + dataNodes[0], fileLen*2, 0, hotInterval, null).getBlocks(); assertEquals(locsAll.length, 4); for(int i = 0; i < blockNum; i++) { @@ -461,7 +468,7 @@ public void testGetBlocksWithHotBlockTimeInterval() throws Exception { } BlockWithLocations[] locs2 = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, hotInterval).getBlocks(); + dataNodes[0], fileLen*2, 0, hotInterval, null).getBlocks(); for(int i = 0; i < 2; i++) { assertTrue(belongToFile(locs2[i], locatedBlocksOld)); } @@ -508,7 +515,7 @@ public void testReadSkipStaleStorage() throws Exception { // check blocks count equals to blockNum BlockWithLocations[] blocks = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, 0).getBlocks(); + dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); assertEquals(blockNum, blocks.length); // calculate the block count on storage[0] @@ -524,13 +531,94 @@ public void testReadSkipStaleStorage() throws Exception { // set storage[0] stale storageInfos[0].setBlockContentsStale(true); blocks = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, 0).getBlocks(); + dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); assertEquals(blockNum - count, blocks.length); // set all storage stale bm0.getDatanodeManager().markAllDatanodesStale(); blocks = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, 0).getBlocks(); + dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); assertEquals(0, blocks.length); } + + @Test + public void testChooseSpecifyStorageType() throws Exception { + final short repFactor = (short) 1; + final int fileLen = BLOCK_SIZE; + final Configuration conf = new HdfsConfiguration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .storageTypes(new StorageType[] {StorageType.DISK, StorageType.SSD}). + storagesPerDatanode(2).build()) { + cluster.waitActive(); + + // Get storage info. + ClientProtocol client = NameNodeProxies.createProxy(conf, + cluster.getFileSystem(0).getUri(), + ClientProtocol.class).getProxy(); + DatanodeInfo[] dataNodes = client.getDatanodeReport(DatanodeReportType.ALL); + BlockManager bm0 = cluster.getNamesystem(0).getBlockManager(); + DatanodeStorageInfo[] storageInfos = bm0.getDatanodeManager() + .getDatanode(dataNodes[0].getDatanodeUuid()).getStorageInfos(); + assert Arrays.stream(storageInfos) + .anyMatch(datanodeStorageInfo -> { + String storageTypeName = datanodeStorageInfo.getStorageType().name(); + return storageTypeName.equals("SSD") || storageTypeName.equals("DISK"); + }) : "No 'SSD' or 'DISK' storage types found."; + + // Create hdfs file. + Path ssdDir = new Path("/testChooseSSD"); + DistributedFileSystem fs = cluster.getFileSystem(); + Path ssdFile = new Path(ssdDir, "file"); + fs.mkdirs(ssdDir); + fs.setStoragePolicy(ssdDir, "ALL_SSD"); + DFSTestUtil.createFile(fs, ssdFile, false, 1024, fileLen, + BLOCK_SIZE, repFactor, 0, true); + DFSTestUtil.waitReplication(fs, ssdFile, repFactor); + BlockLocation[] locations = fs.getClient() + .getBlockLocations(ssdFile.toUri().getPath(), 0, Long.MAX_VALUE); + assertEquals(1, locations.length); + assertEquals("SSD", locations[0].getStorageTypes()[0].name()); + + Path diskDir = new Path("/testChooseDisk"); + fs = cluster.getFileSystem(); + Path diskFile = new Path(diskDir, "file"); + fs.mkdirs(diskDir); + fs.setStoragePolicy(diskDir, "HOT"); + DFSTestUtil.createFile(fs, diskFile, false, 1024, fileLen, + BLOCK_SIZE, repFactor, 0, true); + DFSTestUtil.waitReplication(fs, diskFile, repFactor); + locations = fs.getClient() + .getBlockLocations(diskFile.toUri().getPath(), 0, Long.MAX_VALUE); + assertEquals(1, locations.length); + assertEquals("DISK", locations[0].getStorageTypes()[0].name()); + + InetSocketAddress addr = new InetSocketAddress("localhost", + cluster.getNameNodePort()); + NamenodeProtocol namenode = NameNodeProxies.createProxy(conf, + DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy(); + + // Check blocks count equals to blockNum. + // If StorageType is not specified will get all blocks. + BlockWithLocations[] blocks = namenode.getBlocks( + dataNodes[0], fileLen * 2, 0, 0, + null).getBlocks(); + assertEquals(2, blocks.length); + + // Check the count of blocks with a StorageType of DISK. + blocks = namenode.getBlocks( + dataNodes[0], fileLen * 2, 0, 0, + StorageType.DISK).getBlocks(); + assertEquals(1, blocks.length); + assertEquals("DISK", blocks[0].getStorageTypes()[0].name()); + + // Check the count of blocks with a StorageType of SSD. + blocks = namenode.getBlocks( + dataNodes[0], fileLen * 2, 0, 0, + StorageType.SSD).getBlocks(); + assertEquals(1, blocks.length); + assertEquals("SSD", blocks[0].getStorageTypes()[0].name()); + } + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 2a56c25d0d46e..23d1cb441bb8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1891,7 +1891,7 @@ public BlocksWithLocations answer(InvocationOnMock invocation) numGetBlocksCalls.incrementAndGet(); return blk; }}).when(fsnSpy).getBlocks(any(DatanodeID.class), - anyLong(), anyLong(), anyLong()); + anyLong(), anyLong(), anyLong(), any(StorageType.class)); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java index d69051c8d7af7..dbd76ee614515 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java @@ -277,7 +277,7 @@ private void testBalancerWithObserver(boolean withObserverFailure) int expectedObserverIdx = withObserverFailure ? 3 : 2; int expectedCount = (i == expectedObserverIdx) ? 2 : 0; verify(namesystemSpies.get(i), times(expectedCount)) - .getBlocks(any(), anyLong(), anyLong(), anyLong()); + .getBlocks(any(), anyLong(), anyLong(), anyLong(), any()); } } finally { if (qjmhaCluster != null) { From 077263d9f3205b8bdc62467271322c662570e5c5 Mon Sep 17 00:00:00 2001 From: Benjamin Teke Date: Mon, 6 Nov 2023 10:11:14 +0100 Subject: [PATCH 145/155] YARN-11608. Fix QueueCapacityVectorInfo NPE when accessible labels config is used. (#6250) Co-authored-by: Benjamin Teke --- .../webapp/dao/QueueCapacitiesInfo.java | 7 +- .../TestRMWebServicesCapacitySched.java | 1 + ...scheduler-response-NodeLabelDefaultAPI.xml | 260 +++++++++++++++++- .../scheduler-response-PerUserResources.json | 248 ++++++++++++++++- .../scheduler-response-PerUserResources.xml | 260 +++++++++++++++++- .../resources/webapp/scheduler-response.json | 248 ++++++++++++++++- .../resources/webapp/scheduler-response.xml | 260 +++++++++++++++++- 7 files changed, 1277 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java index 293966b5b8600..3f42a626163c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; /** * DAO which wraps PartitionQueueCapacitiesInfo applicable for a queue @@ -57,8 +58,10 @@ public QueueCapacitiesInfo(CSQueue queue, boolean considerAMUsage) { float weight; float normalizedWeight; for (String partitionName : capacities.getExistingNodeLabels()) { - queueCapacityVectorInfo = new QueueCapacityVectorInfo( - queue.getConfiguredCapacityVector(partitionName)); + QueueCapacityVector queueCapacityVector = queue.getConfiguredCapacityVector(partitionName); + queueCapacityVectorInfo = queueCapacityVector == null ? + new QueueCapacityVectorInfo(new QueueCapacityVector()) : + new QueueCapacityVectorInfo(queue.getConfiguredCapacityVector(partitionName)); usedCapacity = capacities.getUsedCapacity(partitionName) * 100; capacity = capacities.getCapacity(partitionName) * 100; maxCapacity = capacities.getMaximumCapacity(partitionName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 19e58c018fcf3..83b4578b48004 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -150,6 +150,7 @@ private Configuration createConfig() { conf.set("yarn.scheduler.capacity.legacy-queue-mode.enabled", String.valueOf(legacyQueueMode)); conf.set("yarn.scheduler.capacity.root.queues", "a, b, c"); conf.set("yarn.scheduler.capacity.root.a.capacity", "12.5"); + conf.set("yarn.scheduler.capacity.root.a.accessible-node-labels", "root-a-default-label"); conf.set("yarn.scheduler.capacity.root.a.maximum-capacity", "50"); conf.set("yarn.scheduler.capacity.root.a.max-parallel-app", "42"); conf.set("yarn.scheduler.capacity.root.b.capacity", "50"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml index d91322721b9de..4dfa5d9246675 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml @@ -61,7 +61,7 @@ false - * + root-a-default-label 0 0 0 @@ -185,6 +185,117 @@ + + root-a-default-label + + [] + + 0.0 + 0.0 + 100.0 + 0.0 + 0.0 + 0.0 + 0.0 + -1.0 + 0.0 + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + @@ -334,6 +445,153 @@ + + root-a-default-label + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + 4096 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json index 7869b3868ddb5..94976d796cd4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json @@ -62,7 +62,7 @@ } }, "hideReservationQueues" : false, - "nodeLabels" : [ "*" ], + "nodeLabels" : [ "root-a-default-label" ], "allocatedContainers" : 0, "reservedContainers" : 0, "pendingContainers" : 1, @@ -180,6 +180,112 @@ } ] } } + }, { + "partitionName" : "root-a-default-label", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[]" + }, + "capacity" : 0, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 0, + "absoluteUsedCapacity" : 0, + "absoluteMaxCapacity" : 0, + "maxAMLimitPercentage" : 0, + "weight" : -1, + "normalizedWeight" : 0, + "configuredMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "configuredMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } } ] }, "resources" : { @@ -323,6 +429,146 @@ } ] } } + }, { + "partitionName" : "root-a-default-label", + "used" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "reserved" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "pending" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "userAmLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } } ] }, "minEffectiveCapacity" : { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml index d75879059df31..ca4e6901a5c65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml @@ -61,7 +61,7 @@ false - * + root-a-default-label 0 0 1 @@ -185,6 +185,117 @@ + + root-a-default-label + + [] + + 0.0 + 0.0 + 100.0 + 0.0 + 0.0 + 0.0 + 0.0 + -1.0 + 0.0 + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + @@ -334,6 +445,153 @@ + + root-a-default-label + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + 4096 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json index f44adf3af9b6c..202f68107c22b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json @@ -62,7 +62,7 @@ } }, "hideReservationQueues" : false, - "nodeLabels" : [ "*" ], + "nodeLabels" : [ "root-a-default-label" ], "allocatedContainers" : 0, "reservedContainers" : 0, "pendingContainers" : 0, @@ -180,6 +180,112 @@ } ] } } + }, { + "partitionName" : "root-a-default-label", + "queueCapacityVectorInfo" : { + "configuredCapacityVector" : "[]" + }, + "capacity" : 0, + "usedCapacity" : 0, + "maxCapacity" : 100, + "absoluteCapacity" : 0, + "absoluteUsedCapacity" : 0, + "absoluteMaxCapacity" : 0, + "maxAMLimitPercentage" : 0, + "weight" : -1, + "normalizedWeight" : 0, + "configuredMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "configuredMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMinResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "effectiveMaxResource" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 8192, + "minimumAllocation" : 1024, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 4, + "minimumAllocation" : 1, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } } ] }, "resources" : { @@ -323,6 +429,146 @@ } ] } } + }, { + "partitionName" : "root-a-default-label", + "used" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "reserved" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "pending" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amUsed" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "amLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + }, + "userAmLimit" : { + "memory" : 0, + "vCores" : 0, + "resourceInformations" : { + "resourceInformation" : [ { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "memory-mb", + "resourceType" : "COUNTABLE", + "units" : "Mi", + "value" : 0 + }, { + "attributes" : { }, + "maximumAllocation" : 9223372036854775807, + "minimumAllocation" : 0, + "name" : "vcores", + "resourceType" : "COUNTABLE", + "units" : "", + "value" : 0 + } ] + } + } } ] }, "minEffectiveCapacity" : { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml index 91e06c9551c2d..1742c7a765536 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml @@ -61,7 +61,7 @@ false - * + root-a-default-label 0 0 0 @@ -185,6 +185,117 @@ + + root-a-default-label + + [] + + 0.0 + 0.0 + 100.0 + 0.0 + 0.0 + 0.0 + 0.0 + -1.0 + 0.0 + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 8192 + 1024 + memory-mb + COUNTABLE + Mi + 0 + + + + 4 + 1 + vcores + COUNTABLE + + 0 + + + + @@ -334,6 +445,153 @@ + + root-a-default-label + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + + 0 + 0 + + + + 9223372036854775807 + 0 + memory-mb + COUNTABLE + Mi + 0 + + + + 9223372036854775807 + 0 + vcores + COUNTABLE + + 0 + + + + 4096 From ef7fb64764a9b9c77b0dbda48285f186cfcf5c4f Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 6 Nov 2023 16:00:56 +0000 Subject: [PATCH 146/155] HADOOP-18925. S3A: option to enable/disable CopyFromLocalOperation (#6163) Add a new option: fs.s3a.optimized.copy.from.local.enabled This will enable (default) or disable the optimized CopyFromLocalOperation upload operation when copyFromLocalFile() is invoked. When false the superclass implementation is used; duration statistics are still collected, though audit span entries in logs will be for the individual fs operations, not the overall operation. Contributed by Steve Loughran --- .../org/apache/hadoop/fs/s3a/Constants.java | 13 +++ .../apache/hadoop/fs/s3a/S3AFileSystem.java | 93 +++++++++++++------ .../tools/hadoop-aws/troubleshooting_s3a.md | 10 ++ .../fs/s3a/ITestS3ACopyFromLocalFile.java | 51 ++++++++++ 4 files changed, 138 insertions(+), 29 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index f4aeccf1efd10..36c98f2353bea 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1347,4 +1347,17 @@ private Constants() { */ public static final boolean DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT = false; + + /** + * Is the higher performance copy from local file to S3 enabled? + * This switch allows for it to be disabled if there are problems. + * Value: {@value}. + */ + public static final String OPTIMIZED_COPY_FROM_LOCAL = "fs.s3a.optimized.copy.from.local.enabled"; + + /** + * Default value for {@link #OPTIMIZED_COPY_FROM_LOCAL}. + * Value: {@value}. + */ + public static final boolean OPTIMIZED_COPY_FROM_LOCAL_DEFAULT = true; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index f96a378b1cc92..b978707ce00ec 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -462,6 +462,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private String scheme = FS_S3A; + /** + * Flag to indicate that the higher performance copyFromLocalFile implementation + * should be used. + */ + private boolean optimizedCopyFromLocal; + /** Add any deprecated keys. */ @SuppressWarnings("deprecation") private static void addDeprecatedKeys() { @@ -696,6 +702,9 @@ public void initialize(URI name, Configuration originalConf) AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1); vectoredIOContext = populateVectoredIOContext(conf); scheme = (this.uri != null && this.uri.getScheme() != null) ? this.uri.getScheme() : FS_S3A; + optimizedCopyFromLocal = conf.getBoolean(OPTIMIZED_COPY_FROM_LOCAL, + OPTIMIZED_COPY_FROM_LOCAL_DEFAULT); + LOG.debug("Using optimized copyFromLocal implementation: {}", optimizedCopyFromLocal); } catch (SdkException e) { // amazon client exception: stop all services then throw the translation cleanupWithLogger(LOG, span); @@ -4021,9 +4030,9 @@ private boolean s3Exists(final Path path, final Set probes) * the given dst name. * * This version doesn't need to create a temporary file to calculate the md5. - * Sadly this doesn't seem to be used by the shell cp :( + * If {@link Constants#OPTIMIZED_COPY_FROM_LOCAL} is set to false, + * the superclass implementation is used. * - * delSrc indicates if the source should be removed * @param delSrc whether to delete the src * @param overwrite whether to overwrite an existing file * @param src path @@ -4031,35 +4040,59 @@ private boolean s3Exists(final Path path, final Set probes) * @throws IOException IO problem * @throws FileAlreadyExistsException the destination file exists and * overwrite==false - * @throws SdkException failure in the AWS SDK */ @Override @AuditEntryPoint public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { checkNotClosed(); - LOG.debug("Copying local file from {} to {}", src, dst); - trackDurationAndSpan(INVOCATION_COPY_FROM_LOCAL_FILE, dst, - () -> new CopyFromLocalOperation( - createStoreContext(), - src, - dst, - delSrc, - overwrite, - createCopyFromLocalCallbacks()).execute()); + LOG.debug("Copying local file from {} to {} (delSrc={} overwrite={}", + src, dst, delSrc, overwrite); + if (optimizedCopyFromLocal) { + trackDurationAndSpan(INVOCATION_COPY_FROM_LOCAL_FILE, dst, () -> + new CopyFromLocalOperation( + createStoreContext(), + src, + dst, + delSrc, + overwrite, + createCopyFromLocalCallbacks(getActiveAuditSpan())) + .execute()); + } else { + // call the superclass, but still count statistics. + // there is no overall span here, as each FS API call will + // be in its own span. + LOG.debug("Using base copyFromLocalFile implementation"); + trackDurationAndSpan(INVOCATION_COPY_FROM_LOCAL_FILE, dst, () -> { + super.copyFromLocalFile(delSrc, overwrite, src, dst); + return null; + }); + } } + /** + * Create the CopyFromLocalCallbacks; + * protected to assist in mocking. + * @param span audit span. + * @return the callbacks + * @throws IOException failure to get the local fs. + */ protected CopyFromLocalOperation.CopyFromLocalOperationCallbacks - createCopyFromLocalCallbacks() throws IOException { + createCopyFromLocalCallbacks(final AuditSpanS3A span) throws IOException { LocalFileSystem local = getLocal(getConf()); - return new CopyFromLocalCallbacksImpl(local); + return new CopyFromLocalCallbacksImpl(span, local); } protected final class CopyFromLocalCallbacksImpl implements CopyFromLocalOperation.CopyFromLocalOperationCallbacks { + + /** Span to use for all operations. */ + private final AuditSpanS3A span; private final LocalFileSystem local; - private CopyFromLocalCallbacksImpl(LocalFileSystem local) { + private CopyFromLocalCallbacksImpl(final AuditSpanS3A span, + LocalFileSystem local) { + this.span = span; this.local = local; } @@ -4081,20 +4114,18 @@ public boolean deleteLocal(Path path, boolean recursive) throws IOException { @Override public void copyLocalFileFromTo(File file, Path from, Path to) throws IOException { - trackDurationAndSpan( - OBJECT_PUT_REQUESTS, - to, - () -> { - final String key = pathToKey(to); - Progressable progress = null; - PutObjectRequest.Builder putObjectRequestBuilder = - newPutObjectRequestBuilder(key, file.length(), false); - S3AFileSystem.this.invoker.retry("putObject(" + "" + ")", to.toString(), true, - () -> executePut(putObjectRequestBuilder.build(), progress, putOptionsForPath(to), - file)); - - return null; - }); + // the duration of the put is measured, but the active span is the + // constructor-supplied one -this ensures all audit log events are grouped correctly + span.activate(); + trackDuration(getDurationTrackerFactory(), OBJECT_PUT_REQUESTS.getSymbol(), () -> { + final String key = pathToKey(to); + PutObjectRequest.Builder putObjectRequestBuilder = + newPutObjectRequestBuilder(key, file.length(), false); + final String dest = to.toString(); + S3AFileSystem.this.invoker.retry("putObject(" + dest + ")", dest, true, () -> + executePut(putObjectRequestBuilder.build(), null, putOptionsForPath(to), file)); + return null; + }); } @Override @@ -5399,6 +5430,10 @@ public boolean hasPathCapability(final Path path, final String capability) case FS_S3A_CREATE_PERFORMANCE_ENABLED: return performanceCreation; + // is the optimized copy from local enabled. + case OPTIMIZED_COPY_FROM_LOCAL: + return optimizedCopyFromLocal; + default: return super.hasPathCapability(p, cap); } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 8c57db1faf4c1..be359336463ad 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1544,3 +1544,13 @@ software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it completely by setting it to `0`. + +### Debugging Switches + +There are some switches which can be set to enable/disable features and assist +in isolating problems and at least make them "go away". + + +| Key | Default | Action | +|------|---------|----------| +| `fs.s3a.optimized.copy.from.local.enabled` | `true` | [HADOOP-18925](https://issues.apache.org/jira/browse/HADOOP-18925) enable/disable CopyFromLocalOperation. Also a path capability. | diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java index dfac771dd78dc..f9600de6d20cf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java @@ -19,6 +19,8 @@ package org.apache.hadoop.fs.s3a; import java.io.File; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractCopyFromLocalTest; @@ -26,18 +28,67 @@ import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.apache.hadoop.fs.Path; + +import org.assertj.core.api.Assertions; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import static org.apache.hadoop.fs.s3a.Constants.OPTIMIZED_COPY_FROM_LOCAL; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.test.LambdaTestUtils.intercept; +/** + * Test copying files from the local filesystem to S3A. + * Parameterized on whether or not the optimized + * copyFromLocalFile is enabled. + */ +@RunWith(Parameterized.class) public class ITestS3ACopyFromLocalFile extends AbstractContractCopyFromLocalTest { + /** + * Parameterization. + */ + @Parameterized.Parameters(name = "enabled={0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {true}, + {false}, + }); + } + private final boolean enabled; + + public ITestS3ACopyFromLocalFile(final boolean enabled) { + this.enabled = enabled; + } + + @Override + protected Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + + removeBaseAndBucketOverrides(getTestBucketName(conf), conf, + OPTIMIZED_COPY_FROM_LOCAL); + conf.setBoolean(OPTIMIZED_COPY_FROM_LOCAL, enabled); + disableFilesystemCaching(conf); + return conf; + } @Override protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); } + @Test + public void testOptionPropagation() throws Throwable { + Assertions.assertThat(getFileSystem().hasPathCapability(new Path("/"), + OPTIMIZED_COPY_FROM_LOCAL)) + .describedAs("path capability of %s", OPTIMIZED_COPY_FROM_LOCAL) + .isEqualTo(enabled); + + } + @Test public void testLocalFilesOnly() throws Throwable { describe("Copying into other file systems must fail"); From d634deea4ed7b018ece6cef6185abf98bb443338 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 6 Nov 2023 17:52:05 +0000 Subject: [PATCH 147/155] HADOOP-18487. Protobuf 2.5 removal part 2: stop exporting protobuf-2.5 (#6185) Followup to the previous HADOOP-18487 patch: changes the scope of protobuf-2.5 in hadoop-common and elsewhere from "compile" to "provided". This means that protobuf-2.5 is * No longer included in hadoop distributions * No longer exported by hadoop common POM files * No longer exported transitively by other hadoop modules. * No longer listed in LICENSE-binary. Contributed by Steve Loughran --- BUILDING.txt | 24 +++++++++++++++--------- LICENSE-binary | 1 - hadoop-project/pom.xml | 4 ++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 91d317ebe2a40..77561c5546fd3 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -319,15 +319,11 @@ Controlling the redistribution of the protobuf-2.5 dependency the Hadoop codebase; alongside the move to Protobuf 3.x a private successor class, org.apache.hadoop.ipc.internal.ShadedProtobufHelper is now used. - The hadoop-common JAR still declares a dependency on protobuf-2.5, but this - is likely to change in the future. The maven scope of the dependency can be - set with the common.protobuf2.scope option. - It can be set to "provided" in a build: - -Dcommon.protobuf2.scope=provided - If this is done then protobuf-2.5.0.jar will no longer be exported as a dependency, - and will then be omitted from the share/hadoop/common/lib/ directory of - any Hadoop distribution built. Any application declaring a dependency on hadoop-commmon - will no longer get the dependency; if they need it then they must explicitly declare it: + The hadoop-common module no longer exports its compile-time dependency on + protobuf-2.5. Hadoop distributions no longer include it. + Any application declaring a dependency on hadoop-commmon will no longer get + the artifact added to their classpath. + If is still required, then they must explicitly declare it: com.google.protobuf @@ -335,6 +331,16 @@ Controlling the redistribution of the protobuf-2.5 dependency 2.5.0 + In Hadoop builds the scope of the dependency can be set with the + option "common.protobuf2.scope". + This can be upgraded from "provided" to "compile" on the maven command line: + + -Dcommon.protobuf2.scope=compile + + If this is done then protobuf-2.5.0.jar will again be exported as a + hadoop-common dependency, and included in the share/hadoop/common/lib/ + directory of any Hadoop distribution built. + ---------------------------------------------------------------------------------- Building components separately diff --git a/LICENSE-binary b/LICENSE-binary index 3a0e19c5824de..0ec5ee30f5824 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -392,7 +392,6 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js leveldb v1.13 -com.google.protobuf:protobuf-java:2.5.0 com.google.protobuf:protobuf-java:3.6.1 com.google.re2j:re2j:1.1 com.jcraft:jsch:0.1.54 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 5b1c569d21de6..33e34a7d807b2 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -88,8 +88,8 @@ 2.5.0 - - compile + + provided ${common.protobuf2.scope} From 597ceaae3a779e8b31e6c5e1453a56093a455271 Mon Sep 17 00:00:00 2001 From: Anuj Modi <128447756+anujmodi2021@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:56:55 -0800 Subject: [PATCH 148/155] HADOOP-18874: [ABFS] Add Server request ID in Exception Messages thrown to the caller. (#6004) Contributed by Anuj Modi --- .../AbfsRestOperationException.java | 26 ++-- .../ITestAbfsRestOperationException.java | 121 ++++++++++++------ 2 files changed, 97 insertions(+), 50 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java index 6c53762363840..201b3bd2e52d7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java @@ -84,20 +84,22 @@ private static String formatMessage(final AbfsHttpOperation abfsHttpOperation) { // HEAD request response doesn't have StorageErrorCode, StorageErrorMessage. if (abfsHttpOperation.getMethod().equals("HEAD")) { return String.format( - "Operation failed: \"%1$s\", %2$s, HEAD, %3$s", - abfsHttpOperation.getStatusDescription(), - abfsHttpOperation.getStatusCode(), - abfsHttpOperation.getMaskedUrl()); + "Operation failed: \"%1$s\", %2$s, HEAD, %3$ss, rId: %4$s", + abfsHttpOperation.getStatusDescription(), + abfsHttpOperation.getStatusCode(), + abfsHttpOperation.getMaskedUrl(), + abfsHttpOperation.getRequestId()); } return String.format( - "Operation failed: \"%1$s\", %2$s, %3$s, %4$s, %5$s, \"%6$s\"", - abfsHttpOperation.getStatusDescription(), - abfsHttpOperation.getStatusCode(), - abfsHttpOperation.getMethod(), - abfsHttpOperation.getMaskedUrl(), - abfsHttpOperation.getStorageErrorCode(), - // Remove break line to ensure the request id and timestamp can be shown in console. - abfsHttpOperation.getStorageErrorMessage().replaceAll("\\n", " ")); + "Operation failed: \"%1$s\", %2$s, %3$s, %4$s, rId: %5$s, %6$s, \"%7$s\"", + abfsHttpOperation.getStatusDescription(), + abfsHttpOperation.getStatusCode(), + abfsHttpOperation.getMethod(), + abfsHttpOperation.getMaskedUrl(), + abfsHttpOperation.getRequestId(), + abfsHttpOperation.getStorageErrorCode(), + // Remove break line to ensure the request id and timestamp can be shown in console. + abfsHttpOperation.getStorageErrorMessage().replaceAll("\\n", " ")); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index 3fe3557d501dc..2672b676f9b3a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -21,14 +21,12 @@ import java.io.IOException; import org.assertj.core.api.Assertions; -import org.junit.Assert; import org.junit.Test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; import org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -43,7 +41,8 @@ * Verify the AbfsRestOperationException error message format. * */ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest{ - private static final String RETRY_TEST_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider"; + private static final String RETRY_TEST_TOKEN_PROVIDER = + "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider"; public ITestAbfsRestOperationException() throws Exception { super(); @@ -55,17 +54,35 @@ public void testAbfsRestOperationExceptionFormat() throws IOException { Path nonExistedFilePath1 = new Path("nonExistedPath1"); Path nonExistedFilePath2 = new Path("nonExistedPath2"); try { - FileStatus fileStatus = fs.getFileStatus(nonExistedFilePath1); + fs.getFileStatus(nonExistedFilePath1); } catch (Exception ex) { String errorMessage = ex.getLocalizedMessage(); String[] errorFields = errorMessage.split(","); - Assert.assertEquals(4, errorFields.length); + // Expected Fields are: Message, StatusCode, Method, URL, ActivityId(rId) + Assertions.assertThat(errorFields) + .describedAs("Number of Fields in exception message are not as expected") + .hasSize(5); // Check status message, status code, HTTP Request Type and URL. - Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim()); - Assert.assertEquals("404", errorFields[1].trim()); - Assert.assertEquals("HEAD", errorFields[2].trim()); - Assert.assertTrue(errorFields[3].trim().startsWith("http")); + Assertions.assertThat(errorFields[0].trim()) + .describedAs("Error Message Field in exception message is wrong") + .isEqualTo("Operation failed: \"The specified path does not exist.\""); + Assertions.assertThat(errorFields[1].trim()) + .describedAs("Status Code Field in exception message " + + "should be \"404\"") + .isEqualTo("404"); + Assertions.assertThat(errorFields[2].trim()) + .describedAs("Http Rest Method Field in exception message " + + "should be \"HEAD\"") + .isEqualTo("HEAD"); + Assertions.assertThat(errorFields[3].trim()) + .describedAs("Url Field in exception message" + + " should start with \"http\"") + .startsWith("http"); + Assertions.assertThat(errorFields[4].trim()) + .describedAs("ActivityId Field in exception message " + + "should start with \"rId:\"") + .startsWith("rId:"); } try { @@ -74,18 +91,43 @@ public void testAbfsRestOperationExceptionFormat() throws IOException { // verify its format String errorMessage = ex.getLocalizedMessage(); String[] errorFields = errorMessage.split(","); + // Expected Fields are: Message, StatusCode, Method, URL, ActivityId(rId), StorageErrorCode, StorageErrorMessage. Assertions.assertThat(errorFields) - .describedAs("fields in exception of %s", ex) - .hasSize(6); + .describedAs("Number of Fields in exception message are not as expected") + .hasSize(7); // Check status message, status code, HTTP Request Type and URL. - Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim()); - Assert.assertEquals("404", errorFields[1].trim()); - Assert.assertEquals("GET", errorFields[2].trim()); - Assert.assertTrue(errorFields[3].trim().startsWith("http")); + Assertions.assertThat(errorFields[0].trim()) + .describedAs("Error Message Field in exception message is wrong") + .isEqualTo("Operation failed: \"The specified path does not exist.\""); + Assertions.assertThat(errorFields[1].trim()) + .describedAs("Status Code Field in exception message" + + " should be \"404\"") + .isEqualTo("404"); + Assertions.assertThat(errorFields[2].trim()) + .describedAs("Http Rest Method Field in exception message" + + " should be \"GET\"") + .isEqualTo("GET"); + Assertions.assertThat(errorFields[3].trim()) + .describedAs("Url Field in exception message" + + " should start with \"http\"") + .startsWith("http"); + Assertions.assertThat(errorFields[4].trim()) + .describedAs("ActivityId Field in exception message" + + " should start with \"rId:\"") + .startsWith("rId:"); // Check storage error code and storage error message. - Assert.assertEquals("PathNotFound", errorFields[4].trim()); - Assert.assertTrue(errorFields[5].contains("RequestId") - && errorFields[5].contains("Time")); + Assertions.assertThat(errorFields[5].trim()) + .describedAs("StorageErrorCode Field in exception message" + + " should be \"PathNotFound\"") + .isEqualTo("PathNotFound"); + Assertions.assertThat(errorFields[6].trim()) + .describedAs("StorageErrorMessage Field in exception message" + + " should contain \"RequestId\"") + .contains("RequestId"); + Assertions.assertThat(errorFields[6].trim()) + .describedAs("StorageErrorMessage Field in exception message" + + " should contain \"Time\"") + .contains("Time"); } } @@ -101,7 +143,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce Configuration config = new Configuration(this.getRawConfiguration()); String accountName = config.get("fs.azure.abfs.account.name"); - // Setup to configure custom token provider + // Setup to configure custom token provider. config.set("fs.azure.account.auth.type." + accountName, "Custom"); config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs" + ".azurebfs.oauth2.RetryTestTokenProvider"); @@ -109,24 +151,25 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce // Stop filesystem creation as it will lead to calls to store. config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false"); - final AzureBlobFileSystem fs1 = + try (final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), - config); - RetryTestTokenProvider retryTestTokenProvider - = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( - getAccessTokenProvider(fs1)); - retryTestTokenProvider.resetStatusToFirstTokenFetch(); - - intercept(Exception.class, - ()-> { - fs1.getFileStatus(new Path("/")); - }); - - // Number of retries done should be as configured - Assert.assertEquals( - "Number of token fetch retries done does not match with fs.azure" - + ".custom.token.fetch.retry.count configured", numOfRetries, - retryTestTokenProvider.getRetryCount()); + config)) { + RetryTestTokenProvider retryTestTokenProvider + = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( + getAccessTokenProvider(fs1)); + retryTestTokenProvider.resetStatusToFirstTokenFetch(); + + intercept(Exception.class, + () -> { + fs1.getFileStatus(new Path("/")); + }); + + // Number of retries done should be as configured + Assertions.assertThat(retryTestTokenProvider.getRetryCount()) + .describedAs("Number of token fetch retries done does not " + + "match with fs.azure.custom.token.fetch.retry.count configured") + .isEqualTo(numOfRetries); + } } @Test @@ -145,8 +188,10 @@ public void testAuthFailException() throws Exception { final AzureBlobFileSystem fs = getFileSystem(config); try { - fs.getFileStatus(new Path("/")); - fail("Should fail at auth token fetch call"); + intercept(Exception.class, + () -> { + fs.getFileStatus(new Path("/")); + }); } catch (AbfsRestOperationException e) { String errorDesc = "Should throw RestOp exception on AAD failure"; Assertions.assertThat(e.getStatusCode()) From 72d7b43a32c63250d5fb6e3a149846a669c15abe Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 8 Nov 2023 07:41:56 +0800 Subject: [PATCH 149/155] YARN-11548. [Federation] Router Supports Format FederationStateStore. (#6116) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../store/FederationStateStore.java | 7 ++ .../impl/MemoryFederationStateStore.java | 10 +++ .../store/impl/SQLFederationStateStore.java | 36 ++++++++ .../impl/ZookeeperFederationStateStore.java | 88 +++++++++++++------ .../store/sql/FederationQueryRunner.java | 33 +++++++ .../utils/FederationStateStoreFacade.java | 4 + .../impl/FederationStateStoreBaseTest.java | 24 +++++ .../impl/TestSQLFederationStateStore.java | 3 +- .../FederationStateStoreService.java | 5 ++ .../hadoop/yarn/server/router/Router.java | 14 ++- 10 files changed, 191 insertions(+), 33 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java index e197339671466..31ea6578dd8d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java @@ -101,4 +101,11 @@ default void checkVersion() throws Exception { ", but loading version " + loadedVersion); } } + + /** + * We will clear the data in stateStore through the deleteStateStore method. + * + * @throws Exception an exception occurred in delete store. + */ + void deleteStateStore() throws Exception; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 05c7c500fb9b5..c652d300268bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -419,6 +419,16 @@ public void storeVersion() throws Exception { version = ((VersionPBImpl) CURRENT_VERSION_INFO).getProto().toByteArray(); } + @Override + public void deleteStateStore() throws Exception { + membership.clear(); + applications.clear(); + reservations.clear(); + policies.clear(); + sequenceNum = new AtomicInteger(); + masterKeyId = new AtomicInteger(); + } + @Override public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( AddReservationHomeSubClusterRequest request) throws YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index 76b4e20dcdad5..e114271b53ea6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -29,6 +29,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.TimeZone; @@ -216,6 +217,10 @@ public class SQLFederationStateStore implements FederationStateStore { private static final String CALL_SP_LOAD_VERSION = "{call sp_getVersion(?, ?)}"; + private static final List TABLES = new ArrayList<>( + Arrays.asList("applicationsHomeSubCluster", "membership", "policies", "versions", + "reservationsHomeSubCluster", "masterKeys", "delegationTokens", "sequenceTable")); + private Calendar utcCalendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); @@ -1122,6 +1127,11 @@ public void storeVersion() throws Exception { storeVersion(fedVersion, versionComment); } + @Override + public void deleteStateStore() throws Exception { + truncateTable(); + } + /** * Store the Federation Version in the database. * @@ -2077,6 +2087,32 @@ private int querySequenceTable(String sequenceName, boolean isUpdate){ } } + /** + * We will truncate the tables, iterate through each table individually, + * and then clean the tables. + */ + private void truncateTable() { + Connection connection = null; + try { + connection = getConnection(false); + FederationQueryRunner runner = new FederationQueryRunner(); + for (String table : TABLES) { + LOG.info("truncate table = {} start.", table); + runner.truncateTable(connection, table); + LOG.info("truncate table = {} finished.", table); + } + } catch (Exception e) { + throw new RuntimeException("Could not truncate table!", e); + } finally { + // Return to the pool the CallableStatement + try { + FederationStateStoreUtils.returnToPool(LOG, null, connection); + } catch (YarnException e) { + LOG.error("close connection error.", e); + } + } + } + @VisibleForTesting public HikariDataSource getDataSource() { return dataSource; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 8c59b2ad8cf4f..4548cf42ca998 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -226,6 +226,8 @@ public class ZookeeperFederationStateStore implements FederationStateStore { private ZKFederationStateStoreOpDurations opDurations = ZKFederationStateStoreOpDurations.getInstance(); + private Configuration configuration; + /* * Indicates different app attempt state store operations. */ @@ -251,7 +253,7 @@ private final static class AppNodeSplitInfo { public void init(Configuration conf) throws YarnException { LOG.info("Initializing ZooKeeper connection"); - + this.configuration = conf; maxAppsInStateStore = conf.getInt( YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS, YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_MAX_APPLICATIONS); @@ -273,13 +275,8 @@ public void init(Configuration conf) throws YarnException { reservationsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_RESERVATION); versionNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_VERSION); - String hierarchiesPath = getNodePath(appsZNode, ROUTER_APP_ROOT_HIERARCHIES); - routerAppRootHierarchies = new HashMap<>(); - routerAppRootHierarchies.put(0, appsZNode); - for (int splitIndex = 1; splitIndex <= HIERARCHIES_LEVEL; splitIndex++) { - routerAppRootHierarchies.put(splitIndex, - getNodePath(hierarchiesPath, Integer.toString(splitIndex))); - } + // Initialize hierarchical path + initHierarchiesPath(); appIdNodeSplitIndex = conf.getInt(YarnConfiguration.ZK_APPID_NODE_SPLIT_INDEX, YarnConfiguration.DEFAULT_ZK_APPID_NODE_SPLIT_INDEX); @@ -302,26 +299,7 @@ public void init(Configuration conf) throws YarnException { ROUTER_RM_DT_MASTER_KEY_ID_ZNODE_NAME); // Create base znode for each entity - try { - List zkAcl = ZKCuratorManager.getZKAcls(conf); - zkManager.createRootDirRecursively(membershipZNode, zkAcl); - zkManager.createRootDirRecursively(appsZNode, zkAcl); - zkManager.createRootDirRecursively( - getNodePath(appsZNode, ROUTER_APP_ROOT_HIERARCHIES)); - for (int splitIndex = 1; splitIndex <= HIERARCHIES_LEVEL; splitIndex++) { - zkManager.createRootDirRecursively( - routerAppRootHierarchies.get(splitIndex)); - } - zkManager.createRootDirRecursively(policiesZNode, zkAcl); - zkManager.createRootDirRecursively(reservationsZNode, zkAcl); - zkManager.createRootDirRecursively(routerRMDTSecretManagerRoot, zkAcl); - zkManager.createRootDirRecursively(routerRMDTMasterKeysRootPath, zkAcl); - zkManager.createRootDirRecursively(routerRMDelegationTokensRootPath, zkAcl); - zkManager.createRootDirRecursively(versionNode, zkAcl); - } catch (Exception e) { - String errMsg = "Cannot create base directories: " + e.getMessage(); - FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); - } + createBaseZNodeForEachEntity(); // Distributed sequenceNum. try { @@ -831,6 +809,60 @@ public void storeVersion() throws Exception { put(versionNode, data, isUpdate); } + private void initHierarchiesPath() { + String hierarchiesPath = getNodePath(appsZNode, ROUTER_APP_ROOT_HIERARCHIES); + routerAppRootHierarchies = new HashMap<>(); + routerAppRootHierarchies.put(0, appsZNode); + for (int splitIndex = 1; splitIndex <= HIERARCHIES_LEVEL; splitIndex++) { + routerAppRootHierarchies.put(splitIndex, + getNodePath(hierarchiesPath, Integer.toString(splitIndex))); + } + } + + private void createBaseZNodeForEachEntity() throws YarnException { + try { + List zkAcl = ZKCuratorManager.getZKAcls(configuration); + zkManager.createRootDirRecursively(membershipZNode, zkAcl); + zkManager.createRootDirRecursively(appsZNode, zkAcl); + zkManager.createRootDirRecursively( + getNodePath(appsZNode, ROUTER_APP_ROOT_HIERARCHIES)); + for (int splitIndex = 1; splitIndex <= HIERARCHIES_LEVEL; splitIndex++) { + zkManager.createRootDirRecursively( + routerAppRootHierarchies.get(splitIndex)); + } + zkManager.createRootDirRecursively(policiesZNode, zkAcl); + zkManager.createRootDirRecursively(reservationsZNode, zkAcl); + zkManager.createRootDirRecursively(routerRMDTSecretManagerRoot, zkAcl); + zkManager.createRootDirRecursively(routerRMDTMasterKeysRootPath, zkAcl); + zkManager.createRootDirRecursively(routerRMDelegationTokensRootPath, zkAcl); + zkManager.createRootDirRecursively(versionNode, zkAcl); + } catch (Exception e) { + String errMsg = "Cannot create base directories: " + e.getMessage(); + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + } + + @Override + public void deleteStateStore() throws Exception { + + // Cleaning ZNodes and their child nodes; + // after the cleaning is complete, the ZNodes will no longer exist. + zkManager.delete(appsZNode); + zkManager.delete(membershipZNode); + zkManager.delete(policiesZNode); + zkManager.delete(reservationsZNode); + zkManager.delete(routerRMDTSecretManagerRoot); + zkManager.delete(routerRMDTMasterKeysRootPath); + zkManager.delete(routerRMDelegationTokensRootPath); + zkManager.delete(versionNode); + + // Initialize hierarchical path + initHierarchiesPath(); + + // We will continue to create ZNodes to ensure that the base path exists. + createBaseZNodeForEachEntity(); + } + /** * Get the subcluster for an application. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationQueryRunner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationQueryRunner.java index 0cca531d0d9c0..99ab7b2e1f71c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationQueryRunner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationQueryRunner.java @@ -294,6 +294,39 @@ public void updateSequenceTable(Connection connection, String sequenceName, int } } + public void truncateTable(Connection connection, String tableName) + throws SQLException { + DbType dbType = DatabaseProduct.getDbType(connection); + String deleteSQL = getTruncateStatement(dbType, tableName); + boolean committed = false; + Statement statement = null; + try { + statement = connection.createStatement(); + statement.execute(deleteSQL); + connection.commit(); + committed = true; + } catch (SQLException e) { + throw new SQLException("Unable to truncateTable due to: " + e.getMessage()); + } finally { + if (!committed) { + rollbackDBConn(connection); + } + close(statement); + } + } + + private String getTruncateStatement(DbType dbType, String tableName) { + if (isMYSQL(dbType)) { + return ("DELETE FROM \"" + tableName + "\""); + } else { + return("DELETE FROM " + tableName); + } + } + + private boolean isMYSQL(DbType dbType) { + return dbType == DbType.MYSQL; + } + static void rollbackDBConn(Connection dbConn) { try { if (dbConn != null && !dbConn.isClosed()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index d4c259b51605e..aaa475d689835 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -1118,4 +1118,8 @@ public ApplicationSubmissionContext getApplicationSubmissionContext(ApplicationI public FederationCache getFederationCache() { return federationCache; } + + public void deleteStore() throws Exception { + stateStore.deleteStateStore(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java index 626f4637f568e..14107e6622b11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java @@ -91,6 +91,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * Base class for FederationMembershipStateStore implementations. @@ -120,6 +121,7 @@ public void before() throws IOException, YarnException { @After public void after() throws Exception { + testDeleteStateStore(); stateStore.close(); } @@ -1112,4 +1114,26 @@ public void testGetApplicationHomeSubClusterWithContext() throws Exception { assertEquals(subClusterId, applicationHomeSubCluster.getHomeSubCluster()); assertEquals(context, applicationHomeSubCluster.getApplicationSubmissionContext()); } + + public void testDeleteStateStore() throws Exception { + // Step1. We clean the StateStore. + FederationStateStore federationStateStore = this.getStateStore(); + federationStateStore.deleteStateStore(); + + // Step2. When we query the sub-cluster information, it should not exist. + GetSubClustersInfoRequest request = GetSubClustersInfoRequest.newInstance(true); + List subClustersActive = stateStore.getSubClusters(request).getSubClusters(); + assertNotNull(subClustersActive); + assertEquals(0, subClustersActive.size()); + + // Step3. When we query the applications' information, it should not exist. + GetApplicationsHomeSubClusterRequest getRequest = + GetApplicationsHomeSubClusterRequest.newInstance(); + GetApplicationsHomeSubClusterResponse result = + stateStore.getApplicationsHomeSubCluster(getRequest); + assertNotNull(result); + List appsHomeSubClusters = result.getAppsHomeSubClusters(); + assertNotNull(appsHomeSubClusters); + assertEquals(0, appsHomeSubClusters.size()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java index a1071ea0901fd..e5031b759e6d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java @@ -106,6 +106,7 @@ protected FederationStateStore createStateStore() { conf.set(YarnConfiguration.FEDERATION_STATESTORE_SQL_URL, DATABASE_URL + System.currentTimeMillis()); conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS, 10); + conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_SQL_MAXCONNECTIONS, 10); super.setConf(conf); sqlFederationStateStore = new HSQLDBFederationStateStore(); return sqlFederationStateStore; @@ -647,6 +648,6 @@ public void testCheckHikariDataSourceParam() throws SQLException { assertEquals(10000, connTimeOut); assertEquals("YARN-Federation-DataBasePool", poolName); assertEquals(1, minimumIdle); - assertEquals(1, maximumPoolSize); + assertEquals(10, maximumPoolSize); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index d71a7f45e031b..60586c7a6459a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -287,6 +287,11 @@ public void checkVersion() throws Exception { stateStoreClient.checkVersion(); } + @Override + public void deleteStateStore() throws Exception { + stateStoreClient.deleteStateStore(); + } + @Override public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration( GetSubClusterPolicyConfigurationRequest request) throws YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index e4defc308dd7e..bb7bfd55413fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -373,9 +373,15 @@ public static void removeApplication(Configuration conf, String applicationId) LOG.info("Application is deleted from state store"); } - private static void handFormatStateStore() { - // TODO: YARN-11548. [Federation] Router Supports Format FederationStateStore. - System.err.println("format-state-store is not yet supported."); + private static void handFormatStateStore(Configuration conf) { + try { + System.out.println("Deleting Federation state store."); + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(conf); + System.out.println("Federation state store has been cleaned."); + facade.deleteStore(); + } catch (Exception e) { + System.err.println("Delete Federation state store error, exception = " + e); + } } private static void handRemoveApplicationFromStateStore(Configuration conf, @@ -409,7 +415,7 @@ private static void executeRouterCommand(Configuration conf, String[] args) { CommandLine cliParser = new DefaultParser().parse(opts, args); if (CMD_FORMAT_STATE_STORE.equals(cmd)) { - handFormatStateStore(); + handFormatStateStore(conf); } else if (CMD_REMOVE_APPLICATION_FROM_STATE_STORE.equals(cmd)) { if (cliParser.hasOption(removeApplicationFromStateStoreOpt)) { String applicationId = cliParser.getOptionValue(removeApplicationFromStateStoreOpt); From d9a6792ca91f7e501459a2063117c1bd48a9bc42 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:07:54 +0800 Subject: [PATCH 150/155] YARN-11011. Make YARN Router throw Exception to client clearly. (#6211) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../yarn/server/router/RouterServerUtil.java | 12 +++- .../clientrm/FederationClientInterceptor.java | 40 +++++++++---- .../TestFederationClientInterceptorRetry.java | 57 +++++++++++++++++++ .../TestableFederationClientInterceptor.java | 7 +++ 4 files changed, 102 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index 096d8dac0a76c..130503065d6b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -125,8 +125,9 @@ public static void logAndThrowException(Throwable t, String errMsgFormat, Object public static void logAndThrowException(String errMsg, Throwable t) throws YarnException { if (t != null) { - LOG.error(errMsg, t); - throw new YarnException(errMsg, t); + String newErrMsg = getErrorMsg(errMsg, t); + LOG.error(newErrMsg, t); + throw new YarnException(newErrMsg, t); } else { LOG.error(errMsg); throw new YarnException(errMsg); @@ -146,6 +147,13 @@ public static void logAndThrowException(String errMsg) throws YarnException { throw new YarnException(errMsg); } + private static String getErrorMsg(String errMsg, Throwable t) { + if (t.getMessage() != null) { + return errMsg + "" + t.getMessage(); + } + return errMsg; + } + public static R createRequestInterceptorChain(Configuration conf, String pipeLineClassName, String interceptorClassName, Class clazz) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 35b3e6eeb2bd5..f9f08583ba596 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -24,13 +24,13 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.lang.reflect.Method; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Random; import java.util.TreeMap; -import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -842,13 +842,27 @@ Collection invokeConcurrent(ClientMethod request, Class clazz) // Generate parallel Callable tasks for (SubClusterId subClusterId : subClusterIds) { callables.add(() -> { - ApplicationClientProtocol protocol = getClientRMProxyForSubCluster(subClusterId); - String methodName = request.getMethodName(); - Class[] types = request.getTypes(); - Object[] params = request.getParams(); - Method method = ApplicationClientProtocol.class.getMethod(methodName, types); - Object result = method.invoke(protocol, params); - return Pair.of(subClusterId, result); + try { + ApplicationClientProtocol protocol = getClientRMProxyForSubCluster(subClusterId); + String methodName = request.getMethodName(); + Class[] types = request.getTypes(); + Object[] params = request.getParams(); + Method method = ApplicationClientProtocol.class.getMethod(methodName, types); + Object result = method.invoke(protocol, params); + return Pair.of(subClusterId, result); + } catch (Exception e) { + Throwable cause = e.getCause(); + // We use Callable. If the exception thrown here is InvocationTargetException, + // it is a wrapped exception. We need to get the real cause of the error. + if (cause != null && cause instanceof InvocationTargetException) { + cause = cause.getCause(); + } + String errMsg = (cause.getMessage() != null) ? cause.getMessage() : "UNKNOWN"; + YarnException yarnException = + new YarnException(String.format("subClusterId %s exec %s error %s.", + subClusterId, request.getMethodName(), errMsg), e); + return Pair.of(subClusterId, yarnException); + } }); } @@ -862,8 +876,11 @@ Collection invokeConcurrent(ClientMethod request, Class clazz) Pair pair = future.get(); subClusterId = pair.getKey(); Object result = pair.getValue(); + if (result instanceof YarnException) { + throw YarnException.class.cast(result); + } results.put(subClusterId, clazz.cast(result)); - } catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException | ExecutionException | YarnException e) { Throwable cause = e.getCause(); LOG.error("Cannot execute {} on {} : {}", request.getMethodName(), subClusterId.getId(), cause.getMessage()); @@ -877,9 +894,8 @@ Collection invokeConcurrent(ClientMethod request, Class clazz) // All sub-clusters return results to be considered successful, // otherwise an exception will be thrown. if (exceptions != null && !exceptions.isEmpty()) { - Set subClusterIdSets = exceptions.keySet(); - throw new YarnException("invokeConcurrent Failed, An exception occurred in subClusterIds = " + - StringUtils.join(subClusterIdSets, ",")); + throw new YarnException("invokeConcurrent Failed = " + + StringUtils.join(exceptions.values(), ",")); } // return result diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java index 25ca03353e1c7..bf7ef7d17913d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -353,4 +354,60 @@ private void checkSubmitSubCluster(ApplicationId appId, SubClusterId expectSubCl SubClusterId respSubClusterId = responseHomeSubCluster.getHomeSubCluster(); Assert.assertEquals(expectSubCluster, respSubClusterId); } + + @Test + public void testSubmitApplicationTwoBadNodeWithRealError() throws Exception { + LOG.info("Test submitApplication with two bad SubClusters."); + setupCluster(Arrays.asList(bad1, bad2)); + interceptor.setNumSubmitRetries(1); + + final ApplicationId appId = + ApplicationId.newInstance(System.currentTimeMillis(), 5); + + final SubmitApplicationRequest request = mockSubmitApplicationRequest(appId); + + LambdaTestUtils.intercept(YarnException.class, "RM is stopped", + () -> interceptor.submitApplication(request)); + } + + @Test + public void testSubmitApplicationOneBadNodeWithRealError() throws Exception { + LOG.info("Test submitApplication with one bad SubClusters."); + setupCluster(Arrays.asList(bad1)); + interceptor.setNumSubmitRetries(0); + + final ApplicationId appId = + ApplicationId.newInstance(System.currentTimeMillis(), 6); + + final SubmitApplicationRequest request = mockSubmitApplicationRequest(appId); + + LambdaTestUtils.intercept(YarnException.class, "RM is stopped", + () -> interceptor.submitApplication(request)); + } + + @Test + public void testGetClusterMetricsTwoBadNodeWithRealError() throws Exception { + LOG.info("Test getClusterMetrics with two bad SubClusters."); + setupCluster(Arrays.asList(bad1, bad2)); + GetClusterMetricsRequest request = GetClusterMetricsRequest.newInstance(); + + LambdaTestUtils.intercept(YarnException.class, + "subClusterId 1 exec getClusterMetrics error RM is stopped.", + () -> interceptor.getClusterMetrics(request)); + + LambdaTestUtils.intercept(YarnException.class, + "subClusterId 2 exec getClusterMetrics error RM is stopped.", + () -> interceptor.getClusterMetrics(request)); + } + + @Test + public void testGetClusterMetricsOneBadNodeWithRealError() throws Exception { + LOG.info("Test getClusterMetrics with one bad SubClusters."); + setupCluster(Arrays.asList(bad1)); + GetClusterMetricsRequest request = GetClusterMetricsRequest.newInstance(); + + LambdaTestUtils.intercept(YarnException.class, + "subClusterId 1 exec getClusterMetrics error RM is stopped.", + () -> interceptor.getClusterMetrics(request)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java index 8e80bf2c0adb8..9d6b87e8cc990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.api.records.Resource; @@ -126,6 +128,11 @@ public SubmitApplicationResponse submitApplication( throw new ConnectException("RM is stopped"); } + @Override + public GetClusterMetricsResponse getClusterMetrics(GetClusterMetricsRequest request) + throws YarnException { + throw new YarnException("RM is stopped"); + } } /** From 90e9aa272eb8c878196e5f3607452e6b0b6bb31d Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:38:35 +0800 Subject: [PATCH 151/155] YARN-11484. [Federation] Router Supports Yarn Client CLI Cmds. (#6132) --- .../hadoop/mapred/ResourceMgrDelegate.java | 6 + .../protocolrecords/GetQueueInfoRequest.java | 29 +++ .../hadoop/yarn/api/records/QueueInfo.java | 173 ++++++++++++++++++ .../src/main/proto/yarn_protos.proto | 11 ++ .../src/main/proto/yarn_service_protos.proto | 1 + .../hadoop/yarn/client/api/YarnClient.java | 19 +- .../yarn/client/api/impl/YarnClientImpl.java | 21 +++ .../hadoop/yarn/client/cli/QueueCLI.java | 125 ++++++++++++- .../hadoop/yarn/client/cli/YarnCLI.java | 1 + .../yarn/client/util/YarnClientUtils.java | 6 + .../hadoop/yarn/client/cli/TestYarnCLI.java | 94 ++++++++++ .../impl/pb/GetQueueInfoRequestPBImpl.java | 16 ++ .../api/records/impl/pb/QueueInfoPBImpl.java | 132 +++++++++++++ .../capacity/CSQueueInfoProvider.java | 1 + .../scheduler/fair/FSQueue.java | 33 +++- .../clientrm/FederationClientInterceptor.java | 57 +++++- .../clientrm/RouterYarnClientUtils.java | 41 +++++ .../TestFederationClientInterceptor.java | 26 ++- 18 files changed, 777 insertions(+), 15 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 81619067aa3e2..c3d078603c93b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -399,6 +399,12 @@ public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo( return client.getQueueInfo(queueName); } + @Override + public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo( + String queueName, String subClusterId) throws YarnException, IOException { + return client.getQueueInfo(queueName, subClusterId); + } + @Override public List getAllQueues() throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java index 0e33e21d211c4..e28fad373a76f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java @@ -46,6 +46,19 @@ public abstract class GetQueueInfoRequest { return request; } + @Public + @Stable + public static GetQueueInfoRequest newInstance(String queueName, boolean includeApplications, + boolean includeChildQueues, boolean recursive, String subClusterId) { + GetQueueInfoRequest request = Records.newRecord(GetQueueInfoRequest.class); + request.setQueueName(queueName); + request.setIncludeApplications(includeApplications); + request.setIncludeChildQueues(includeChildQueues); + request.setRecursive(recursive); + request.setSubClusterId(subClusterId); + return request; + } + /** * Get the queue name for which to get queue information. * @return queue name for which to get queue information @@ -114,5 +127,21 @@ public abstract class GetQueueInfoRequest { @Public @Stable public abstract void setRecursive(boolean recursive); + + /** + * Get SubClusterId. + * @return SubClusterId. + */ + @Public + @Stable + public abstract String getSubClusterId(); + + /** + * Set SubClusterId. + * @param subClusterId SubClusterId. + */ + @Public + @Stable + public abstract void setSubClusterId(String subClusterId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java index c4d78f0c7ae22..749976cfd1653 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java @@ -340,4 +340,177 @@ public abstract void setQueueConfigurations( @Unstable public abstract void setIntraQueuePreemptionDisabled( boolean intraQueuePreemptionDisabled); + + /** + * Get Scheduler type. + * + * @return SchedulerType. + */ + @Public + @Stable + public abstract String getSchedulerType(); + + /** + * Set Scheduler type. + * @param schedulerType scheduler Type. + */ + @Private + @Unstable + public abstract void setSchedulerType(String schedulerType); + + /** + * Get the minimum resource VCore. + * @return minimum resource VCore. + */ + @Public + @Stable + public abstract int getMinResourceVCore(); + + /** + * Set the minimum resource VCore. + * @param vCore minimum resource VCore. + */ + @Private + @Unstable + public abstract void setMinResourceVCore(int vCore); + + /** + * Get the minimum resource Memory. + * @return minimum resource Memory. + */ + @Public + @Stable + public abstract long getMinResourceMemory(); + + /** + * Set the minimum resource Memory. + * @param memory minimum resource Memory. + */ + @Private + @Unstable + public abstract void setMinResourceMemory(long memory); + + /** + * Get the maximum resource VCore. + * @return maximum resource VCore. + */ + @Public + @Stable + public abstract int getMaxResourceVCore(); + + /** + * Set the maximum resource Memory. + * @param vCore maximum resource VCore. + */ + @Private + @Unstable + public abstract void setMaxResourceVCore(int vCore); + + /** + * Get the maximum resource Memory. + * @return maximum resource Memory. + */ + @Public + @Stable + public abstract long getMaxResourceMemory(); + + /** + * Set the maximum resource Memory. + * @param memory maximum resource Memory. + */ + @Private + @Unstable + public abstract void setMaxResourceMemory(long memory); + + /** + * Get the reserved resource VCore. + * @return reserved resource VCore. + */ + @Public + @Stable + public abstract int getReservedResourceVCore(); + + /** + * Set the reserved resource VCore. + * @param vCore reserved resource VCore. + */ + @Private + @Unstable + public abstract void setReservedResourceVCore(int vCore); + + /** + * Get the reserved resource Memory. + * @return reserved resource Memory. + */ + @Public + @Stable + public abstract long getReservedResourceMemory(); + + /** + * Set the reserved resource Memory. + * @param memory reserved resource Memory. + */ + @Private + @Unstable + public abstract void setReservedResourceMemory(long memory); + + /** + * Get the SteadyFairShare VCore. + * @return SteadyFairShare VCore. + */ + @Public + @Stable + public abstract int getSteadyFairShareVCore(); + + /** + * Set the SteadyFairShare VCore. + * @param vCore SteadyFairShare VCore. + */ + @Private + @Unstable + public abstract void setSteadyFairShareVCore(int vCore); + + /** + * Get the SteadyFairShare Memory. + * @return SteadyFairShare Memory. + */ + @Public + @Stable + public abstract long getSteadyFairShareMemory(); + + /** + * Set the SteadyFairShare Memory. + * @param memory SteadyFairShare Memory. + */ + @Private + @Unstable + public abstract void setSteadyFairShareMemory(long memory); + + /** + * Get the SubClusterId. + * @return the SubClusterId. + */ + @Public + @Stable + public abstract String getSubClusterId(); + + /** + * Set the SubClusterId. + * @param subClusterId the SubClusterId. + */ + @Private + @Unstable + public abstract void setSubClusterId(String subClusterId); + + /** + * Get the MaxRunningApp. + * @return The number of MaxRunningApp. + */ + @Public + @Stable + public abstract int getMaxRunningApp(); + + @Private + @Unstable + public abstract void setMaxRunningApp(int maxRunningApp); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 71c102f4f8577..3a23aa0412547 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -636,6 +636,17 @@ message QueueInfoProto { optional float weight = 14; optional string queuePath = 15; optional int32 maxParallelApps = 16; + optional string schedulerType = 17; + optional int32 minResourceVCore = 18; + optional int64 minResourceMemory = 19; + optional int32 maxResourceVCore = 20; + optional int64 maxResourceMemory = 21; + optional int32 reservedResourceVCore = 22; + optional int64 reservedResourceMemory = 23; + optional int32 steadyFairShareVCore = 24; + optional int64 steadyFairShareMemory = 25; + optional string subClusterId = 26; + optional int32 maxRunningApp = 27; } message QueueConfigurationsProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index a86cbbae1169c..8838b86f3fb92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -232,6 +232,7 @@ message GetQueueInfoRequestProto { optional bool includeApplications = 2; optional bool includeChildQueues = 3; optional bool recursive = 4; + optional string subClusterId = 5; } message GetQueueInfoResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index b59831ebe8c3f..d036987ae8b8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -464,11 +464,28 @@ public abstract Token getRMDelegationToken(Text renewer) * @throws YarnException * in case of errors or if YARN rejects the request due to * access-control restrictions. - * @throws IOException + * @throws IOException I/O exception has occurred. */ public abstract QueueInfo getQueueInfo(String queueName) throws YarnException, IOException; + /** + *

    + * Get information ({@link QueueInfo}) about a given queue. + *

    + * + * @param queueName + * Name of the queue whose information is needed. + * @param subClusterId sub-cluster Id. + * @return queue information. + * @throws YarnException + * in case of errors or if YARN rejects the request due to + * access-control restrictions. + * @throws IOException I/O exception has occurred. + */ + public abstract QueueInfo getQueueInfo(String queueName, String subClusterId) + throws YarnException, IOException; + /** *

    * Get information ({@link QueueInfo}) about all queues, recursively if there diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 19d03a7da7341..ca30ff865bf08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -735,6 +735,18 @@ public Token getRMDelegationToken(Text renewer) return request; } + private GetQueueInfoRequest getQueueInfoRequest( + String queueName, String subClusterId, boolean includeApplications, + boolean includeChildQueues, boolean recursive) { + GetQueueInfoRequest request = Records.newRecord(GetQueueInfoRequest.class); + request.setQueueName(queueName); + request.setSubClusterId(subClusterId); + request.setIncludeApplications(includeApplications); + request.setIncludeChildQueues(includeChildQueues); + request.setRecursive(recursive); + return request; + } + @Override public QueueInfo getQueueInfo(String queueName) throws YarnException, IOException { @@ -744,6 +756,15 @@ public QueueInfo getQueueInfo(String queueName) throws YarnException, return rmClient.getQueueInfo(request).getQueueInfo(); } + @Override + public QueueInfo getQueueInfo(String queueName, + String subClusterId) throws YarnException, IOException { + GetQueueInfoRequest request = + getQueueInfoRequest(queueName, subClusterId, true, false, false); + Records.newRecord(GetQueueInfoRequest.class); + return rmClient.getQueueInfo(request).getQueueInfo(); + } + @Override public List getQueueAclsInfo() throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java index 2df7aeb8d0f3f..db2b2cdf53f8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java @@ -32,6 +32,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.MissingArgumentException; import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.ToolRunner; @@ -42,6 +43,8 @@ import org.apache.hadoop.classification.VisibleForTesting; +import static org.apache.hadoop.yarn.client.util.YarnClientUtils.isYarnFederationEnabled; + @Private @Unstable public class QueueCLI extends YarnCLI { @@ -70,7 +73,8 @@ public int run(String[] args) throws Exception { "All child queues are displayed according to the parent queue. " + "If the value is all, all queues are displayed."); opts.getOption(LIST_CMD).setArgName("Parent Queue Name"); - + opts.addOption(OPTION_SUBCLUSTERID, true, "We support setting subClusterId in " + + "YARN Federation mode to specify specific subClusters."); CommandLine cliParser = null; try { @@ -82,11 +86,19 @@ public int run(String[] args) throws Exception { } createAndStartYarnClient(); if (cliParser.hasOption(STATUS_CMD)) { - if (args.length != 2) { + // Our possible options are -status root.a, + // -subcluster sc-1, we will have up to 4 args + if (args.length > 4) { printUsage(opts); return -1; } - return listQueue(cliParser.getOptionValue(STATUS_CMD)); + String queue = cliParser.getOptionValue(STATUS_CMD); + String subClusterId = cliParser.getOptionValue(OPTION_SUBCLUSTERID); + if (isYarnFederationEnabled(getConf()) && StringUtils.isNotBlank(subClusterId)) { + return listQueue(queue, subClusterId); + } else { + return listQueue(queue); + } } else if (cliParser.hasOption(HELP_CMD)) { printUsage(opts); return 0; @@ -116,9 +128,10 @@ void printUsage(Options opts) { /** * Lists the Queue Information matching the given queue name. * - * @param queueName - * @throws YarnException - * @throws IOException + * @param queueName Queue name to be queried. + * @throws YarnException YarnException indicates exceptions from yarn servers. + * @throws IOException I/O exception has occurred. + * @return 0, the command execution is successful; -1, the command execution fails. */ private int listQueue(String queueName) throws YarnException, IOException { int rc; @@ -127,6 +140,9 @@ private int listQueue(String queueName) throws YarnException, IOException { QueueInfo queueInfo = client.getQueueInfo(queueName); if (queueInfo != null) { + if (isYarnFederationEnabled(getConf())) { + writer.println("Using YARN Federation mode."); + } writer.println("Queue Information : "); printQueueInfo(writer, queueInfo); rc = 0; @@ -139,6 +155,41 @@ private int listQueue(String queueName) throws YarnException, IOException { return rc; } + /** + * Lists the Queue Information matching the given queue name. + * + * @param queueName Queue name to be queried. + * @param subClusterId Subcluster id. + * @throws YarnException YarnException indicates exceptions from yarn servers. + * @throws IOException I/O exception has occurred. + * @return 0, the command execution is successful; -1, the command execution fails. + */ + private int listQueue(String queueName, String subClusterId) + throws YarnException, IOException { + int rc; + PrintWriter writer = new PrintWriter( + new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); + QueueInfo queueInfo = client.getQueueInfo(queueName, subClusterId); + if (queueInfo != null) { + if (isYarnFederationEnabled(getConf())) { + writer.println("Using YARN Federation mode."); + } + if (StringUtils.isNotBlank(subClusterId)) { + writer.println("SubClusterId : " + subClusterId + ", Queue Information : "); + } else { + writer.println("Queue Information : "); + } + printQueueInfo(writer, queueInfo); + rc = 0; + } else { + writer.println("Cannot get queue from RM by queueName = " + queueName + + ", subClusterId = " + subClusterId + " please check."); + rc = -1; + } + writer.flush(); + return rc; + } + /** * List information about all child queues based on the parent queue. * @param parentQueueName The name of the payment queue. @@ -174,6 +225,66 @@ private int listChildQueues(String parentQueueName) throws IOException, YarnExce } private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) { + String schedulerType = queueInfo.getSchedulerType(); + + if (StringUtils.equals("FairScheduler", schedulerType)) { + printFairSchedulerQueue(writer, queueInfo); + } else { + printQueue(writer, queueInfo); + } + } + + /** + * Print Queue information of FairScheduler. + * + * @param writer PrintWriter. + * @param queueInfo Queue Information. + */ + private void printFairSchedulerQueue(PrintWriter writer, QueueInfo queueInfo) { + String generateQueueInfoMessage = generateQueueInfoMessage(queueInfo); + writer.print(generateQueueInfoMessage); + } + + private String generateQueueInfoMessage(QueueInfo queueInfo) { + StringBuilder stringBuilder = new StringBuilder(); + if (queueInfo.getSchedulerType() != null) { + stringBuilder.append("Scheduler Name : ").append(queueInfo.getSchedulerType()).append("\n"); + } + stringBuilder.append("Queue Name : ").append(queueInfo.getQueueName()).append("\n"); + DecimalFormat df = new DecimalFormat("0.00"); + stringBuilder.append("\tWeight : ").append(df.format(queueInfo.getWeight())).append("\n"); + stringBuilder.append("\tState : ").append(queueInfo.getQueueState()).append("\n"); + stringBuilder.append("\tMinResource : ").append("").append("\n"); + stringBuilder.append("\tMaxResource : ").append("").append("\n"); + stringBuilder.append("\tReservedResource : ").append("").append("\n"); + stringBuilder.append("\tSteadyFairShare : ").append("").append("\n"); + Boolean queuePreemption = queueInfo.getPreemptionDisabled(); + if (queuePreemption != null) { + stringBuilder.append("\tQueue Preemption : ") + .append(queuePreemption ? "enabled" : "disabled").append("\n"); + } + return stringBuilder.toString(); + } + + /** + * Print Queue information. + * + * @param writer PrintWriter. + * @param queueInfo Queue Information. + */ + private void printQueue(PrintWriter writer, QueueInfo queueInfo) { + if (queueInfo.getSchedulerType() != null) { + writer.print("Scheduler Name : "); + writer.println(queueInfo.getSchedulerType()); + } writer.print("Queue Name : "); writer.println(queueInfo.getQueueName()); writer.print("Queue Path : "); @@ -208,7 +319,7 @@ private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) { } labelList.append(nodeLabel); } - writer.println(labelList.toString()); + writer.println(labelList); Boolean preemptStatus = queueInfo.getPreemptionDisabled(); if (preemptStatus != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java index c1e02d5fd1ea0..04a0c35f8bd5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java @@ -37,6 +37,7 @@ public abstract class YarnCLI extends Configured implements Tool { public static final String MOVE_TO_QUEUE_CMD = "movetoqueue"; public static final String HELP_CMD = "help"; public static final String SIGNAL_CMD = "signal"; + public static final String OPTION_SUBCLUSTERID = "subClusterId"; protected PrintStream sysout; protected PrintStream syserr; protected YarnClient client; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java index 049dbd7962c73..4aab5146b1bf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java @@ -250,4 +250,10 @@ public String run() throws Exception { }); return challenge; } + + public static boolean isYarnFederationEnabled(Configuration conf) { + boolean isEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + return isEnabled; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 0558edae50340..2bcf84c086bd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -88,6 +88,8 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -1969,6 +1971,98 @@ public void testGetQueueInfoWithEmptyNodeLabel() throws Exception { String queueInfoStr = baos.toString("UTF-8"); Assert.assertEquals(queueInfoStr, sysOutStream.toString()); } + + @Test + public void testGetQueueInfoWithFairScheduler() throws Exception { + // In this test case, we will simulate the queue information of fairScheduler + // and check the results of the queue information. + QueueCLI cli = createAndGetQueueCLI(); + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); + queueInfo.setQueueName("queueA"); + queueInfo.setSchedulerType("FairScheduler"); + queueInfo.setQueueState(QueueState.RUNNING); + queueInfo.setCapacity(0.3f); + queueInfo.setCurrentCapacity(0.1f); + queueInfo.setWeight(0.3f); + queueInfo.setMinResourceVCore(1); + queueInfo.setMinResourceMemory(1024); + queueInfo.setMaxResourceVCore(10); + queueInfo.setMaxResourceMemory(8192); + queueInfo.setReservedResourceVCore(0); + queueInfo.setReservedResourceMemory(0); + queueInfo.setSteadyFairShareVCore(10); + queueInfo.setSteadyFairShareMemory(8192); + queueInfo.setMaxRunningApp(10); + queueInfo.setPreemptionDisabled(true); + when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); + int result = cli.run(new String[]{"-status", "queueA"}); + assertEquals(0, result); + verify(client).getQueueInfo("queueA"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Queue Information : "); + pw.println("Scheduler Name : FairScheduler"); + pw.println("Queue Name : queueA"); + pw.println("\tWeight : 0.30"); + pw.println("\tState : RUNNING"); + pw.println("\tMinResource : "); + pw.println("\tMaxResource : "); + pw.println("\tReservedResource : "); + pw.println("\tSteadyFairShare : "); + pw.println("\tQueue Preemption : enabled"); + pw.close(); + String queueInfoStr = baos.toString("UTF-8"); + Assert.assertEquals(queueInfoStr, sysOutStream.toString()); + } + + @Test + public void testGetQueueInfoWithFairSchedulerAndSubClusterId() throws Exception { + // In this test case, + // we simulated printing FairScheduler queue information in YARN Federation mode. + QueueCLI cli = createAndGetQueueCLI(); + Configuration config = new Configuration(); + config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + cli.setConf(config); + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); + queueInfo.setQueueName("queueA"); + queueInfo.setSchedulerType("FairScheduler"); + queueInfo.setQueueState(QueueState.RUNNING); + queueInfo.setCapacity(0.3f); + queueInfo.setCurrentCapacity(0.1f); + queueInfo.setWeight(0.3f); + queueInfo.setMinResourceVCore(1); + queueInfo.setMinResourceMemory(1024); + queueInfo.setMaxResourceVCore(10); + queueInfo.setMaxResourceMemory(8192); + queueInfo.setReservedResourceVCore(0); + queueInfo.setReservedResourceMemory(0); + queueInfo.setSteadyFairShareVCore(10); + queueInfo.setSteadyFairShareMemory(8192); + queueInfo.setMaxRunningApp(10); + queueInfo.setPreemptionDisabled(true); + when(client.getQueueInfo(any(String.class), any(String.class))).thenReturn(queueInfo); + int result = cli.run(new String[]{"-status", "queueA", "-subClusterId", "SC-1"}); + assertEquals(0, result); + verify(client).getQueueInfo("queueA", "SC-1"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Using YARN Federation mode."); + pw.println("SubClusterId : SC-1, Queue Information : "); + pw.println("Scheduler Name : FairScheduler"); + pw.println("Queue Name : " + "queueA"); + pw.println("\tWeight : " + "0.30"); + pw.println("\tState : " + "RUNNING"); + pw.println("\tMinResource : " + ""); + pw.println("\tMaxResource : " + ""); + pw.println("\tReservedResource : " + ""); + pw.println("\tSteadyFairShare : " + ""); + pw.println("\tQueue Preemption : " + "enabled"); + pw.close(); + String queueInfoStr = baos.toString("UTF-8"); + Assert.assertEquals(queueInfoStr, sysOutStream.toString()); + } @Test public void testGetQueueInfoWithNonExistedQueue() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java index 978ecd368cbcf..a6945245f89ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java @@ -96,6 +96,22 @@ public void setRecursive(boolean recursive) { builder.setRecursive(recursive); } + @Override + public String getSubClusterId() { + GetQueueInfoRequestProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasSubClusterId()) ? p.getSubClusterId() : ""; + } + + @Override + public void setSubClusterId(String subClusterId) { + maybeInitBuilder(); + if (subClusterId == null) { + builder.clearSubClusterId(); + return; + } + builder.setSubClusterId(subClusterId); + } + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = GetQueueInfoRequestProto.newBuilder(proto); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java index 5643ff30d4c78..43251cee18451 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java @@ -553,4 +553,136 @@ public void setIntraQueuePreemptionDisabled( maybeInitBuilder(); builder.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled); } + + @Override + public String getSchedulerType() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasSchedulerType()) ? p.getSchedulerType() : null; + } + + @Override + public void setSchedulerType(String schedulerType) { + maybeInitBuilder(); + builder.setSchedulerType(schedulerType); + } + + @Override + public int getMinResourceVCore() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasMinResourceVCore()) ? p.getMinResourceVCore() : 0; + } + + @Override + public void setMinResourceVCore(int vCore) { + maybeInitBuilder(); + builder.setMinResourceVCore(vCore); + } + + @Override + public long getMinResourceMemory() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasMaxResourceVCore()) ? p.getMaxResourceVCore() : 0; + } + + @Override + public void setMinResourceMemory(long memory) { + maybeInitBuilder(); + builder.setMinResourceMemory(memory); + } + + @Override + public int getMaxResourceVCore() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasMaxResourceVCore()) ? p.getMaxResourceVCore() : 0; + } + + @Override + public void setMaxResourceVCore(int vCore) { + maybeInitBuilder(); + builder.setMinResourceVCore(vCore); + } + + @Override + public long getMaxResourceMemory() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasMaxResourceMemory()) ? p.getMaxResourceMemory() : 0; + } + + @Override + public void setMaxResourceMemory(long memory) { + maybeInitBuilder(); + builder.setMaxResourceMemory(memory); + } + + @Override + public int getReservedResourceVCore() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasReservedResourceVCore()) ? p.getReservedResourceVCore() : 0; + } + + @Override + public void setReservedResourceVCore(int vCore) { + maybeInitBuilder(); + builder.setReservedResourceVCore(vCore); + } + + @Override + public long getReservedResourceMemory() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasReservedResourceMemory()) ? p.getReservedResourceMemory() : 0; + } + + @Override + public void setReservedResourceMemory(long memory) { + maybeInitBuilder(); + builder.setReservedResourceMemory(memory); + } + + @Override + public int getSteadyFairShareVCore() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasSteadyFairShareVCore()) ? p.getSteadyFairShareVCore() : 0; + } + + @Override + public void setSteadyFairShareVCore(int vCore) { + maybeInitBuilder(); + builder.setSteadyFairShareVCore(vCore); + } + + @Override + public long getSteadyFairShareMemory() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasSteadyFairShareMemory()) ? p.getSteadyFairShareMemory() : 0; + } + + @Override + public void setSteadyFairShareMemory(long memory) { + maybeInitBuilder(); + builder.setSteadyFairShareMemory(memory); + } + + @Override + public String getSubClusterId() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasSubClusterId()) ? p.getSubClusterId() : null; + } + + @Override + public void setSubClusterId(String subClusterId) { + maybeInitBuilder(); + builder.setSubClusterId(subClusterId); + } + + @Override + public int getMaxRunningApp() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasMaxRunningApp()) ? p.getMaxRunningApp() : 0; + } + + @Override + public void setMaxRunningApp(int maxRunningApp) { + maybeInitBuilder(); + builder.setMaxRunningApp(maxRunningApp); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java index 8daca2bc26be6..fb2c8c4abe1e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java @@ -39,6 +39,7 @@ private CSQueueInfoProvider() { public static QueueInfo getQueueInfo(AbstractCSQueue csQueue) { QueueInfo queueInfo = RECORD_FACTORY.newRecordInstance(QueueInfo.class); + queueInfo.setSchedulerType("CapacityScheduler"); queueInfo.setQueueName(csQueue.getQueuePathObject().getLeafName()); queueInfo.setQueuePath(csQueue.getQueuePathObject().getFullPath()); queueInfo.setAccessibleNodeLabels(csQueue.getAccessibleNodeLabels()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 3e99ebae9f606..41e33848eaecf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -234,6 +234,7 @@ public Priority getPriority() { @Override public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) { QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); + queueInfo.setSchedulerType("FairScheduler"); queueInfo.setQueueName(getQueueName()); if (scheduler.getClusterResource().getMemorySize() == 0) { @@ -250,7 +251,37 @@ public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) { getFairShare().getMemorySize()); } - ArrayList childQueueInfos = new ArrayList(); + // set Weight + queueInfo.setWeight(getWeight()); + + // set MinShareResource + Resource minShareResource = getMinShare(); + queueInfo.setMinResourceVCore(minShareResource.getVirtualCores()); + queueInfo.setMinResourceMemory(minShareResource.getMemorySize()); + + // set MaxShareResource + Resource maxShareResource = + Resources.componentwiseMin(getMaxShare(), scheduler.getClusterResource()); + queueInfo.setMaxResourceVCore(maxShareResource.getVirtualCores()); + queueInfo.setMaxResourceMemory(maxShareResource.getMemorySize()); + + // set ReservedResource + Resource newReservedResource = getReservedResource(); + queueInfo.setReservedResourceVCore(newReservedResource.getVirtualCores()); + queueInfo.setReservedResourceMemory(newReservedResource.getMemorySize()); + + // set SteadyFairShare + Resource newSteadyFairShare = getSteadyFairShare(); + queueInfo.setSteadyFairShareVCore(newSteadyFairShare.getVirtualCores()); + queueInfo.setSteadyFairShareMemory(newSteadyFairShare.getMemorySize()); + + // set MaxRunningApp + queueInfo.setMaxRunningApp(getMaxRunningApps()); + + // set Preemption + queueInfo.setPreemptionDisabled(isPreemptable()); + + ArrayList childQueueInfos = new ArrayList<>(); if (includeChildQueues) { Collection childQueues = getChildQueues(); for (FSQueue child : childQueues) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index f9f08583ba596..71e265be1b9c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -27,6 +27,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Random; @@ -902,6 +903,40 @@ Collection invokeConcurrent(ClientMethod request, Class clazz) return results.values(); } + Collection invoke(ClientMethod request, Class clazz, String subClusterId) + throws YarnException { + + // Get Active SubClusters + Map subClusterInfoMap = federationFacade.getSubClusters(true); + + // According to subCluster of string type, convert to SubClusterId type + SubClusterId subClusterIdKey = SubClusterId.newInstance(subClusterId); + + // If the provided subCluster is not Active or does not exist, + // an exception will be returned directly. + if (!subClusterInfoMap.containsKey(subClusterIdKey)) { + throw new YarnException("subClusterId = " + subClusterId + " is not an active subCluster."); + } + + try { + ApplicationClientProtocol protocol = getClientRMProxyForSubCluster(subClusterIdKey); + String methodName = request.getMethodName(); + Class[] types = request.getTypes(); + Object[] params = request.getParams(); + Method method = ApplicationClientProtocol.class.getMethod(methodName, types); + Object result = method.invoke(protocol, params); + if (result != null) { + return Collections.singletonList(clazz.cast(result)); + } + } catch (Exception e) { + throw new YarnException("invoke Failed, An exception occurred in subClusterId = " + + subClusterId, e); + } + + throw new YarnException("invoke Failed, An exception occurred in subClusterId = " + + subClusterId); + } + @Override public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) throws YarnException, IOException { @@ -933,6 +968,21 @@ public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) throw new YarnException("Unable to get cluster nodes."); } + /** + *

    The interface used by clients to get information about queues + * from the ResourceManager.

    + * + *

    The client, via {@link GetQueueInfoRequest}, can ask for details such + * as used/total resources, child queues, running applications etc.

    + * + *

    In secure mode,the ResourceManager verifies access before + * providing the information.

    + * + * @param request request to get queue information + * @return queue information + * @throws YarnException exceptions from yarn servers. + * @throws IOException io error occur. + */ @Override public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) throws YarnException, IOException { @@ -943,13 +993,18 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) TARGET_CLIENT_RM_SERVICE, msg); RouterServerUtil.logAndThrowException(msg, null); } + String rSubCluster = request.getSubClusterId(); long startTime = clock.getTime(); ClientMethod remoteMethod = new ClientMethod("getQueueInfo", new Class[]{GetQueueInfoRequest.class}, new Object[]{request}); Collection queues = null; try { - queues = invokeConcurrent(remoteMethod, GetQueueInfoResponse.class); + if (StringUtils.isNotBlank(rSubCluster)) { + queues = invoke(remoteMethod, GetQueueInfoResponse.class, rSubCluster); + } else { + queues = invokeConcurrent(remoteMethod, GetQueueInfoResponse.class); + } } catch (Exception ex) { routerMetrics.incrGetQueueInfoFailedRetrieved(); String msg = "Unable to get queue [" + request.getQueueName() + "] to exception."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java index 481005b478d60..feba3f0cad106 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java @@ -423,6 +423,47 @@ public static GetQueueInfoResponse mergeQueues( if (queueInfo.getAccessibleNodeLabels() != null) { accessibleNodeLabels.addAll(queueInfo.getAccessibleNodeLabels()); } + + // set min resourceVCore + queueInfo.setMinResourceVCore(queueInfo.getMinResourceVCore() + + response.getQueueInfo().getMinResourceVCore()); + + // set min resourceMemory + queueInfo.setMinResourceMemory(queueInfo.getMinResourceMemory() + + response.getQueueInfo().getMinResourceMemory()); + + // set max resourceVCore + queueInfo.setMinResourceVCore(queueInfo.getMaxResourceVCore() + + response.getQueueInfo().getMaxResourceVCore()); + + // set max resourceMemory + queueInfo.setMinResourceMemory(queueInfo.getMaxResourceMemory() + + response.getQueueInfo().getMaxResourceMemory()); + + // set reserved resourceVCore + queueInfo.setReservedResourceVCore(queueInfo.getReservedResourceVCore() + + response.getQueueInfo().getMaxResourceVCore()); + + // set reserved resourceMemory + queueInfo.setReservedResourceMemory(queueInfo.getReservedResourceMemory() + + response.getQueueInfo().getMaxResourceMemory()); + + // set maxRunningApp + queueInfo.setMaxRunningApp(queueInfo.getMaxRunningApp() + + response.getQueueInfo().getMaxRunningApp()); + + // set steadyFairShareVCore + queueInfo.setSteadyFairShareVCore(queueInfo.getSteadyFairShareVCore() + + response.getQueueInfo().getSteadyFairShareVCore()); + + // set steadyFairShareMemory + queueInfo.setSteadyFairShareMemory(queueInfo.getSteadyFairShareMemory() + + response.getQueueInfo().getSteadyFairShareMemory()); + + // set Weight + queueInfo.setWeight(queueInfo.getWeight() + + response.getQueueInfo().getWeight()); + if (response.getQueueInfo() != null) { accessibleNodeLabels.addAll(response.getQueueInfo().getAccessibleNodeLabels()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java index 82eb1da9ae489..3c2c7b4d3b307 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java @@ -1167,11 +1167,27 @@ public void testGetQueueInfo() throws Exception { QueueInfo queueInfo = response.getQueueInfo(); Assert.assertNotNull(queueInfo); - Assert.assertEquals(queueInfo.getQueueName(), "root"); - Assert.assertEquals(queueInfo.getCapacity(), 4.0, 0); - Assert.assertEquals(queueInfo.getCurrentCapacity(), 0.0, 0); - Assert.assertEquals(queueInfo.getChildQueues().size(), 12, 0); - Assert.assertEquals(queueInfo.getAccessibleNodeLabels().size(), 1); + Assert.assertEquals("root", queueInfo.getQueueName()); + Assert.assertEquals(4.0, queueInfo.getCapacity(), 0); + Assert.assertEquals(0.0, queueInfo.getCurrentCapacity(), 0); + Assert.assertEquals(12, queueInfo.getChildQueues().size(), 0); + Assert.assertEquals(1, queueInfo.getAccessibleNodeLabels().size()); + } + + @Test + public void testSubClusterGetQueueInfo() throws IOException, YarnException { + // We have set up a unit test where we access queue information for subcluster1. + GetQueueInfoResponse response = interceptor.getQueueInfo( + GetQueueInfoRequest.newInstance("root", true, true, true, "1")); + Assert.assertNotNull(response); + + QueueInfo queueInfo = response.getQueueInfo(); + Assert.assertNotNull(queueInfo); + Assert.assertEquals("root", queueInfo.getQueueName()); + Assert.assertEquals(1.0, queueInfo.getCapacity(), 0); + Assert.assertEquals(0.0, queueInfo.getCurrentCapacity(), 0); + Assert.assertEquals(3, queueInfo.getChildQueues().size(), 0); + Assert.assertEquals(1, queueInfo.getAccessibleNodeLabels().size()); } @Test From f58945d7d172fe70984a259eeb8e26da05624c91 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 8 Nov 2023 06:25:21 -0800 Subject: [PATCH 152/155] HDFS-16791. Add getEnclosingRoot() API to filesystem interface and implementations (#6198) The enclosing root path is a common ancestor that should be used for temp and staging dirs as well as within encryption zones and other restricted directories. Contributed by Tom McCormick --- .../apache/hadoop/fs/AbstractFileSystem.java | 18 +++ .../java/org/apache/hadoop/fs/FileSystem.java | 18 +++ .../apache/hadoop/fs/FilterFileSystem.java | 5 + .../java/org/apache/hadoop/fs/FilterFs.java | 5 + .../hadoop/fs/viewfs/ViewFileSystem.java | 37 +++++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 17 ++ .../site/markdown/filesystem/filesystem.md | 35 +++- .../hadoop/fs/TestGetEnclosingRoot.java | 94 +++++++++++ .../apache/hadoop/fs/TestHarFileSystem.java | 2 + .../AbstractContractGetEnclosingRoot.java | 103 ++++++++++++ .../TestLocalFSContractGetEnclosingRoot.java | 32 ++++ .../TestRawlocalContractGetEnclosingRoot.java | 32 ++++ .../org/apache/hadoop/hdfs/DFSClient.java | 10 ++ .../hadoop/hdfs/DFSOpsCountStatistics.java | 1 + .../hadoop/hdfs/DistributedFileSystem.java | 27 ++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 8 + .../ClientNamenodeProtocolTranslatorPB.java | 18 +++ .../main/proto/ClientNamenodeProtocol.proto | 10 ++ .../hadoop/hdfs/protocol/TestReadOnly.java | 1 + .../router/RouterClientProtocol.java | 33 ++++ .../federation/router/RouterRpcServer.java | 5 + .../router/TestRouterMountTable.java | 25 +++ ...amenodeProtocolServerSideTranslatorPB.java | 22 ++- .../hdfs/server/namenode/FSNamesystem.java | 15 ++ .../server/namenode/NameNodeRpcServer.java | 7 + .../fs/viewfs/TestViewFileSystemHdfs.java | 149 ++++++++++++++---- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 27 ++++ .../hdfs/TestDistributedFileSystem.java | 5 + .../apache/hadoop/hdfs/TestEnclosingRoot.java | 149 ++++++++++++++++++ 29 files changed, 878 insertions(+), 32 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetEnclosingRoot.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetEnclosingRoot.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractGetEnclosingRoot.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractGetEnclosingRoot.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEnclosingRoot.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index a4737c548c8fa..63b5bc7d94ac3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -1638,6 +1638,24 @@ public MultipartUploaderBuilder createMultipartUploader(Path basePath) return null; } + /** + * Return path of the enclosing root for a given path + * The enclosing root path is a common ancestor that should be used for temp and staging dirs + * as well as within encryption zones and other restricted directories. + * + * Call makeQualified on the param path to ensure its part of the correct filesystem + * + * @param path file path to find the enclosing root path for + * @return a path to the enclosing root + * @throws IOException early checks like failure to resolve path cause IO failures + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Path getEnclosingRoot(Path path) throws IOException { + makeQualified(path); + return makeQualified(new Path("/")); + } + /** * Helper method that throws an {@link UnsupportedOperationException} for the * current {@link FileSystem} method being called. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 52425211dc688..0213772ab6a5c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -4944,6 +4944,24 @@ public CompletableFuture build() throws IOException { } + /** + * Return path of the enclosing root for a given path. + * The enclosing root path is a common ancestor that should be used for temp and staging dirs + * as well as within encryption zones and other restricted directories. + * + * Call makeQualified on the param path to ensure its part of the correct filesystem. + * + * @param path file path to find the enclosing root path for + * @return a path to the enclosing root + * @throws IOException early checks like failure to resolve path cause IO failures + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Path getEnclosingRoot(Path path) throws IOException { + this.makeQualified(path); + return this.makeQualified(new Path("/")); + } + /** * Create a multipart uploader. * @param basePath file path under which all files are uploaded diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index cdbe51e330701..a85cf2ff5a17e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -732,6 +732,11 @@ protected CompletableFuture openFileWithOptions( return fs.openFileWithOptions(pathHandle, parameters); } + @Override + public Path getEnclosingRoot(Path path) throws IOException { + return fs.getEnclosingRoot(path); + } + @Override public boolean hasPathCapability(final Path path, final String capability) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index 7d979b37b4a50..df010e3dae7f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -459,4 +459,9 @@ public MultipartUploaderBuilder createMultipartUploader(final Path basePath) throws IOException { return myFs.createMultipartUploader(basePath); } + + @Override + public Path getEnclosingRoot(Path path) throws IOException { + return myFs.getEnclosingRoot(path); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index b4cf96ea5996a..c111886e369aa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1370,6 +1370,24 @@ public boolean hasPathCapability(Path path, String capability) } } + @Override + public Path getEnclosingRoot(Path path) throws IOException { + InodeTree.ResolveResult res; + try { + res = fsState.resolve(getUriPath(path), true); + } catch (FileNotFoundException ex) { + NotInMountpointException mountPointEx = + new NotInMountpointException(path, + String.format("getEnclosingRoot - %s", ex.getMessage())); + mountPointEx.initCause(ex); + throw mountPointEx; + } + Path mountPath = new Path(res.resolvedPath); + Path enclosingPath = res.targetFileSystem.getEnclosingRoot(new Path(getUriPath(path))); + return fixRelativePart(this.makeQualified(enclosingPath.depth() > mountPath.depth() + ? enclosingPath : mountPath)); + } + /** * An instance of this class represents an internal dir of the viewFs * that is internal dir of the mount table. @@ -1919,6 +1937,25 @@ public Collection getAllStoragePolicies() } return allPolicies; } + + @Override + public Path getEnclosingRoot(Path path) throws IOException { + InodeTree.ResolveResult res; + try { + res = fsState.resolve((path.toString()), true); + } catch (FileNotFoundException ex) { + NotInMountpointException mountPointEx = + new NotInMountpointException(path, + String.format("getEnclosingRoot - %s", ex.getMessage())); + mountPointEx.initCause(ex); + throw mountPointEx; + } + Path fullPath = new Path(res.resolvedPath); + Path enclosingPath = res.targetFileSystem.getEnclosingRoot(path); + return enclosingPath.depth() > fullPath.depth() + ? enclosingPath + : fullPath; + } } enum RenameStrategy { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 5f54c9cdd06aa..3d85015bea47e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -1477,5 +1477,22 @@ public void setStoragePolicy(Path path, String policyName) throws IOException { throw readOnlyMountTable("setStoragePolicy", path); } + + @Override + public Path getEnclosingRoot(Path path) throws IOException { + InodeTree.ResolveResult res; + try { + res = fsState.resolve((path.toString()), true); + } catch (FileNotFoundException ex) { + NotInMountpointException mountPointEx = + new NotInMountpointException(path, + String.format("getEnclosingRoot - %s", ex.getMessage())); + mountPointEx.initCause(ex); + throw mountPointEx; + } + Path fullPath = new Path(res.resolvedPath); + Path enclosingPath = res.targetFileSystem.getEnclosingRoot(path); + return enclosingPath.depth() > fullPath.depth() ? enclosingPath : fullPath; + } } } diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index fafe2819cf66a..5fba8a2515bb4 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -601,7 +601,40 @@ on the filesystem. 1. The outcome of this operation MUST be identical to the value of `getFileStatus(P).getBlockSize()`. -1. By inference, it MUST be > 0 for any file of length > 0. +2. By inference, it MUST be > 0 for any file of length > 0. + +### `Path getEnclosingRoot(Path p)` + +This method is used to find a root directory for a path given. This is useful for creating +staging and temp directories in the same enclosing root directory. There are constraints around how +renames are allowed to atomically occur (ex. across hdfs volumes or across encryption zones). + +For any two paths p1 and p2 that do not have the same enclosing root, `rename(p1, p2)` is expected to fail or will not +be atomic. + +For object stores, even with the same enclosing root, there is no guarantee file or directory rename is atomic + +The following statement is always true: +`getEnclosingRoot(p) == getEnclosingRoot(getEnclosingRoot(p))` + + +```python +path in ancestors(FS, p) or path == p: +isDir(FS, p) +``` + +#### Preconditions + +The path does not have to exist, but the path does need to be valid and reconcilable by the filesystem +* if a linkfallback is used all paths are reconcilable +* if a linkfallback is not used there must be a mount point covering the path + + +#### Postconditions + +* The path returned will not be null, if there is no deeper enclosing root, the root path ("/") will be returned. +* The path returned is a directory + ## State Changing Operations diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetEnclosingRoot.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetEnclosingRoot.java new file mode 100644 index 0000000000000..8bbab36d53096 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetEnclosingRoot.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.HadoopTestBase; +import org.junit.Test; + +public class TestGetEnclosingRoot extends HadoopTestBase { + @Test + public void testEnclosingRootEquivalence() throws IOException { + FileSystem fs = getFileSystem(); + Path root = path("/"); + Path foobar = path("/foo/bar"); + + assertEquals(root, fs.getEnclosingRoot(root)); + assertEquals(root, fs.getEnclosingRoot(foobar)); + assertEquals(root, fs.getEnclosingRoot(fs.getEnclosingRoot(foobar))); + assertEquals(fs.getEnclosingRoot(root), fs.getEnclosingRoot(foobar)); + + assertEquals(root, fs.getEnclosingRoot(path(foobar.toString()))); + assertEquals(root, fs.getEnclosingRoot(fs.getEnclosingRoot(path(foobar.toString())))); + assertEquals(fs.getEnclosingRoot(root), fs.getEnclosingRoot(path(foobar.toString()))); + } + + @Test + public void testEnclosingRootPathExists() throws Exception { + FileSystem fs = getFileSystem(); + Path root = path("/"); + Path foobar = path("/foo/bar"); + fs.mkdirs(foobar); + + assertEquals(root, fs.getEnclosingRoot(foobar)); + assertEquals(root, fs.getEnclosingRoot(path(foobar.toString()))); + } + + @Test + public void testEnclosingRootPathDNE() throws Exception { + FileSystem fs = getFileSystem(); + Path foobar = path("/foo/bar"); + Path root = path("/"); + + assertEquals(root, fs.getEnclosingRoot(foobar)); + assertEquals(root, fs.getEnclosingRoot(path(foobar.toString()))); + } + + @Test + public void testEnclosingRootWrapped() throws Exception { + FileSystem fs = getFileSystem(); + Path root = path("/"); + + assertEquals(root, fs.getEnclosingRoot(new Path("/foo/bar"))); + + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("foo"); + Path p = ugi.doAs((PrivilegedExceptionAction) () -> { + FileSystem wFs = getFileSystem(); + return wFs.getEnclosingRoot(new Path("/foo/bar")); + }); + assertEquals(root, p); + } + + private FileSystem getFileSystem() throws IOException { + return FileSystem.get(new Configuration()); + } + + /** + * Create a path under the test path provided by + * the FS contract. + * @param filepath path string in + * @return a path qualified by the test filesystem + * @throws IOException IO problems + */ + private Path path(String filepath) throws IOException { + return getFileSystem().makeQualified( + new Path(filepath)); + }} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index b227e16908828..0287b7ec1fb84 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -255,6 +255,8 @@ MultipartUploaderBuilder createMultipartUploader(Path basePath) FSDataOutputStream append(Path f, int bufferSize, Progressable progress, boolean appendToNewBlock) throws IOException; + + Path getEnclosingRoot(Path path) throws IOException; } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetEnclosingRoot.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetEnclosingRoot.java new file mode 100644 index 0000000000000..9564c31725d06 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetEnclosingRoot.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public abstract class AbstractContractGetEnclosingRoot extends AbstractFSContractTestBase { + private static final Logger LOG = LoggerFactory.getLogger(AbstractContractGetEnclosingRoot.class); + + @Test + public void testEnclosingRootEquivalence() throws IOException { + FileSystem fs = getFileSystem(); + Path root = path("/"); + Path foobar = path("/foo/bar"); + + assertEquals("Ensure getEnclosingRoot on the root directory returns the root directory", + root, fs.getEnclosingRoot(foobar)); + assertEquals("Ensure getEnclosingRoot called on itself returns the root directory", + root, fs.getEnclosingRoot(fs.getEnclosingRoot(foobar))); + assertEquals( + "Ensure getEnclosingRoot for different paths in the same enclosing root " + + "returns the same path", + fs.getEnclosingRoot(root), fs.getEnclosingRoot(foobar)); + assertEquals("Ensure getEnclosingRoot on a path returns the root directory", + root, fs.getEnclosingRoot(methodPath())); + assertEquals("Ensure getEnclosingRoot called on itself on a path returns the root directory", + root, fs.getEnclosingRoot(fs.getEnclosingRoot(methodPath()))); + assertEquals( + "Ensure getEnclosingRoot for different paths in the same enclosing root " + + "returns the same path", + fs.getEnclosingRoot(root), + fs.getEnclosingRoot(methodPath())); + } + + + @Test + public void testEnclosingRootPathExists() throws Exception { + FileSystem fs = getFileSystem(); + Path root = path("/"); + Path foobar = methodPath(); + fs.mkdirs(foobar); + + assertEquals( + "Ensure getEnclosingRoot returns the root directory when the root directory exists", + root, fs.getEnclosingRoot(foobar)); + assertEquals("Ensure getEnclosingRoot returns the root directory when the directory exists", + root, fs.getEnclosingRoot(foobar)); + } + + @Test + public void testEnclosingRootPathDNE() throws Exception { + FileSystem fs = getFileSystem(); + Path foobar = path("/foo/bar"); + Path root = path("/"); + + // . + assertEquals( + "Ensure getEnclosingRoot returns the root directory even when the path does not exist", + root, fs.getEnclosingRoot(foobar)); + assertEquals( + "Ensure getEnclosingRoot returns the root directory even when the path does not exist", + root, fs.getEnclosingRoot(methodPath())); + } + + @Test + public void testEnclosingRootWrapped() throws Exception { + FileSystem fs = getFileSystem(); + Path root = path("/"); + + assertEquals("Ensure getEnclosingRoot returns the root directory when the directory exists", + root, fs.getEnclosingRoot(new Path("/foo/bar"))); + + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("foo"); + Path p = ugi.doAs((PrivilegedExceptionAction) () -> { + FileSystem wFs = getContract().getTestFileSystem(); + return wFs.getEnclosingRoot(new Path("/foo/bar")); + }); + assertEquals("Ensure getEnclosingRoot works correctly within a wrapped FileSystem", root, p); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractGetEnclosingRoot.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractGetEnclosingRoot.java new file mode 100644 index 0000000000000..9819245ba6867 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractGetEnclosingRoot.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetEnclosingRoot; +import org.apache.hadoop.fs.contract.AbstractFSContract; + + +public class TestLocalFSContractGetEnclosingRoot + extends AbstractContractGetEnclosingRoot { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractGetEnclosingRoot.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractGetEnclosingRoot.java new file mode 100644 index 0000000000000..7e99cb7b88ed6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractGetEnclosingRoot.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetEnclosingRoot; +import org.apache.hadoop.fs.contract.AbstractFSContract; + + +public class TestRawlocalContractGetEnclosingRoot extends AbstractContractGetEnclosingRoot { + + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 116ed41703858..cbe7516b0ede0 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3529,4 +3529,14 @@ public DatanodeInfo[] slowDatanodeReport() throws IOException { } } + public Path getEnclosingRoot(String src) throws IOException { + checkOpen(); + try (TraceScope ignored = newPathTraceScope("getEnclosingRoot", src)) { + return namenode.getEnclosingRoot(src); + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + UnresolvedPathException.class); + } + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 05d9c562392cf..3fbd40d41d01a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -64,6 +64,7 @@ public enum OpType { GET_EC_CODECS("op_get_ec_codecs"), GET_EC_POLICY("op_get_ec_policy"), GET_EC_POLICIES("op_get_ec_policies"), + GET_ENCLOSING_ROOT("op_get_enclosing_root"), GET_ENCRYPTION_ZONE("op_get_encryption_zone"), GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"), GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM), diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 16093c8e9920c..17c39f6c55b75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -4011,4 +4011,31 @@ public LocatedBlocks next(final FileSystem fs, final Path p) } }.resolve(this, absF); } + + /** + * Return path of the enclosing root for a given path + * The enclosing root path is a common ancestor that should be used for temp and staging dirs + * as well as within encryption zones and other restricted directories. + * + * @param path file path to find the enclosing root path for + * @return a path to the enclosing root + * @throws IOException early checks like failure to resolve path cause IO failures + */ + public Path getEnclosingRoot(final Path path) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_ENCLOSING_ROOT); + Preconditions.checkNotNull(path); + Path absF = fixRelativePart(path); + return new FileSystemLinkResolver() { + @Override + public Path doCall(final Path p) throws IOException { + return dfs.getEnclosingRoot(getPathName(p)); + } + + @Override + public Path next(final FileSystem fs, final Path p) throws IOException { + return fs.getEnclosingRoot(p); + } + }.resolve(this, absF); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 4f2da496a1a3d..b56b7916ff798 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.hdfs.AddBlockFlag; @@ -1888,4 +1889,11 @@ BatchedEntries listOpenFiles(long prevId, @ReadOnly DatanodeInfo[] getSlowDatanodeReport() throws IOException; + /** + * Get the enclosing root for a path. + */ + @Idempotent + @ReadOnly(isCoordinated = true) + Path getEnclosingRoot(String src) throws IOException; + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index ae4a84ead6552..543f0a58e6ec6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.XAttr; @@ -127,6 +128,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEnclosingRootRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEnclosingRootResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto; @@ -1669,4 +1672,19 @@ public HAServiceProtocol.HAServiceState getHAServiceState() } } + @Override + public Path getEnclosingRoot(String filename) throws IOException { + final GetEnclosingRootRequestProto.Builder builder = + GetEnclosingRootRequestProto.newBuilder(); + builder.setFilename(filename); + final GetEnclosingRootRequestProto req = builder.build(); + try { + final GetEnclosingRootResponseProto response = + rpcProxy.getEnclosingRoot(null, req); + return new Path(response.getEnclosingRootPath()); + } catch (ServiceException e) { + throw getRemoteException(e); + } + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 60792b5b6c94c..5ad3fe96f08f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -428,6 +428,14 @@ message GetPreferredBlockSizeResponseProto { message GetSlowDatanodeReportRequestProto { } +message GetEnclosingRootRequestProto { + optional string filename = 1; +} + +message GetEnclosingRootResponseProto { + optional string enclosingRootPath = 1; +} + message GetSlowDatanodeReportResponseProto { repeated DatanodeInfoProto datanodeInfoProto = 1; } @@ -1080,4 +1088,6 @@ service ClientNamenodeProtocol { returns(HAServiceStateResponseProto); rpc getSlowDatanodeReport(GetSlowDatanodeReportRequestProto) returns(GetSlowDatanodeReportResponseProto); + rpc getEnclosingRoot(GetEnclosingRootRequestProto) + returns(GetEnclosingRootResponseProto); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java index f7ea7bcd76dde..c225a98c8244a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java @@ -56,6 +56,7 @@ public class TestReadOnly { "listCachePools", "getAclStatus", "getEZForPath", + "getEnclosingRoot", "listEncryptionZones", "listReencryptionStatus", "getXAttrs", diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index ba0abc11e0276..1f18608deda21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -147,6 +147,9 @@ public class RouterClientProtocol implements ClientProtocol { /** Time out when getting the mount statistics. */ private long mountStatusTimeOut; + /** Default nameservice enabled. */ + private final boolean defaultNameServiceEnabled; + /** Identifier for the super user. */ private String superUser; /** Identifier for the super group. */ @@ -196,6 +199,9 @@ public class RouterClientProtocol implements ClientProtocol { this.routerCacheAdmin = new RouterCacheAdmin(rpcServer); this.securityManager = rpcServer.getRouterSecurityManager(); this.rbfRename = new RouterFederationRename(rpcServer, conf); + this.defaultNameServiceEnabled = conf.getBoolean( + RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE, + RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE_DEFAULT); } @Override @@ -1967,6 +1973,33 @@ public DatanodeInfo[] getSlowDatanodeReport() throws IOException { return rpcServer.getSlowDatanodeReport(true, 0); } + @Override + public Path getEnclosingRoot(String src) throws IOException { + Path mountPath = null; + if (defaultNameServiceEnabled) { + mountPath = new Path("/"); + } + + if (subclusterResolver instanceof MountTableResolver) { + MountTableResolver mountTable = (MountTableResolver) subclusterResolver; + if (mountTable.getMountPoint(src) != null) { + mountPath = new Path(mountTable.getMountPoint(src).getSourcePath()); + } + } + + if (mountPath == null) { + throw new IOException(String.format("No mount point for %s", src)); + } + + EncryptionZone zone = getEZForPath(src); + if (zone == null) { + return mountPath; + } else { + Path zonePath = new Path(zone.getPath()); + return zonePath.depth() > mountPath.depth() ? zonePath : mountPath; + } + } + @Override public HAServiceProtocol.HAServiceState getHAServiceState() { if (rpcServer.isSafeMode()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 2aa2eae5305d0..9d7c1263f09bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -1610,6 +1610,11 @@ public DatanodeInfo[] getSlowDatanodeReport() throws IOException { return clientProto.getSlowDatanodeReport(); } + @Override // ClientProtocol + public Path getEnclosingRoot(String src) throws IOException { + return clientProto.getEnclosingRoot(src); + } + @Override // NamenodeProtocol public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long minBlockSize, long hotBlockTimeInterval, StorageType storageType) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index a346c1a241a80..8812f83febb80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -763,4 +763,29 @@ public void testListStatusMountPoint() throws Exception { nnFs0.delete(new Path("/testLsMountEntryDest"), true); } } + + @Test + public void testGetEnclosingRoot() throws Exception { + + // Add a read only entry + MountTable readOnlyEntry = MountTable.newInstance( + "/readonly", Collections.singletonMap("ns0", "/testdir")); + readOnlyEntry.setReadOnly(true); + assertTrue(addMountTable(readOnlyEntry)); + assertEquals(routerFs.getEnclosingRoot(new Path("/readonly")), new Path("/readonly")); + + assertEquals(routerFs.getEnclosingRoot(new Path("/regular")), new Path("/")); + assertEquals(routerFs.getEnclosingRoot(new Path("/regular")), + routerFs.getEnclosingRoot(routerFs.getEnclosingRoot(new Path("/regular")))); + + // Add a regular entry + MountTable regularEntry = MountTable.newInstance( + "/regular", Collections.singletonMap("ns0", "/testdir")); + assertTrue(addMountTable(regularEntry)); + assertEquals(routerFs.getEnclosingRoot(new Path("/regular")), new Path("/regular")); + + // path does not need to exist + assertEquals(routerFs.getEnclosingRoot(new Path("/regular/pathDNE")), new Path("/regular")); + + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 79c122cf5bae0..25fcdc3080df0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -25,8 +25,6 @@ import java.util.Map; import java.util.stream.Collectors; -import org.apache.hadoop.thirdparty.protobuf.ByteString; -import org.apache.hadoop.thirdparty.protobuf.ProtocolStringList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; @@ -40,6 +38,7 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.BatchedDirectoryListing; @@ -134,6 +133,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEnclosingRootRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEnclosingRootResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto; @@ -303,7 +304,8 @@ import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto; import org.apache.hadoop.security.token.Token; - +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.ProtocolStringList; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -2074,4 +2076,18 @@ public GetSlowDatanodeReportResponseProto getSlowDatanodeReport(RpcController co throw new ServiceException(e); } } + + @Override + public GetEnclosingRootResponseProto getEnclosingRoot( + RpcController controller, GetEnclosingRootRequestProto req) + throws ServiceException { + try { + Path enclosingRootPath = server.getEnclosingRoot(req.getFilename()); + return GetEnclosingRootResponseProto.newBuilder() + .setEnclosingRootPath(enclosingRootPath.toUri().toString()) + .build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7918daf6b9db8..3c6efb78dd1fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -9109,4 +9109,19 @@ private void checkBlockLocationsWhenObserver(LocatedBlocks blocks, String src) } } } + + /** + * Get the enclosing root for the specified path. + * + * @param srcArg the path of a file or directory to get the EZ for. + * @return the enclosing root of the path or null if none. + */ + Path getEnclosingRoot(final String srcArg) throws IOException { + EncryptionZone ez = getEZForPath(srcArg); + if (ez != null) { + return new Path(ez.getPath()); + } else { + return new Path("/"); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index f02688d1629f4..3a8357f4bddf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -2677,4 +2677,11 @@ public Long getNextSPSPath() throws IOException { } return namesystem.getBlockManager().getSPSManager().getNextPathId(); } + + @Override // ClientProtocol + public Path getEnclosingRoot(String src) + throws IOException { + checkNNStartup(); + return namesystem.getEnclosingRoot(src); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 62dc3076d00a6..afa6ccaf2dad4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -57,6 +57,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -191,34 +192,38 @@ Path getTrashRootInFallBackFS() throws IOException { @Test public void testTrashRootsAfterEncryptionZoneDeletion() throws Exception { - final Path zone = new Path("/EZ"); - fsTarget.mkdirs(zone); - final Path zone1 = new Path("/EZ/zone1"); - fsTarget.mkdirs(zone1); - - DFSTestUtil.createKey("test_key", cluster, CONF); - HdfsAdmin hdfsAdmin = new HdfsAdmin(cluster.getURI(0), CONF); - final EnumSet provisionTrash = - EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH); - hdfsAdmin.createEncryptionZone(zone1, "test_key", provisionTrash); - - final Path encFile = new Path(zone1, "encFile"); - DFSTestUtil.createFile(fsTarget, encFile, 10240, (short) 1, 0xFEED); - - Configuration clientConf = new Configuration(CONF); - clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1); - clientConf.set("fs.default.name", fsTarget.getUri().toString()); - FsShell shell = new FsShell(clientConf); - - //Verify file deletion within EZ - DFSTestUtil.verifyDelete(shell, fsTarget, encFile, true); - assertTrue("ViewFileSystem trash roots should include EZ file trash", - (fsView.getTrashRoots(true).size() == 1)); - - //Verify deletion of EZ - DFSTestUtil.verifyDelete(shell, fsTarget, zone, true); - assertTrue("ViewFileSystem trash roots should include EZ zone trash", - (fsView.getTrashRoots(true).size() == 2)); + try { + final Path zone = new Path("/EZ"); + fsTarget.mkdirs(zone); + final Path zone1 = new Path("/EZ/zone1"); + fsTarget.mkdirs(zone1); + + DFSTestUtil.createKey("test_key", cluster, CONF); + HdfsAdmin hdfsAdmin = new HdfsAdmin(cluster.getURI(0), CONF); + final EnumSet provisionTrash = + EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH); + hdfsAdmin.createEncryptionZone(zone1, "test_key", provisionTrash); + + final Path encFile = new Path(zone1, "encFile"); + DFSTestUtil.createFile(fsTarget, encFile, 10240, (short) 1, 0xFEED); + + Configuration clientConf = new Configuration(CONF); + clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1); + clientConf.set("fs.default.name", fsTarget.getUri().toString()); + FsShell shell = new FsShell(clientConf); + + //Verify file deletion within EZ + DFSTestUtil.verifyDelete(shell, fsTarget, encFile, true); + assertTrue("ViewFileSystem trash roots should include EZ file trash", + (fsView.getTrashRoots(true).size() == 1)); + + //Verify deletion of EZ + DFSTestUtil.verifyDelete(shell, fsTarget, zone, true); + assertTrue("ViewFileSystem trash roots should include EZ zone trash", + (fsView.getTrashRoots(true).size() == 2)); + } finally { + DFSTestUtil.deleteKey("test_key", cluster); + } } @Test @@ -506,4 +511,92 @@ public void testInternalDirectoryPermissions() throws IOException { assertEquals(fs.getFileStatus(subDirOfInternalDir).getPermission(), fs.getFileStatus(subDirOfRealDir).getPermission()); } + + private Path getViewFsPath(Path path, FileSystem fs) { + return fs.makeQualified(path); + } + + private Path getViewFsPath(String path, FileSystem fs) { + return getViewFsPath(new Path(path), fs); + } + + @Test + public void testEnclosingRootsBase() throws Exception { + try { + final Path zone = new Path("/data/EZ"); + fsTarget.mkdirs(zone); + final Path zone1 = new Path("/data/EZ/zone1"); + fsTarget.mkdirs(zone1); + + DFSTestUtil.createKey("test_key", cluster, 0, CONF); + HdfsAdmin hdfsAdmin = new HdfsAdmin(cluster.getURI(0), CONF); + final EnumSet provisionTrash = + EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH); + hdfsAdmin.createEncryptionZone(zone1, "test_key", provisionTrash); + assertEquals(fsView.getEnclosingRoot(zone), getViewFsPath("/data", fsView)); + assertEquals(fsView.getEnclosingRoot(zone1), getViewFsPath(zone1, fsView)); + + Path nn02Ez = new Path("/mountOnNn2/EZ"); + fsTarget2.mkdirs(nn02Ez); + assertEquals(fsView.getEnclosingRoot((nn02Ez)), getViewFsPath("/mountOnNn2", fsView)); + HdfsAdmin hdfsAdmin2 = new HdfsAdmin(cluster.getURI(1), CONF); + DFSTestUtil.createKey("test_key", cluster, 1, CONF); + hdfsAdmin2.createEncryptionZone(nn02Ez, "test_key", provisionTrash); + assertEquals(fsView.getEnclosingRoot((nn02Ez)), getViewFsPath(nn02Ez, fsView)); + assertEquals(fsView.getEnclosingRoot(new Path(nn02Ez, "dir/dir2/file")), + getViewFsPath(nn02Ez, fsView)); + + // With viewfs:// scheme + assertEquals(fsView.getEnclosingRoot(fsView.getWorkingDirectory()), + getViewFsPath("/user", fsView)); + } finally { + DFSTestUtil.deleteKey("test_key", cluster, 0); + } + } + + @Test + public void testEnclosingRootFailure() throws Exception { + LambdaTestUtils.intercept(NotInMountpointException.class, + ()-> fsView.getEnclosingRoot(new Path("/does/not/exist"))); + + final Path zone = new Path("/data/EZ"); + Path fs1 = fsTarget.makeQualified(zone); + + LambdaTestUtils.intercept(IllegalArgumentException.class, + ()-> fsView.getEnclosingRoot(fs1)); + LambdaTestUtils.intercept(IllegalArgumentException.class, + ()-> fsView.getEnclosingRoot(new Path("hdfs://fakeAuthority/"))); + } + + @Test + public void testEnclosingRootWrapped() throws Exception { + try { + final Path zone = new Path("/data/EZ"); + fsTarget.mkdirs(zone); + final Path zone1 = new Path("/data/EZ/testZone1"); + fsTarget.mkdirs(zone1); + + DFSTestUtil.createKey("test_key", cluster, 0, CONF); + HdfsAdmin hdfsAdmin = new HdfsAdmin(cluster.getURI(0), CONF); + final EnumSet provisionTrash = + EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH); + hdfsAdmin.createEncryptionZone(zone1, "test_key", provisionTrash); + + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("foo"); + Path p = ugi.doAs((PrivilegedExceptionAction) () -> { + FileSystem wFs = FileSystem.get(FsConstants.VIEWFS_URI, this.conf); + return wFs.getEnclosingRoot(zone); + }); + assertEquals(p, getViewFsPath("/data", fsView)); + p = ugi.doAs((PrivilegedExceptionAction) () -> { + FileSystem wFs = FileSystem.get(FsConstants.VIEWFS_URI, this.conf); + return wFs.getEnclosingRoot(zone1); + }); + assertEquals(p, getViewFsPath(zone1, fsView)); + + + } finally { + DFSTestUtil.deleteKey("test_key", cluster, 0); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 594b46f6cacb0..e816edd3110de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1875,6 +1875,33 @@ public static void createKey(String keyName, MiniDFSCluster cluster, provider.flush(); } + /** + * Helper function to delete a key in the Key Provider. Defaults + * to the first indexed NameNode's Key Provider. + * + * @param keyName The name of the key to create + * @param cluster The cluster to create it in + */ + public static void deleteKey(String keyName, MiniDFSCluster cluster) + throws NoSuchAlgorithmException, IOException { + deleteKey(keyName, cluster, 0); + } + + /** + * Helper function to delete a key in the Key Provider. + * + * @param keyName The name of the key to create + * @param cluster The cluster to create it in + * @param idx The NameNode index + */ + public static void deleteKey(String keyName, MiniDFSCluster cluster, int idx) + throws NoSuchAlgorithmException, IOException { + NameNode nn = cluster.getNameNode(idx); + KeyProvider provider = nn.getNamesystem().getProvider(); + provider.deleteKey(keyName); + provider.flush(); + } + /** * @return the node which is expected to run the recovery of the * given block, which is known to be under construction inside the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 9e8c11d7b0660..8eb048c14235c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -969,6 +969,11 @@ public void testStatistics2() throws IOException, NoSuchAlgorithmException { checkStatistics(dfs, ++readOps, writeOps, 0); checkOpStatistics(OpType.GET_ENCRYPTION_ZONE, opCount + 1); + opCount = getOpStatistics(OpType.GET_ENCLOSING_ROOT); + dfs.getEnclosingRoot(dir); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_ENCLOSING_ROOT, opCount + 1); + opCount = getOpStatistics(OpType.GET_SNAPSHOTTABLE_DIRECTORY_LIST); dfs.getSnapshottableDirListing(); checkStatistics(dfs, ++readOps, writeOps, 0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEnclosingRoot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEnclosingRoot.java new file mode 100644 index 0000000000000..85204e7a1a5b3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEnclosingRoot.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import java.io.File; +import java.util.EnumSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector; +import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager; +import org.apache.hadoop.test.AbstractHadoopTestBase; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestEnclosingRoot extends AbstractHadoopTestBase { + private static final Logger LOG = LoggerFactory.getLogger(TestEnclosingRoot.class); + private static final String TEST_KEY = "test_key"; + private static final EnumSet NO_TRASH = + EnumSet.of(CreateEncryptionZoneFlag.NO_TRASH); + + private Configuration conf; + private FileSystemTestHelper fsHelper; + + private MiniDFSCluster cluster; + private HdfsAdmin dfsAdmin; + private DistributedFileSystem fs; + private File testRootDir; + + private String getKeyProviderURI() { + return JavaKeyStoreProvider.SCHEME_NAME + "://file" + + new Path(testRootDir.toString(), "test.jks").toUri(); + } + + @Before + public void setup() throws Exception { + conf = new HdfsConfiguration(); + fsHelper = new FileSystemTestHelper(); + // Set up java key store + String testRoot = fsHelper.getTestRootDir(); + testRootDir = new File(testRoot).getAbsoluteFile(); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + getKeyProviderURI()); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + // Lower the batch size for testing + conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, + 2); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + GenericTestUtils.setLogLevel( + LoggerFactory.getLogger(EncryptionZoneManager.class), Level.TRACE); + fs = cluster.getFileSystem(); + dfsAdmin = new HdfsAdmin(cluster.getURI(), conf); + setProvider(); + // Create a test key + DFSTestUtil.createKey(TEST_KEY, cluster, conf); + } + + protected void setProvider() { + // Need to set the client's KeyProvider to the NN's for JKS, + // else the updates do not get flushed properly + fs.getClient().setKeyProvider(cluster.getNameNode().getNamesystem() + .getProvider()); + } + + @After + public void teardown() { + try { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } finally { + EncryptionFaultInjector.instance = new EncryptionFaultInjector(); + } + } + + @Test + /** + * Testing basic operations for getEnclosingRoot with dfs/DistributedFileSystem + */ + public void testBasicOperations() throws Exception { + final Path rootDir = new Path("/"); + final Path zone1 = new Path(rootDir, "zone1"); + + // Ensure that the root "/" returns the root without mount points or encryption zones + assertThat(fs.getEnclosingRoot(rootDir)) + .describedAs("enclosing root of %s", rootDir) + .isEqualTo(rootDir); + + // Ensure a dir returns the root without mount points or encryption zones + assertThat(fs.getEnclosingRoot(zone1)) + .describedAs("enclosing root of %s", zone1) + .isEqualTo(rootDir); + + // create an encryption zone + fs.mkdirs(zone1); + dfsAdmin.createEncryptionZone(zone1, TEST_KEY, NO_TRASH); + + // Ensure that the root "/" returns the root with an encryption zone present + assertThat(fs.getEnclosingRoot(rootDir)) + .describedAs("enclosing root of %s", rootDir) + .isEqualTo(rootDir); + + // Ensure that the encryption zone path itself returns correctly as itself + assertThat(fs.getEnclosingRoot(zone1)) + .describedAs("enclosing root of %s", zone1) + .isEqualTo(zone1); + + // Ensure that a path where the file does not exist returns the encryption zone root path + final Path zone1FileDNE = new Path(zone1, "newDNE.txt"); + assertThat(fs.getEnclosingRoot(zone1FileDNE)) + .describedAs("enclosing root of %s", zone1FileDNE) + .isEqualTo(zone1); + + // Ensure that a path where the dir does not exist returns the encryption zone root path + final Path zone1DirDNE = new Path(zone1, "zone2/newDNE.txt"); + assertThat(fs.getEnclosingRoot(zone1DirDNE)) + .describedAs("enclosing root of %s", zone1DirDNE) + .isEqualTo(zone1); + } +} From a32097a921b6a256c82ee8c2a83aa1990c635e0d Mon Sep 17 00:00:00 2001 From: K0K0V0K <109747532+K0K0V0K@users.noreply.github.com> Date: Thu, 9 Nov 2023 10:14:14 +0100 Subject: [PATCH 153/155] HADOOP-18954. Filter NaN values from JMX json interface. (#6229). Reviewed-by: Ferenc Erdelyi Signed-off-by: He Xiaoqiao --- .../fs/CommonConfigurationKeysPublic.java | 8 +++ .../org/apache/hadoop/http/HttpServer2.java | 15 ++++- .../org/apache/hadoop/jmx/JMXJsonServlet.java | 51 ++++++++++----- .../hadoop/jmx/JMXJsonServletNaNFiltered.java | 49 ++++++++++++++ .../src/main/resources/core-default.xml | 12 ++++ .../apache/hadoop/jmx/TestJMXJsonServlet.java | 9 ++- .../jmx/TestJMXJsonServletNaNFiltered.java | 65 +++++++++++++++++++ .../metrics2/lib/TestMutableMetrics.java | 2 + 8 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServletNaNFiltered.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServletNaNFiltered.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 006144e64ad15..24a3167b3db2d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1076,5 +1076,13 @@ public class CommonConfigurationKeysPublic { public static final String IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL = "ipc.server.metrics.update.runner.interval"; public static final int IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL_DEFAULT = 5000; + + /** + * @see + * + * core-default.xml + */ + public static final String JMX_NAN_FILTER = "hadoop.http.jmx.nan-filter.enabled"; + public static final boolean JMX_NAN_FILTER_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 515148e929817..d4bfa41b21c4c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -56,6 +56,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.jmx.JMXJsonServletNaNFiltered; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import com.sun.jersey.spi.container.servlet.ServletContainer; @@ -117,6 +118,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.JMX_NAN_FILTER; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.JMX_NAN_FILTER_DEFAULT; + /** * Create a Jetty embedded server to answer http requests. The primary goal is * to serve up status information for the server. There are three contexts: @@ -785,7 +789,7 @@ private void initializeWebServer(String name, String hostName, } } - addDefaultServlets(); + addDefaultServlets(conf); addPrometheusServlet(conf); addAsyncProfilerServlet(contexts, conf); } @@ -976,12 +980,17 @@ private void setContextAttributes(ServletContextHandler context, /** * Add default servlets. + * @param configuration the hadoop configuration */ - protected void addDefaultServlets() { + protected void addDefaultServlets(Configuration configuration) { // set up default servlets addServlet("stacks", "/stacks", StackServlet.class); addServlet("logLevel", "/logLevel", LogLevel.Servlet.class); - addServlet("jmx", "/jmx", JMXJsonServlet.class); + addServlet("jmx", "/jmx", + configuration.getBoolean(JMX_NAN_FILTER, JMX_NAN_FILTER_DEFAULT) + ? JMXJsonServletNaNFiltered.class + : JMXJsonServlet.class + ); addServlet("conf", "/conf", ConfServlet.class); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java index 85f2d2828562d..f089db502783e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java @@ -17,12 +17,12 @@ package org.apache.hadoop.jmx; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import org.apache.hadoop.http.HttpServer2; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Array; +import java.util.Iterator; +import java.util.Set; import javax.management.AttributeNotFoundException; import javax.management.InstanceNotFoundException; import javax.management.IntrospectionException; @@ -42,12 +42,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.PrintWriter; -import java.lang.management.ManagementFactory; -import java.lang.reflect.Array; -import java.util.Iterator; -import java.util.Set; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.hadoop.http.HttpServer2; /* * This servlet is based off of the JMXProxyServlet from Tomcat 7.0.14. It has @@ -136,6 +138,7 @@ public class JMXJsonServlet extends HttpServlet { * Json Factory to create Json generators for write objects in json format */ protected transient JsonFactory jsonFactory; + /** * Initialize this servlet. */ @@ -386,10 +389,10 @@ private void writeAttribute(JsonGenerator jg, ObjectName oname, MBeanAttributeIn private void writeAttribute(JsonGenerator jg, String attName, Object value) throws IOException { jg.writeFieldName(attName); - writeObject(jg, value); + writeObject(jg, value, attName); } - private void writeObject(JsonGenerator jg, Object value) throws IOException { + private void writeObject(JsonGenerator jg, Object value, String attName) throws IOException { if(value == null) { jg.writeNull(); } else { @@ -399,9 +402,11 @@ private void writeObject(JsonGenerator jg, Object value) throws IOException { int len = Array.getLength(value); for (int j = 0; j < len; j++) { Object item = Array.get(value, j); - writeObject(jg, item); + writeObject(jg, item, attName); } jg.writeEndArray(); + } else if (extraCheck(value)) { + extraWrite(value, attName, jg); } else if(value instanceof Number) { Number n = (Number)value; jg.writeNumber(n.toString()); @@ -421,7 +426,7 @@ private void writeObject(JsonGenerator jg, Object value) throws IOException { TabularData tds = (TabularData)value; jg.writeStartArray(); for(Object entry : tds.values()) { - writeObject(jg, entry); + writeObject(jg, entry, attName); } jg.writeEndArray(); } else { @@ -429,4 +434,18 @@ private void writeObject(JsonGenerator jg, Object value) throws IOException { } } } + + /** + * In case you need to modify the logic, how java objects transforms to json, + * you can overwrite this method to return true in case special handling + * @param value the object what should be judged + * @return true, if it needs special transformation + */ + protected boolean extraCheck(Object value) { + return false; + } + + protected void extraWrite(Object value, String attName, JsonGenerator jg) throws IOException { + throw new NotImplementedException(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServletNaNFiltered.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServletNaNFiltered.java new file mode 100644 index 0000000000000..40e3c1a168f9d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServletNaNFiltered.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.jmx; + +import java.io.IOException; +import java.util.Objects; + +import com.fasterxml.jackson.core.JsonGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * For example in case of MutableGauge we are using numbers, + * but not implementing Number interface, + * so we skip class check here because we can not be sure NaN values are wrapped + * with classes which implements the Number interface + */ +public class JMXJsonServletNaNFiltered extends JMXJsonServlet { + + private static final Logger LOG = + LoggerFactory.getLogger(JMXJsonServletNaNFiltered.class); + + @Override + protected boolean extraCheck(Object value) { + return Objects.equals("NaN", Objects.toString(value).trim()); + } + + @Override + protected void extraWrite(Object value, String attName, JsonGenerator jg) throws IOException { + LOG.debug("The {} attribute with value: {} was identified as NaN " + + "and will be replaced with 0.0", attName, value); + jg.writeNumber(0.0); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 31d980353bf26..c86fd8b98609b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -65,6 +65,18 @@     + + hadoop.http.jmx.nan-filter.enabled + false + + The REST API of the JMX interface can return with NaN values + if the attribute represent a 0.0/0.0 value. + Some JSON parser by default can not parse json attributes like foo:NaN. + If this filter is enabled the NaN values will be converted to 0.0 values, + to make json parse less complicated. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java index 035090ef65b72..ba7de6f437ee5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java @@ -62,10 +62,15 @@ public static void assertReFind(String re, String value) { result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory")); assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result); assertReFind("\"modelerType\"", result); - + + System.setProperty("THE_TEST_OF_THE_NAN_VALUES", String.valueOf(Float.NaN)); result = readOutput(new URL(baseUrl, "/jmx")); assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result); - + assertReFind( + "\"key\"\\s*:\\s*\"THE_TEST_OF_THE_NAN_VALUES\"\\s*,\\s*\"value\"\\s*:\\s*\"NaN\"", + result + ); + // test to get an attribute of a mbean result = readOutput(new URL(baseUrl, "/jmx?get=java.lang:type=Memory::HeapMemoryUsage")); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServletNaNFiltered.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServletNaNFiltered.java new file mode 100644 index 0000000000000..52a52be80a35c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServletNaNFiltered.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.jmx; + +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.http.HttpServerFunctionalTest; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.JMX_NAN_FILTER; + +public class TestJMXJsonServletNaNFiltered extends HttpServerFunctionalTest { + private static HttpServer2 server; + private static URL baseUrl; + + @BeforeClass public static void setup() throws Exception { + Configuration configuration = new Configuration(); + configuration.setBoolean(JMX_NAN_FILTER, true); + server = createTestServer(configuration); + server.start(); + baseUrl = getServerURL(server); + } + + @AfterClass public static void cleanup() throws Exception { + server.stop(); + } + + public static void assertReFind(String re, String value) { + Pattern p = Pattern.compile(re); + Matcher m = p.matcher(value); + assertTrue("'"+p+"' does not match "+value, m.find()); + } + + @Test public void testQuery() throws Exception { + System.setProperty("THE_TEST_OF_THE_NAN_VALUES", String.valueOf(Float.NaN)); + String result = readOutput(new URL(baseUrl, "/jmx")); + assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result); + assertReFind( + "\"key\"\\s*:\\s*\"THE_TEST_OF_THE_NAN_VALUES\"\\s*,\\s*\"value\"\\s*:\\s*0.0", + result + ); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java index 2cc3d706f0c20..1ebc0cbdbf23d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java @@ -634,5 +634,7 @@ public void testMutableGaugeFloat() { assertEquals(3.2f, mgf.value(), 0.0); mgf.incr(); assertEquals(4.2f, mgf.value(), 0.0); + mgf.set(Float.NaN); + assertEquals(Float.NaN, mgf.value(), 0.0); } } From 342e6caba189a5639a26ff3f27ac6af7ef5bd881 Mon Sep 17 00:00:00 2001 From: LiuGuH <444506464@qq.com> Date: Tue, 14 Nov 2023 02:16:31 +0800 Subject: [PATCH 154/155] HDFS-17249. Fix TestDFSUtil.testIsValidName() unit test failure (#6249) Contributed by liuguanghua. --- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 7 +++-- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 27 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 71cff2e3915b0..b2fc472aad835 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -661,9 +661,12 @@ public static boolean isValidName(String src) { String[] components = StringUtils.split(src, '/'); for (int i = 0; i < components.length; i++) { String element = components[i]; + // For Windows, we must allow the : in the drive letter. + if (Shell.WINDOWS && i == 1 && element.endsWith(":")) { + continue; + } if (element.equals(".") || - // For Windows, we must allow the : in the drive letter. - (!Shell.WINDOWS && i == 1 && element.contains(":")) || + (element.contains(":")) || (element.contains("/"))) { return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index f8e8e4120c43f..5d7110d3d9a8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -83,6 +83,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Shell; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -865,13 +866,25 @@ public void testLocalhostReverseLookup() { @Test (timeout=15000) public void testIsValidName() { - assertFalse(DFSUtil.isValidName("/foo/../bar")); - assertFalse(DFSUtil.isValidName("/foo/./bar")); - assertFalse(DFSUtil.isValidName("/foo//bar")); - assertTrue(DFSUtil.isValidName("/")); - assertTrue(DFSUtil.isValidName("/bar/")); - assertFalse(DFSUtil.isValidName("/foo/:/bar")); - assertFalse(DFSUtil.isValidName("/foo:bar")); + String validPaths[] = new String[]{"/", "/bar/"}; + for (String path : validPaths) { + assertTrue("Should have been accepted '" + path + "'", DFSUtil.isValidName(path)); + } + + String invalidPaths[] = + new String[]{"/foo/../bar", "/foo/./bar", "/foo//bar", "/foo/:/bar", "/foo:bar"}; + for (String path : invalidPaths) { + assertFalse("Should have been rejected '" + path + "'", DFSUtil.isValidName(path)); + } + + String windowsPath = "/C:/foo/bar"; + if (Shell.WINDOWS) { + assertTrue("Should have been accepted '" + windowsPath + "' in windows os.", + DFSUtil.isValidName(windowsPath)); + } else { + assertFalse("Should have been rejected '" + windowsPath + "' in unix os.", + DFSUtil.isValidName(windowsPath)); + } } @Test(timeout=5000) From 000a39ba2d2131ac158e23b35eae8c1329681bff Mon Sep 17 00:00:00 2001 From: Anuj Modi <128447756+anujmodi2021@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:36:33 -0800 Subject: [PATCH 155/155] HADOOP-18872: [ABFS] [BugFix] Misreporting Retry Count for Sub-sequential and Parallel Operations (#6019) Contributed by Anuj Modi --- .../fs/azurebfs/services/AbfsClient.java | 201 ++++++++++-------- .../azurebfs/services/AbfsRestOperation.java | 33 ++- .../ITestAzureBlobFileSystemDelete.java | 31 ++- .../ITestAzureBlobFileSystemListStatus.java | 141 +++++++++--- .../fs/azurebfs/TestTracingContext.java | 76 +++---- .../azurebfs/services/AbfsClientTestUtil.java | 162 ++++++++++++++ .../fs/azurebfs/services/ITestAbfsClient.java | 3 +- .../services/ITestAbfsRestOperation.java | 1 + .../TestAbfsRestOperationMockFailures.java | 47 +--- 9 files changed, 502 insertions(+), 193 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 45cb538d0b007..9c1f590da9c5a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -260,26 +260,25 @@ AbfsUriQueryBuilder createDefaultUriQueryBuilder() { return abfsUriQueryBuilder; } - public AbfsRestOperation createFilesystem(TracingContext tracingContext) throws AzureBlobFileSystemException { + public AbfsRestOperation createFilesystem(TracingContext tracingContext) + throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); final AbfsUriQueryBuilder abfsUriQueryBuilder = new AbfsUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( - AbfsRestOperationType.CreateFileSystem, - this, - HTTP_METHOD_PUT, - url, - requestHeaders); + final AbfsRestOperation op = getAbfsRestOperation( + AbfsRestOperationType.CreateFileSystem, + HTTP_METHOD_PUT, url, requestHeaders); op.execute(tracingContext); return op; } - public AbfsRestOperation setFilesystemProperties(final String properties, TracingContext tracingContext) throws AzureBlobFileSystemException { + public AbfsRestOperation setFilesystemProperties(final String properties, + TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - // JDK7 does not support PATCH, so to workaround the issue we will use + // JDK7 does not support PATCH, so to work around the issue we will use // PUT and specify the real method in the X-Http-Method-Override header. requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE, HTTP_METHOD_PATCH)); @@ -291,9 +290,8 @@ public AbfsRestOperation setFilesystemProperties(final String properties, Tracin abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.SetFileSystemProperties, - this, HTTP_METHOD_PUT, url, requestHeaders); @@ -316,9 +314,8 @@ public AbfsRestOperation listPath(final String relativePath, final boolean recur appendSASTokenToQuery(relativePath, SASTokenProvider.LIST_OPERATION, abfsUriQueryBuilder); final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.ListPaths, - this, HTTP_METHOD_GET, url, requestHeaders); @@ -333,9 +330,8 @@ public AbfsRestOperation getFilesystemProperties(TracingContext tracingContext) abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.GetFileSystemProperties, - this, HTTP_METHOD_HEAD, url, requestHeaders); @@ -350,9 +346,8 @@ public AbfsRestOperation deleteFilesystem(TracingContext tracingContext) throws abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.DeleteFileSystem, - this, HTTP_METHOD_DELETE, url, requestHeaders); @@ -396,9 +391,8 @@ public AbfsRestOperation createPath(final String path, final boolean isFile, fin appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.CreatePath, - this, HTTP_METHOD_PUT, url, requestHeaders); @@ -431,9 +425,8 @@ public AbfsRestOperation acquireLease(final String path, int duration, TracingCo final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.LeasePath, - this, HTTP_METHOD_POST, url, requestHeaders); @@ -451,9 +444,8 @@ public AbfsRestOperation renewLease(final String path, final String leaseId, final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.LeasePath, - this, HTTP_METHOD_POST, url, requestHeaders); @@ -471,9 +463,8 @@ public AbfsRestOperation releaseLease(final String path, final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.LeasePath, - this, HTTP_METHOD_POST, url, requestHeaders); @@ -491,9 +482,8 @@ public AbfsRestOperation breakLease(final String path, final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.LeasePath, - this, HTTP_METHOD_POST, url, requestHeaders); @@ -646,9 +636,8 @@ private boolean checkIsDir(AbfsHttpOperation result) { @VisibleForTesting AbfsRestOperation createRenameRestOperation(URL url, List requestHeaders) { - AbfsRestOperation op = new AbfsRestOperation( + AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.RenamePath, - this, HTTP_METHOD_PUT, url, requestHeaders); @@ -766,7 +755,8 @@ public AbfsRestOperation append(final String path, final byte[] buffer, abfsUriQueryBuilder, cachedSasToken); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = getAbfsRestOperationForAppend(AbfsRestOperationType.Append, + final AbfsRestOperation op = getAbfsRestOperation( + AbfsRestOperationType.Append, HTTP_METHOD_PUT, url, requestHeaders, @@ -801,7 +791,7 @@ public AbfsRestOperation append(final String path, final byte[] buffer, if (reqParams.isAppendBlob() && appendSuccessCheckOp(op, path, (reqParams.getPosition() + reqParams.getLength()), tracingContext)) { - final AbfsRestOperation successOp = getAbfsRestOperationForAppend( + final AbfsRestOperation successOp = getAbfsRestOperation( AbfsRestOperationType.Append, HTTP_METHOD_PUT, url, @@ -819,38 +809,6 @@ && appendSuccessCheckOp(op, path, return op; } - /** - * Returns the rest operation for append. - * @param operationType The AbfsRestOperationType. - * @param httpMethod specifies the httpMethod. - * @param url specifies the url. - * @param requestHeaders This includes the list of request headers. - * @param buffer The buffer to write into. - * @param bufferOffset The buffer offset. - * @param bufferLength The buffer Length. - * @param sasTokenForReuse The sasToken. - * @return AbfsRestOperation op. - */ - @VisibleForTesting - AbfsRestOperation getAbfsRestOperationForAppend(final AbfsRestOperationType operationType, - final String httpMethod, - final URL url, - final List requestHeaders, - final byte[] buffer, - final int bufferOffset, - final int bufferLength, - final String sasTokenForReuse) { - return new AbfsRestOperation( - operationType, - this, - httpMethod, - url, - requestHeaders, - buffer, - bufferOffset, - bufferLength, sasTokenForReuse); - } - /** * Returns true if the status code lies in the range of user error. * @param responseStatusCode http response status code. @@ -907,9 +865,8 @@ public AbfsRestOperation flush(final String path, final long position, abfsUriQueryBuilder, cachedSasToken); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.Flush, - this, HTTP_METHOD_PUT, url, requestHeaders, sasTokenForReuse); @@ -934,9 +891,8 @@ public AbfsRestOperation setPathProperties(final String path, final String prope appendSASTokenToQuery(path, SASTokenProvider.SET_PROPERTIES_OPERATION, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.SetPathProperties, - this, HTTP_METHOD_PUT, url, requestHeaders); @@ -963,9 +919,8 @@ public AbfsRestOperation getPathStatus(final String path, final boolean includeP appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.GetPathStatus, - this, HTTP_METHOD_HEAD, url, requestHeaders); @@ -988,9 +943,8 @@ public AbfsRestOperation read(final String path, final long position, final byte abfsUriQueryBuilder, cachedSasToken); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.ReadFile, - this, HTTP_METHOD_GET, url, requestHeaders, @@ -1063,9 +1017,8 @@ public AbfsRestOperation deleteIdempotencyCheckOp(final AbfsRestOperation op) { && DEFAULT_DELETE_CONSIDERED_IDEMPOTENT) { // Server has returned HTTP 404, which means path no longer // exists. Assuming delete result to be idempotent, return success. - final AbfsRestOperation successOp = new AbfsRestOperation( + final AbfsRestOperation successOp = getAbfsRestOperation( AbfsRestOperationType.DeletePath, - this, HTTP_METHOD_DELETE, op.getUrl(), op.getRequestHeaders()); @@ -1098,9 +1051,8 @@ public AbfsRestOperation setOwner(final String path, final String owner, final S appendSASTokenToQuery(path, SASTokenProvider.SET_OWNER_OPERATION, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.SetOwner, - this, AbfsHttpConstants.HTTP_METHOD_PUT, url, requestHeaders); @@ -1124,9 +1076,8 @@ public AbfsRestOperation setPermission(final String path, final String permissio appendSASTokenToQuery(path, SASTokenProvider.SET_PERMISSION_OPERATION, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.SetPermissions, - this, AbfsHttpConstants.HTTP_METHOD_PUT, url, requestHeaders); @@ -1159,9 +1110,8 @@ public AbfsRestOperation setAcl(final String path, final String aclSpecString, f appendSASTokenToQuery(path, SASTokenProvider.SET_ACL_OPERATION, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.SetAcl, - this, AbfsHttpConstants.HTTP_METHOD_PUT, url, requestHeaders); @@ -1184,9 +1134,8 @@ public AbfsRestOperation getAclStatus(final String path, final boolean useUPN, appendSASTokenToQuery(path, SASTokenProvider.GET_ACL_OPERATION, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( + final AbfsRestOperation op = getAbfsRestOperation( AbfsRestOperationType.GetAcl, - this, AbfsHttpConstants.HTTP_METHOD_HEAD, url, requestHeaders); @@ -1211,9 +1160,11 @@ public AbfsRestOperation checkAccess(String path, String rwx, TracingContext tra abfsUriQueryBuilder.addQuery(QUERY_FS_ACTION, rwx); appendSASTokenToQuery(path, SASTokenProvider.CHECK_ACCESS_OPERATION, abfsUriQueryBuilder); URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - AbfsRestOperation op = new AbfsRestOperation( - AbfsRestOperationType.CheckAccess, this, - AbfsHttpConstants.HTTP_METHOD_HEAD, url, createDefaultHeaders()); + AbfsRestOperation op = getAbfsRestOperation( + AbfsRestOperationType.CheckAccess, + AbfsHttpConstants.HTTP_METHOD_HEAD, + url, + createDefaultHeaders()); op.execute(tracingContext); return op; } @@ -1238,7 +1189,7 @@ public static String getDirectoryQueryParameter(final String path) { } /** - * If configured for SAS AuthType, appends SAS token to queryBuilder + * If configured for SAS AuthType, appends SAS token to queryBuilder. * @param path * @param operation * @param queryBuilder @@ -1250,7 +1201,7 @@ private String appendSASTokenToQuery(String path, String operation, AbfsUriQuery } /** - * If configured for SAS AuthType, appends SAS token to queryBuilder + * If configured for SAS AuthType, appends SAS token to queryBuilder. * @param path * @param operation * @param queryBuilder @@ -1459,4 +1410,82 @@ public void addCallback(ListenableFuture future, FutureCallback callba protected AccessTokenProvider getTokenProvider() { return tokenProvider; } + + /** + * Creates an AbfsRestOperation with additional parameters for buffer and SAS token. + * + * @param operationType The type of the operation. + * @param httpMethod The HTTP method of the operation. + * @param url The URL associated with the operation. + * @param requestHeaders The list of HTTP headers for the request. + * @param buffer The byte buffer containing data for the operation. + * @param bufferOffset The offset within the buffer where the data starts. + * @param bufferLength The length of the data within the buffer. + * @param sasTokenForReuse The SAS token for reusing authentication. + * @return An AbfsRestOperation instance. + */ + AbfsRestOperation getAbfsRestOperation(final AbfsRestOperationType operationType, + final String httpMethod, + final URL url, + final List requestHeaders, + final byte[] buffer, + final int bufferOffset, + final int bufferLength, + final String sasTokenForReuse) { + return new AbfsRestOperation( + operationType, + this, + httpMethod, + url, + requestHeaders, + buffer, + bufferOffset, + bufferLength, + sasTokenForReuse); + } + + /** + * Creates an AbfsRestOperation with basic parameters and no buffer or SAS token. + * + * @param operationType The type of the operation. + * @param httpMethod The HTTP method of the operation. + * @param url The URL associated with the operation. + * @param requestHeaders The list of HTTP headers for the request. + * @return An AbfsRestOperation instance. + */ + AbfsRestOperation getAbfsRestOperation(final AbfsRestOperationType operationType, + final String httpMethod, + final URL url, + final List requestHeaders) { + return new AbfsRestOperation( + operationType, + this, + httpMethod, + url, + requestHeaders + ); + } + + /** + * Creates an AbfsRestOperation with parameters including request headers and SAS token. + * + * @param operationType The type of the operation. + * @param httpMethod The HTTP method of the operation. + * @param url The URL associated with the operation. + * @param requestHeaders The list of HTTP headers for the request. + * @param sasTokenForReuse The SAS token for reusing authentication. + * @return An AbfsRestOperation instance. + */ + AbfsRestOperation getAbfsRestOperation(final AbfsRestOperationType operationType, + final String httpMethod, + final URL url, + final List requestHeaders, + final String sasTokenForReuse) { + return new AbfsRestOperation( + operationType, + this, + httpMethod, + url, + requestHeaders, sasTokenForReuse); + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 6402be72ddc37..f40cd2cea81ed 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -82,6 +82,11 @@ public class AbfsRestOperation { */ private String failureReason; + /** + * This variable stores the tracing context used for last Rest Operation. + */ + private TracingContext lastUsedTracingContext; + /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -197,10 +202,13 @@ String getSasToken() { public void execute(TracingContext tracingContext) throws AzureBlobFileSystemException { + // Since this might be a sub-sequential or parallel rest operation + // triggered by a single file system call, using a new tracing context. + lastUsedTracingContext = createNewTracingContext(tracingContext); try { IOStatisticsBinding.trackDurationOfInvocation(abfsCounters, AbfsStatistic.getStatNameFromHttpCall(method), - () -> completeExecute(tracingContext)); + () -> completeExecute(lastUsedTracingContext)); } catch (AzureBlobFileSystemException aze) { throw aze; } catch (IOException e) { @@ -214,7 +222,7 @@ public void execute(TracingContext tracingContext) * HTTP operations. * @param tracingContext TracingContext instance to track correlation IDs */ - private void completeExecute(TracingContext tracingContext) + void completeExecute(TracingContext tracingContext) throws AzureBlobFileSystemException { // see if we have latency reports from the previous requests String latencyHeader = getClientLatency(); @@ -409,4 +417,25 @@ private void incrementCounter(AbfsStatistic statistic, long value) { abfsCounters.incrementCounter(statistic, value); } } + + /** + * Creates a new Tracing context before entering the retry loop of a rest operation. + * This will ensure all rest operations have unique + * tracing context that will be used for all the retries. + * @param tracingContext original tracingContext. + * @return tracingContext new tracingContext object created from original one. + */ + @VisibleForTesting + public TracingContext createNewTracingContext(final TracingContext tracingContext) { + return new TracingContext(tracingContext); + } + + /** + * Returns the tracing contest used for last rest operation made. + * @return tracingContext lasUserTracingContext. + */ + @VisibleForTesting + public final TracingContext getLastTracingContext() { + return lastUsedTracingContext; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index 1f0ff667522da..57f5702f74fab 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -29,10 +29,14 @@ import org.assertj.core.api.Assertions; import org.junit.Assume; import org.junit.Test; +import org.mockito.Mockito; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil; import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient; @@ -61,7 +65,6 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; - /** * Test delete operation. */ @@ -257,14 +260,15 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception { // Case 2: Mimic retried case // Idempotency check on Delete always returns success - AbfsRestOperation idempotencyRetOp = ITestAbfsClient.getRestOp( + AbfsRestOperation idempotencyRetOp = Mockito.spy(ITestAbfsClient.getRestOp( DeletePath, mockClient, HTTP_METHOD_DELETE, ITestAbfsClient.getTestUrl(mockClient, "/NonExistingPath"), - ITestAbfsClient.getTestRequestHeaders(mockClient)); + ITestAbfsClient.getTestRequestHeaders(mockClient))); idempotencyRetOp.hardSetResult(HTTP_OK); doReturn(idempotencyRetOp).when(mockClient).deleteIdempotencyCheckOp(any()); TracingContext tracingContext = getTestTracingContext(fs, false); + doReturn(tracingContext).when(idempotencyRetOp).createNewTracingContext(any()); when(mockClient.deletePath("/NonExistingPath", false, null, tracingContext)) .thenCallRealMethod(); @@ -283,4 +287,25 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception { mockStore.delete(new Path("/NonExistingPath"), false, getTestTracingContext(fs, false)); } + @Test + public void deleteBlobDirParallelThreadToDeleteOnDifferentTracingContext() + throws Exception { + Configuration configuration = getRawConfiguration(); + AzureBlobFileSystem fs = Mockito.spy( + (AzureBlobFileSystem) FileSystem.newInstance(configuration)); + AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore()); + AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient()); + + Mockito.doReturn(spiedStore).when(fs).getAbfsStore(); + spiedStore.setClient(spiedClient); + + fs.mkdirs(new Path("/testDir")); + fs.create(new Path("/testDir/file1")); + fs.create(new Path("/testDir/file2")); + + AbfsClientTestUtil.hookOnRestOpsForTracingContextSingularity(spiedClient); + + fs.delete(new Path("/testDir"), true); + fs.close(); + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java index 8d1330b5ea7dd..e7f57b8af54d0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java @@ -20,6 +20,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; @@ -28,6 +29,8 @@ import java.util.concurrent.Future; import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.stubbing.Stubber; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -36,16 +39,30 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultEntrySchema; +import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultSchema; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat; import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator; import org.apache.hadoop.fs.contract.ContractTestUtils; +import static java.net.HttpURLConnection.HTTP_OK; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_LIST_MAX_RESULTS; +import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists; import static org.apache.hadoop.fs.contract.ContractTestUtils.rename; import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; /** * Test listStatus operation. @@ -53,6 +70,7 @@ public class ITestAzureBlobFileSystemListStatus extends AbstractAbfsIntegrationTest { private static final int TEST_FILES_NUMBER = 6000; + private static final String TEST_CONTINUATION_TOKEN = "continuation"; public ITestAzureBlobFileSystemListStatus() throws Exception { super(); @@ -62,34 +80,105 @@ public ITestAzureBlobFileSystemListStatus() throws Exception { public void testListPath() throws Exception { Configuration config = new Configuration(this.getRawConfiguration()); config.set(AZURE_LIST_MAX_RESULTS, "5000"); - final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem - .newInstance(getFileSystem().getUri(), config); - final List> tasks = new ArrayList<>(); - - ExecutorService es = Executors.newFixedThreadPool(10); - for (int i = 0; i < TEST_FILES_NUMBER; i++) { - final Path fileName = new Path("/test" + i); - Callable callable = new Callable() { - @Override - public Void call() throws Exception { - touch(fileName); - return null; - } - }; - - tasks.add(es.submit(callable)); - } - - for (Future task : tasks) { - task.get(); + try (final AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem + .newInstance(getFileSystem().getUri(), config)) { + final List> tasks = new ArrayList<>(); + + ExecutorService es = Executors.newFixedThreadPool(10); + for (int i = 0; i < TEST_FILES_NUMBER; i++) { + final Path fileName = new Path("/test" + i); + Callable callable = new Callable() { + @Override + public Void call() throws Exception { + touch(fileName); + return null; + } + }; + + tasks.add(es.submit(callable)); + } + + for (Future task : tasks) { + task.get(); + } + + es.shutdownNow(); + fs.registerListener( + new TracingHeaderValidator(getConfiguration().getClientCorrelationId(), + fs.getFileSystemId(), FSOperationType.LISTSTATUS, true, 0)); + FileStatus[] files = fs.listStatus(new Path("/")); + assertEquals(TEST_FILES_NUMBER, files.length /* user directory */); } + } - es.shutdownNow(); - fs.registerListener( - new TracingHeaderValidator(getConfiguration().getClientCorrelationId(), - fs.getFileSystemId(), FSOperationType.LISTSTATUS, true, 0)); - FileStatus[] files = fs.listStatus(new Path("/")); - assertEquals(TEST_FILES_NUMBER, files.length /* user directory */); + /** + * Test to verify that each paginated call to ListBlobs uses a new tracing context. + * @throws Exception + */ + @Test + public void testListPathTracingContext() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + final AzureBlobFileSystem spiedFs = Mockito.spy(fs); + final AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore()); + final AbfsClient spiedClient = Mockito.spy(fs.getAbfsClient()); + final TracingContext spiedTracingContext = Mockito.spy( + new TracingContext( + fs.getClientCorrelationId(), fs.getFileSystemId(), + FSOperationType.LISTSTATUS, true, TracingHeaderFormat.ALL_ID_FORMAT, null)); + + Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore(); + spiedStore.setClient(spiedClient); + spiedFs.setWorkingDirectory(new Path("/")); + + AbfsClientTestUtil.setMockAbfsRestOperationForListPathOperation(spiedClient, + (httpOperation) -> { + + ListResultEntrySchema entry = new ListResultEntrySchema() + .withName("a") + .withIsDirectory(true); + List paths = new ArrayList<>(); + paths.add(entry); + paths.clear(); + entry = new ListResultEntrySchema() + .withName("abc.txt") + .withIsDirectory(false); + paths.add(entry); + ListResultSchema schema1 = new ListResultSchema().withPaths(paths); + ListResultSchema schema2 = new ListResultSchema().withPaths(paths); + + when(httpOperation.getListResultSchema()).thenReturn(schema1) + .thenReturn(schema2); + when(httpOperation.getResponseHeader( + HttpHeaderConfigurations.X_MS_CONTINUATION)) + .thenReturn(TEST_CONTINUATION_TOKEN) + .thenReturn(EMPTY_STRING); + + Stubber stubber = Mockito.doThrow( + new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE)); + stubber.doNothing().when(httpOperation).processResponse( + nullable(byte[].class), nullable(int.class), nullable(int.class)); + + when(httpOperation.getStatusCode()).thenReturn(-1).thenReturn(HTTP_OK); + return httpOperation; + }); + + List fileStatuses = new ArrayList<>(); + spiedStore.listStatus(new Path("/"), "", fileStatuses, true, null, spiedTracingContext); + + // Assert that there were 2 paginated ListPath calls were made 1 and 2. + // 1. Without continuation token + Mockito.verify(spiedClient, times(1)).listPath( + "/", false, + spiedFs.getAbfsStore().getAbfsConfiguration().getListMaxResults(), + null, spiedTracingContext); + // 2. With continuation token + Mockito.verify(spiedClient, times(1)).listPath( + "/", false, + spiedFs.getAbfsStore().getAbfsConfiguration().getListMaxResults(), + TEST_CONTINUATION_TOKEN, spiedTracingContext); + + // Assert that none of the API calls used the same tracing header. + Mockito.verify(spiedTracingContext, times(0)).constructHeader(any(), any()); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java index 23e65ed2dd246..2da530364c13b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java @@ -83,44 +83,46 @@ public void checkCorrelationConfigValidation(String clientCorrelationId, boolean includeInHeader) throws Exception { Configuration conf = getRawConfiguration(); conf.set(FS_AZURE_CLIENT_CORRELATIONID, clientCorrelationId); - AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(conf); - - String correlationID = fs.getClientCorrelationId(); - if (includeInHeader) { - Assertions.assertThat(correlationID) - .describedAs("Correlation ID should match config when valid") - .isEqualTo(clientCorrelationId); - } else { - Assertions.assertThat(correlationID) - .describedAs("Invalid ID should be replaced with empty string") - .isEqualTo(EMPTY_STRING); + try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(conf)) { + + String correlationID = fs.getClientCorrelationId(); + if (includeInHeader) { + Assertions.assertThat(correlationID) + .describedAs("Correlation ID should match config when valid") + .isEqualTo(clientCorrelationId); + } else { + Assertions.assertThat(correlationID) + .describedAs("Invalid ID should be replaced with empty string") + .isEqualTo(EMPTY_STRING); + } + TracingContext tracingContext = new TracingContext(clientCorrelationId, + fs.getFileSystemId(), FSOperationType.TEST_OP, + TracingHeaderFormat.ALL_ID_FORMAT, null); + boolean isNamespaceEnabled = fs.getIsNamespaceEnabled(tracingContext); + String path = getRelativePath(new Path("/testDir")); + String permission = isNamespaceEnabled + ? getOctalNotation(FsPermission.getDirDefault()) + : null; + String umask = isNamespaceEnabled + ? getOctalNotation(FsPermission.getUMask(fs.getConf())) + : null; + + //request should not fail for invalid clientCorrelationID + AbfsRestOperation op = fs.getAbfsClient() + .createPath(path, false, true, permission, umask, false, null, + tracingContext); + + int statusCode = op.getResult().getStatusCode(); + Assertions.assertThat(statusCode).describedAs("Request should not fail") + .isEqualTo(HTTP_CREATED); + + String requestHeader = op.getResult().getClientRequestId().replace("[", "") + .replace("]", ""); + Assertions.assertThat(requestHeader) + .describedAs("Client Request Header should match TracingContext") + .isEqualTo(op.getLastTracingContext().getHeader()); + } - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fs.getFileSystemId(), FSOperationType.TEST_OP, - TracingHeaderFormat.ALL_ID_FORMAT, null); - boolean isNamespaceEnabled = fs.getIsNamespaceEnabled(tracingContext); - String path = getRelativePath(new Path("/testDir")); - String permission = isNamespaceEnabled - ? getOctalNotation(FsPermission.getDirDefault()) - : null; - String umask = isNamespaceEnabled - ? getOctalNotation(FsPermission.getUMask(fs.getConf())) - : null; - - //request should not fail for invalid clientCorrelationID - AbfsRestOperation op = fs.getAbfsClient() - .createPath(path, false, true, permission, umask, false, null, - tracingContext); - - int statusCode = op.getResult().getStatusCode(); - Assertions.assertThat(statusCode).describedAs("Request should not fail") - .isEqualTo(HTTP_CREATED); - - String requestHeader = op.getResult().getClientRequestId().replace("[", "") - .replace("]", ""); - Assertions.assertThat(requestHeader) - .describedAs("Client Request Header should match TracingContext") - .isEqualTo(tracingContext.getHeader()); } @Ignore diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java new file mode 100644 index 0000000000000..875682fe20320 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientTestUtil.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.locks.ReentrantLock; + +import org.assertj.core.api.Assertions; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.util.functional.FunctionRaisingIOE; + +import static java.net.HttpURLConnection.HTTP_OK; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_GET; +import static org.apache.hadoop.fs.azurebfs.services.AuthType.OAuth; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; + +/** + * Utility class to help defining mock behavior on AbfsClient and AbfsRestOperation + * objects which are protected inside services package. + */ +public final class AbfsClientTestUtil { + + private AbfsClientTestUtil() { + + } + + public static void setMockAbfsRestOperationForListPathOperation( + final AbfsClient spiedClient, + FunctionRaisingIOE functionRaisingIOE) + throws Exception { + ExponentialRetryPolicy retryPolicy = Mockito.mock(ExponentialRetryPolicy.class); + AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class); + AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation( + AbfsRestOperationType.ListPaths, + spiedClient, + HTTP_METHOD_GET, + null, + new ArrayList<>() + )); + + Mockito.doReturn(abfsRestOperation).when(spiedClient).getAbfsRestOperation( + eq(AbfsRestOperationType.ListPaths), any(), any(), any()); + + addGeneralMockBehaviourToAbfsClient(spiedClient, retryPolicy); + addGeneralMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation); + + functionRaisingIOE.apply(httpOperation); + } + + /** + * Adding general mock behaviour to AbfsRestOperation and AbfsHttpOperation + * to avoid any NPE occurring. These will avoid any network call made and + * will return the relevant exception or return value directly. + * @param abfsRestOperation to be mocked + * @param httpOperation to be mocked + * @throws IOException + */ + public static void addGeneralMockBehaviourToRestOpAndHttpOp(final AbfsRestOperation abfsRestOperation, + final AbfsHttpOperation httpOperation) throws IOException { + HttpURLConnection httpURLConnection = Mockito.mock(HttpURLConnection.class); + Mockito.doNothing().when(httpURLConnection) + .setRequestProperty(nullable(String.class), nullable(String.class)); + Mockito.doReturn(httpURLConnection).when(httpOperation).getConnection(); + Mockito.doReturn("").when(abfsRestOperation).getClientLatency(); + Mockito.doReturn(httpOperation).when(abfsRestOperation).createHttpOperation(); + } + + /** + * Adding general mock behaviour to AbfsClient to avoid any NPE occurring. + * These will avoid any network call made and will return the relevant exception or return value directly. + * @param abfsClient to be mocked + * @param retryPolicy to be mocked + * @throws IOException + */ + public static void addGeneralMockBehaviourToAbfsClient(final AbfsClient abfsClient, + final ExponentialRetryPolicy retryPolicy) throws IOException { + Mockito.doReturn(OAuth).when(abfsClient).getAuthType(); + Mockito.doReturn("").when(abfsClient).getAccessToken(); + AbfsThrottlingIntercept intercept = Mockito.mock( + AbfsThrottlingIntercept.class); + Mockito.doReturn(intercept).when(abfsClient).getIntercept(); + Mockito.doNothing() + .when(intercept) + .sendingRequest(any(), nullable(AbfsCounters.class)); + Mockito.doNothing().when(intercept).updateMetrics(any(), any()); + + Mockito.doReturn(retryPolicy).when(abfsClient).getRetryPolicy(); + Mockito.doReturn(true) + .when(retryPolicy) + .shouldRetry(nullable(Integer.class), nullable(Integer.class)); + Mockito.doReturn(false).when(retryPolicy).shouldRetry(0, HTTP_OK); + Mockito.doReturn(false).when(retryPolicy).shouldRetry(1, HTTP_OK); + Mockito.doReturn(false).when(retryPolicy).shouldRetry(2, HTTP_OK); + } + + public static void hookOnRestOpsForTracingContextSingularity(AbfsClient client) { + Set tracingContextSet = new HashSet<>(); + ReentrantLock lock = new ReentrantLock(); + Answer answer = new Answer() { + @Override + public Object answer(final InvocationOnMock invocationOnMock) + throws Throwable { + AbfsRestOperation op = Mockito.spy((AbfsRestOperation) invocationOnMock.callRealMethod()); + Mockito.doAnswer(completeExecuteInvocation -> { + lock.lock(); + try { + TracingContext context = completeExecuteInvocation.getArgument(0); + Assertions.assertThat(tracingContextSet).doesNotContain(context); + tracingContextSet.add(context); + } finally { + lock.unlock(); + } + return completeExecuteInvocation.callRealMethod(); + }).when(op).completeExecute(Mockito.any(TracingContext.class)); + return op; + } + }; + + Mockito.doAnswer(answer) + .when(client) + .getAbfsRestOperation(Mockito.any(AbfsRestOperationType.class), + Mockito.anyString(), Mockito.any(URL.class), Mockito.anyList(), + Mockito.nullable(byte[].class), Mockito.anyInt(), Mockito.anyInt(), + Mockito.nullable(String.class)); + Mockito.doAnswer(answer) + .when(client) + .getAbfsRestOperation(Mockito.any(AbfsRestOperationType.class), + Mockito.anyString(), Mockito.any(URL.class), Mockito.anyList()); + Mockito.doAnswer(answer) + .when(client) + .getAbfsRestOperation(Mockito.any(AbfsRestOperationType.class), + Mockito.anyString(), Mockito.any(URL.class), Mockito.anyList(), + Mockito.nullable(String.class)); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java index 18d1e3917f24e..6707c593f5a76 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java @@ -58,6 +58,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -592,7 +593,7 @@ public void testExpectHundredContinue() throws Exception { // Mock the restOperation for the client. Mockito.doReturn(op) .when(testClient) - .getAbfsRestOperationForAppend(Mockito.any(), + .getAbfsRestOperation(eq(AbfsRestOperationType.Append), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.nullable(int.class), Mockito.nullable(int.class), Mockito.any()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsRestOperation.java index 6ffe2e2773bbf..6574a808f92bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsRestOperation.java @@ -290,6 +290,7 @@ public void testExpectHundredContinue() throws Exception { TracingContext tracingContext = Mockito.spy(new TracingContext("abcd", "abcde", FSOperationType.APPEND, TracingHeaderFormat.ALL_ID_FORMAT, null)); + Mockito.doReturn(tracingContext).when(op).createNewTracingContext(Mockito.any()); switch (errorType) { case WRITE: diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperationMockFailures.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperationMockFailures.java index bfa524a25e600..b302a1fa939e7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperationMockFailures.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperationMockFailures.java @@ -18,9 +18,7 @@ package org.apache.hadoop.fs.azurebfs.services; -import java.io.IOException; import java.io.InterruptedIOException; -import java.net.HttpURLConnection; import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; @@ -39,7 +37,8 @@ import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT; import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT; -import static org.apache.hadoop.fs.azurebfs.services.AuthType.OAuth; +import static org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil.addGeneralMockBehaviourToAbfsClient; +import static org.apache.hadoop.fs.azurebfs.services.AbfsClientTestUtil.addGeneralMockBehaviourToRestOpAndHttpOp; import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION; import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE; import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION; @@ -166,7 +165,7 @@ private void testClientRequestIdForStatusRetry(int status, AbfsClient abfsClient = Mockito.mock(AbfsClient.class); ExponentialRetryPolicy retryPolicy = Mockito.mock( ExponentialRetryPolicy.class); - addMockBehaviourToAbfsClient(abfsClient, retryPolicy); + addGeneralMockBehaviourToAbfsClient(abfsClient, retryPolicy); AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation( @@ -178,7 +177,7 @@ private void testClientRequestIdForStatusRetry(int status, )); AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class); - addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation); + addGeneralMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation); Mockito.doNothing() .doNothing() @@ -202,6 +201,8 @@ private void testClientRequestIdForStatusRetry(int status, TracingContext tracingContext = Mockito.mock(TracingContext.class); Mockito.doNothing().when(tracingContext).setRetryCount(nullable(int.class)); + Mockito.doReturn(tracingContext) + .when(abfsRestOperation).createNewTracingContext(any()); int[] count = new int[1]; count[0] = 0; @@ -225,7 +226,7 @@ private void testClientRequestIdForTimeoutRetry(Exception[] exceptions, AbfsClient abfsClient = Mockito.mock(AbfsClient.class); ExponentialRetryPolicy retryPolicy = Mockito.mock( ExponentialRetryPolicy.class); - addMockBehaviourToAbfsClient(abfsClient, retryPolicy); + addGeneralMockBehaviourToAbfsClient(abfsClient, retryPolicy); AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation( @@ -237,7 +238,7 @@ private void testClientRequestIdForTimeoutRetry(Exception[] exceptions, )); AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class); - addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation); + addGeneralMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation); Stubber stubber = Mockito.doThrow(exceptions[0]); for (int iteration = 1; iteration < len; iteration++) { @@ -253,6 +254,7 @@ private void testClientRequestIdForTimeoutRetry(Exception[] exceptions, TracingContext tracingContext = Mockito.mock(TracingContext.class); Mockito.doNothing().when(tracingContext).setRetryCount(nullable(int.class)); + Mockito.doReturn(tracingContext).when(abfsRestOperation).createNewTracingContext(any()); int[] count = new int[1]; count[0] = 0; @@ -268,35 +270,4 @@ private void testClientRequestIdForTimeoutRetry(Exception[] exceptions, abfsRestOperation.execute(tracingContext); Assertions.assertThat(count[0]).isEqualTo(len + 1); } - - private void addMockBehaviourToRestOpAndHttpOp(final AbfsRestOperation abfsRestOperation, - final AbfsHttpOperation httpOperation) throws IOException { - HttpURLConnection httpURLConnection = Mockito.mock(HttpURLConnection.class); - Mockito.doNothing() - .when(httpURLConnection) - .setRequestProperty(nullable(String.class), nullable(String.class)); - Mockito.doReturn(httpURLConnection).when(httpOperation).getConnection(); - Mockito.doReturn("").when(abfsRestOperation).getClientLatency(); - Mockito.doReturn(httpOperation).when(abfsRestOperation).createHttpOperation(); - } - - private void addMockBehaviourToAbfsClient(final AbfsClient abfsClient, - final ExponentialRetryPolicy retryPolicy) throws IOException { - Mockito.doReturn(OAuth).when(abfsClient).getAuthType(); - Mockito.doReturn("").when(abfsClient).getAccessToken(); - AbfsThrottlingIntercept intercept = Mockito.mock( - AbfsThrottlingIntercept.class); - Mockito.doReturn(intercept).when(abfsClient).getIntercept(); - Mockito.doNothing() - .when(intercept) - .sendingRequest(any(), nullable(AbfsCounters.class)); - Mockito.doNothing().when(intercept).updateMetrics(any(), any()); - - Mockito.doReturn(retryPolicy).when(abfsClient).getRetryPolicy(); - Mockito.doReturn(true) - .when(retryPolicy) - .shouldRetry(nullable(Integer.class), nullable(Integer.class)); - Mockito.doReturn(false).when(retryPolicy).shouldRetry(1, HTTP_OK); - Mockito.doReturn(false).when(retryPolicy).shouldRetry(2, HTTP_OK); - } }