From f9dacde7a2a2cf2ae922630c6e2b6b51fe8d6e65 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 5 Nov 2021 14:21:15 +0000 Subject: [PATCH] Revert "HADOOP-17873. ABFS: Fix transient failures in ITestAbfsStreamStatistics and ITestAbfsRestOperationException (#3341)" This reverts commit 82658a22d6e443fcf443be01b36763cdbab96ce9. --- hadoop-tools/hadoop-azure/pom.xml | 2 -- .../oauth2/CustomTokenProviderAdapter.java | 6 ---- .../fs/azurebfs/services/AbfsClient.java | 5 ---- .../azurebfs/AbstractAbfsIntegrationTest.java | 5 ---- .../ITestAbfsRestOperationException.java | 13 ++++----- .../oauth2/RetryTestTokenProvider.java | 29 +++++-------------- .../fs/azurebfs/services/TestAbfsClient.java | 4 --- 7 files changed, 13 insertions(+), 51 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 075716e91e64ce..1896e15d27018c 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -555,7 +555,6 @@ **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java - **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/services/ITestReadBufferManager.java @@ -598,7 +597,6 @@ **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/services/ITestReadBufferManager.java - **/azurebfs/ITestAbfsStreamStatistics*.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java index 6ae0962fbfd82d..14914101e5cca5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.net.URI; -import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,9 +138,4 @@ public String getUserAgentSuffix() { String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, ""); return suffix != null ? suffix : ""; } - - @VisibleForTesting - protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() { - return adaptee; - } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 643cb66fa2b393..17d5c03a819ffe 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1223,9 +1223,4 @@ public ListenableFuture submit(Runnable runnable) { public void addCallback(ListenableFuture future, FutureCallback callback) { Futures.addCallback(future, callback, executorService); } - - @VisibleForTesting - protected AccessTokenProvider getTokenProvider() { - return tokenProvider; - } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index e2c274c3b353b9..56d553819feaed 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -37,12 +37,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; -import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; @@ -243,9 +241,6 @@ public Hashtable call() throws Exception { } } - public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) { - return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient()); - } public void loadConfiguredFileSystem() throws Exception { // disable auto-creation of filesystem diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index b1705cad015686..a71e7bc815f759 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -111,10 +111,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config); - RetryTestTokenProvider retryTestTokenProvider - = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( - getAccessTokenProvider(fs1)); - retryTestTokenProvider.resetStatusToFirstTokenFetch(); + RetryTestTokenProvider.ResetStatusToFirstTokenFetch(); intercept(Exception.class, ()-> { @@ -122,10 +119,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce }); // Number of retries done should be as configured - Assert.assertEquals( - "Number of token fetch retries done does not match with fs.azure" - + ".custom.token.fetch.retry.count configured", numOfRetries, - retryTestTokenProvider.getRetryCount()); + Assert.assertTrue( + "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount + + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries + + ")", RetryTestTokenProvider.reTryCount == numOfRetries); } @Test diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 7427add29086c6..3566ebbaaaa2a5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -30,12 +30,12 @@ */ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { - private static final Logger LOG = LoggerFactory.getLogger( - RetryTestTokenProvider.class); - // Need to track first token fetch otherwise will get counted as a retry too. - private boolean isThisFirstTokenFetch = true; - private int retryCount = 0; + private static boolean isThisFirstTokenFetch = true; + public static int reTryCount = 0; + + private static final Logger LOG = LoggerFactory + .getLogger(RetryTestTokenProvider.class); @Override public void initialize(Configuration configuration, String accountName) @@ -43,13 +43,9 @@ public void initialize(Configuration configuration, String accountName) } - /** - * Clear earlier retry details and reset RetryTestTokenProvider instance to - * state of first access token fetch call. - */ - public void resetStatusToFirstTokenFetch() { + public static void ResetStatusToFirstTokenFetch() { isThisFirstTokenFetch = true; - retryCount = 0; + reTryCount = 0; } @Override @@ -57,7 +53,7 @@ public String getAccessToken() throws IOException { if (isThisFirstTokenFetch) { isThisFirstTokenFetch = false; } else { - retryCount++; + reTryCount++; } LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens"); @@ -68,13 +64,4 @@ public String getAccessToken() throws IOException { public Date getExpiryTime() { return new Date(); } - - public static RetryTestTokenProvider getCurrentRetryTestProviderInstance( - AccessTokenProvider customTokenProvider) { - return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee(); - } - - public int getRetryCount() { - return retryCount; - } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 0a1dca7e7d8d74..a725bf3175a5c1 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -395,8 +395,4 @@ public static AbfsRestOperation getRestOp(AbfsRestOperationType type, url, requestHeaders); } - - public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) { - return client.getTokenProvider(); - } }