diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index 2999ff57f438e..2ac15488d82b9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -30,12 +30,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -143,19 +143,7 @@ public void testCustomTokenFetchRetryCount() throws Exception { public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception { AzureBlobFileSystem fs = this.getFileSystem(); - - Configuration config = new Configuration(this.getRawConfiguration()); - String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); - // Setup to configure custom token provider. - config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom"); - config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, - RETRY_TEST_TOKEN_PROVIDER); - config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(numOfRetries)); - // Stop filesystem creation as it will lead to calls to store. - config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); - config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( - FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); - + Configuration config = getEntries(numOfRetries); try (final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config)) { @@ -179,17 +167,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce @Test public void testAuthFailException() throws Exception { - Configuration config = new Configuration(getRawConfiguration()); - String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); - // Setup to configure custom token provider - config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom"); - config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, - RETRY_TEST_TOKEN_PROVIDER); - // Stop filesystem creation as it will lead to calls to store. - config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); - config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( - FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); - + Configuration config = getEntries(0); final AzureBlobFileSystem fs = getFileSystem(config); AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> { fs.getFileStatus(new Path("/")); @@ -205,4 +183,20 @@ public void testAuthFailException() throws Exception { .describedAs("Incorrect error message: " + errorDesc) .contains("Auth failure: "); } + + private Configuration getEntries(final int numOfRetries) { + Configuration config = new Configuration(this.getRawConfiguration()); + String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); + // Setup to configure custom token provider. + config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom"); + config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName), + RETRY_TEST_TOKEN_PROVIDER); + config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString( + numOfRetries)); + // Stop filesystem creation as it will lead to calls to store. + config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); + config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( + FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); + return config; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java index c95d2c2ffd032..a7c6b1f76359c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider; @@ -80,7 +79,7 @@ public void testSASTokenProviderInitializeException() throws Exception { final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); - testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true"); // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index 5732ce8cfea70..058678aa4d8cd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -44,6 +44,7 @@ import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -286,12 +287,15 @@ public void testAccountSpecificConfig() throws Exception { this.getAccountName())); String testAccountName = "testAccount.dfs.core.windows.net"; String otherAccountName = "otherAccount.dfs.core.windows.net"; + String dummyAcountKey = "dummyKey"; String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName); String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName); // Set both account specific and account agnostic config for test account rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR); rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR); + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, testAccountName), dummyAcountKey); + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, otherAccountName), dummyAcountKey); // Assert that account specific config takes precedence rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java index 25e5f8e739c21..9ead6d196382e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java @@ -67,9 +67,9 @@ public class ITestExponentialRetryPolicy extends AbstractAbfsIntegrationTest { private static final String TEST_PATH = "/testfile"; private static final double MULTIPLYING_FACTOR = 1.5; private static final int ANALYSIS_PERIOD = 10000; - private static final String dummyAccountName = "dummy.dfs.core.windows.net"; - private static final String dummyAccountName1 = "dummy1.dfs.core.windows.net"; - private static final String dummyAccountKey = "dummyKey"; + private static final String DUMMY_ACCOUNT_NAME = "dummy.dfs.core.windows.net"; + private static final String DUMMY_ACCOUNT_NAME_1 = "dummy1.dfs.core.windows.net"; + private static final String DUMMY_ACCOUNT_KEY = "dummyKey"; public ITestExponentialRetryPolicy() throws Exception { super(); @@ -102,12 +102,14 @@ public void testThrottlingIntercept() throws Exception { final Configuration configuration = new Configuration(); configuration.addResource(TEST_CONFIGURATION_FILE_NAME); configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, false); - configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName, dummyAccountKey); - configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName1, dummyAccountKey); + configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + DUMMY_ACCOUNT_NAME, + DUMMY_ACCOUNT_KEY); + configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + DUMMY_ACCOUNT_NAME_1, + DUMMY_ACCOUNT_KEY); // On disabling throttling AbfsNoOpThrottlingIntercept object is returned AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, - dummyAccountName); + DUMMY_ACCOUNT_NAME); AbfsThrottlingIntercept intercept; AbfsClient abfsClient = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration); intercept = abfsClient.getIntercept(); @@ -119,7 +121,7 @@ public void testThrottlingIntercept() throws Exception { configuration.setBoolean(FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, true); // On enabling throttling AbfsClientThrottlingIntercept object is returned AbfsConfiguration abfsConfiguration1 = new AbfsConfiguration(configuration, - dummyAccountName1); + DUMMY_ACCOUNT_NAME_1); AbfsClient abfsClient1 = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1); intercept = abfsClient1.getIntercept(); Assertions.assertThat(intercept)