Follow Up Changes

apache · Oct 9, 2024 · d755255 · d755255
1 parent acca3fb
commit d755255
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 34 deletions.
diff --git a/...op-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/...op-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -30,12 +30,12 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -143,19 +143,7 @@ public void testCustomTokenFetchRetryCount() throws Exception {
 
   public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception {
     AzureBlobFileSystem fs = this.getFileSystem();
-
-    Configuration config = new Configuration(this.getRawConfiguration());
-    String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
-    // Setup to configure custom token provider.
-    config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom");
-    config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName,
-        RETRY_TEST_TOKEN_PROVIDER);
-    config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(numOfRetries));
-    // Stop filesystem creation as it will lead to calls to store.
-    config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
-    config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
-        FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + "");
-
+    Configuration config = getEntries(numOfRetries);
     try (final AzureBlobFileSystem fs1 =
         (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
         config)) {
@@ -179,17 +167,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
 
   @Test
   public void testAuthFailException() throws Exception {
-    Configuration config = new Configuration(getRawConfiguration());
-    String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
-    // Setup to configure custom token provider
-    config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom");
-    config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName,
-        RETRY_TEST_TOKEN_PROVIDER);
-    // Stop filesystem creation as it will lead to calls to store.
-    config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
-    config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
-        FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + "");
-
+    Configuration config = getEntries(0);
     final AzureBlobFileSystem fs = getFileSystem(config);
     AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> {
       fs.getFileStatus(new Path("/"));
@@ -205,4 +183,20 @@ public void testAuthFailException() throws Exception {
         .describedAs("Incorrect error message: " + errorDesc)
         .contains("Auth failure: ");
   }
+
+  private Configuration getEntries(final int numOfRetries) {
+    Configuration config = new Configuration(this.getRawConfiguration());
+    String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
+    // Setup to configure custom token provider.
+    config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom");
+    config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName),
+        RETRY_TEST_TOKEN_PROVIDER);
+    config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(
+        numOfRetries));
+    // Stop filesystem creation as it will lead to calls to store.
+    config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
+    config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
+        FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + "");
+    return config;
+  }
 }
diff --git a/...re/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/...re/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.conf.Configuration;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException;
 import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider;
@@ -80,7 +79,7 @@ public void testSASTokenProviderInitializeException() throws Exception {
     final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
     Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
     testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
-    testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
+    testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true");
     // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
     testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
 

diff --git a/...ls/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/...ls/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -44,6 +44,7 @@
 import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doReturn;
@@ -286,12 +287,15 @@ public void testAccountSpecificConfig() throws Exception {
         this.getAccountName()));
     String testAccountName = "testAccount.dfs.core.windows.net";
     String otherAccountName = "otherAccount.dfs.core.windows.net";
+    String dummyAcountKey = "dummyKey";
     String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
     String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
 
     // Set both account specific and account agnostic config for test account
     rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
     rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
+    rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, testAccountName), dummyAcountKey);
+    rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, otherAccountName), dummyAcountKey);
     // Assert that account specific config takes precedence
     rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
     assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);

diff --git a/...ure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java b/...ure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java
@@ -67,9 +67,9 @@ public class ITestExponentialRetryPolicy extends AbstractAbfsIntegrationTest {
   private static final String TEST_PATH = "/testfile";
   private static final double MULTIPLYING_FACTOR = 1.5;
   private static final int ANALYSIS_PERIOD = 10000;
-  private static final String dummyAccountName = "dummy.dfs.core.windows.net";
-  private static final String dummyAccountName1 = "dummy1.dfs.core.windows.net";
-  private static final String dummyAccountKey = "dummyKey";
+  private static final String DUMMY_ACCOUNT_NAME = "dummy.dfs.core.windows.net";
+  private static final String DUMMY_ACCOUNT_NAME_1 = "dummy1.dfs.core.windows.net";
+  private static final String DUMMY_ACCOUNT_KEY = "dummyKey";
 
   public ITestExponentialRetryPolicy() throws Exception {
     super();
@@ -102,12 +102,14 @@ public void testThrottlingIntercept() throws Exception {
     final Configuration configuration = new Configuration();
     configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
     configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, false);
-    configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName, dummyAccountKey);
-    configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName1, dummyAccountKey);
+    configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + DUMMY_ACCOUNT_NAME,
+        DUMMY_ACCOUNT_KEY);
+    configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + DUMMY_ACCOUNT_NAME_1,
+        DUMMY_ACCOUNT_KEY);
 
     // On disabling throttling AbfsNoOpThrottlingIntercept object is returned
     AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
-        dummyAccountName);
+        DUMMY_ACCOUNT_NAME);
     AbfsThrottlingIntercept intercept;
     AbfsClient abfsClient = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration);
     intercept = abfsClient.getIntercept();
@@ -119,7 +121,7 @@ public void testThrottlingIntercept() throws Exception {
     configuration.setBoolean(FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, true);
     // On enabling throttling AbfsClientThrottlingIntercept object is returned
     AbfsConfiguration abfsConfiguration1 = new AbfsConfiguration(configuration,
-        dummyAccountName1);
+        DUMMY_ACCOUNT_NAME_1);
     AbfsClient abfsClient1 = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1);
     intercept = abfsClient1.getIntercept();
     Assertions.assertThat(intercept)