Merge pull request apache#51 from ABFSDriver/prefixFix

DFS prefix changes
saxenapranav · May 12, 2023 · 18b9e1f · 18b9e1f
2 parents 28f30c4 + 0e76fa3
commit 18b9e1f
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 13 deletions.
diff --git a/...p-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/...p-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -328,8 +328,7 @@ private static String convertTestUrls(
     }
 
     if (data != null) {
-      data = data.replace("." + fromDnsPrefix + ".",
-          "." + toDnsPrefix + ".");
+      data = data.replace(fromDnsPrefix , toDnsPrefix);
     }
     return data;
   }

diff --git a/...oop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java b/...oop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java
@@ -29,14 +29,14 @@
 public final class FileSystemUriSchemes {
   public static final String ABFS_SCHEME = "abfs";
   public static final String ABFS_SECURE_SCHEME = "abfss";
-  public static final String ABFS_DNS_PREFIX = "dfs";
+  public static final String ABFS_DNS_PREFIX = ".dfs.";
 
   public static final String HTTP_SCHEME = "http";
   public static final String HTTPS_SCHEME = "https";
 
   public static final String WASB_SCHEME = "wasb";
   public static final String WASB_SECURE_SCHEME = "wasbs";
-  public static final String WASB_DNS_PREFIX = "blob";
+  public static final String WASB_DNS_PREFIX = ".blob.";
 
   private FileSystemUriSchemes() {}
-}
+}
diff --git a/...hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/...hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -421,8 +421,7 @@ private static String convertTestUrls(
     }
 
     if (data != null) {
-      data = data.replace("." + fromDnsPrefix + ".",
-          "." + toDnsPrefix + ".");
+      data = data.replace(fromDnsPrefix, toDnsPrefix);
     }
     return data;
   }

diff --git a/...zure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java b/...zure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java
@@ -22,6 +22,7 @@
 import java.lang.reflect.Field;
 import java.util.List;
 
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
 import org.junit.Assume;
 import org.junit.Test;
@@ -44,6 +45,8 @@
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID;
@@ -84,16 +87,23 @@ private void setTestUserFs() throws Exception {
     if (this.testUserFs != null) {
       return;
     }
+    AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+    String accountName = this.getAccountName();
+    if (abfsStore.getPrefixMode() == PrefixMode.BLOB) {
+      if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) {
+        accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+      }
+    }
     checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT
-        + "." + getAccountName());
+        + "." + accountName);
     Configuration conf = getRawConfiguration();
     setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID,
         FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID);
     setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_SECRET,
         FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET);
     conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name());
     conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "."
-        + getAccountName(), ClientCredsTokenProvider.class.getName());
+        + accountName, ClientCredsTokenProvider.class.getName());
     conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
         false);
     this.testUserFs = FileSystem.newInstance(getRawConfiguration());

diff --git a/...oop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/...oop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -74,6 +74,8 @@
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_BLOB_MKDIR_OVERWRITE;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
 import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_META_HDI_ISFOLDER;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -756,9 +758,16 @@ public void testMkdirOnExistingFileWithImplicitParentDir() throws Exception {
    */
   @Test
   public void testCPKOverBlob() throws Exception {
-    Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+    AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+    Assume.assumeTrue(abfsStore.getPrefixMode() == PrefixMode.BLOB);
     Configuration configuration = Mockito.spy(getRawConfiguration());
-    configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + getAccountName(), "abcd");
+    String accountName = getAccountName();
+    if (abfsStore.getPrefixMode() == PrefixMode.BLOB) {
+      if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) {
+        accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+      }
+    }
+    configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, "abcd");
     intercept(InvalidConfigurationValueException.class, () -> (AzureBlobFileSystem) FileSystem.newInstance(configuration));
   }
 
@@ -767,9 +776,16 @@ public void testCPKOverBlob() throws Exception {
    */
   @Test
   public void testCPKOverBlobEmptyKey() throws Exception {
-    Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB);
+    AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
+    Assume.assumeTrue(abfsStore.getPrefixMode() == PrefixMode.BLOB);
     Configuration configuration = Mockito.spy(getRawConfiguration());
-    configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + getAccountName(), "");
+    String accountName = getAccountName();
+    if (abfsStore.getPrefixMode() == PrefixMode.BLOB) {
+      if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) {
+        accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+      }
+    }
+    configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, "");
     intercept(InvalidConfigurationValueException.class, () -> (AzureBlobFileSystem) FileSystem.newInstance(configuration));
   }
 

diff --git a/...op-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java b/...op-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.fs.azurebfs;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.services.PrefixMode;
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -28,6 +29,8 @@
 
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 public class ITestSharedKeyAuth extends AbstractAbfsIntegrationTest {
@@ -42,7 +45,13 @@ public void testWithWrongSharedKey() throws Exception {
     Configuration config = this.getRawConfiguration();
     config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION,
         true);
+    AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
     String accountName = this.getAccountName();
+    if (abfsStore.getPrefixMode() == PrefixMode.BLOB) {
+      if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) {
+        accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX);
+      }
+    }
     String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName;
     // a wrong sharedKey
     String secret = "XjUjsGherkDpljuyThd7RpljhR6uhsFjhlxRpmhgD12lnj7lhfRn8kgPt5"