From 317e55c714a21d87639c08a9c92f8cec9825143f Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 11 May 2023 02:04:11 -0700 Subject: [PATCH 1/2] DFS prefix changes --- .../fs/azurebfs/AzureBlobFileSystem.java | 3 +-- .../constants/FileSystemUriSchemes.java | 4 ++-- .../azurebfs/AbstractAbfsIntegrationTest.java | 3 +-- .../ITestAzureBlobFileSystemCheckAccess.java | 14 +++++++++++-- .../ITestAzureBlobFileSystemCreate.java | 20 +++++++++++++++++-- .../fs/azurebfs/ITestSharedKeyAuth.java | 9 +++++++++ 6 files changed, 43 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 37a449f6829d1..10b45ffc0d121 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -328,8 +328,7 @@ private static String convertTestUrls( } if (data != null) { - data = data.replace("." + fromDnsPrefix + ".", - "." + toDnsPrefix + "."); + data = data.replace(fromDnsPrefix , toDnsPrefix); } return data; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java index c7a0cdad605ab..3219d6235cd70 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java @@ -29,14 +29,14 @@ public final class FileSystemUriSchemes { public static final String ABFS_SCHEME = "abfs"; public static final String ABFS_SECURE_SCHEME = "abfss"; - public static final String ABFS_DNS_PREFIX = "dfs"; + public static final String ABFS_DNS_PREFIX = ".dfs."; public static final String HTTP_SCHEME = "http"; public static final String HTTPS_SCHEME = "https"; public static final String WASB_SCHEME = "wasb"; public static final String WASB_SECURE_SCHEME = "wasbs"; - public static final String WASB_DNS_PREFIX = "blob"; + public static final String WASB_DNS_PREFIX = ".blob."; private FileSystemUriSchemes() {} } \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 7dbd9a4c95a42..932b7638a4848 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -421,8 +421,7 @@ private static String convertTestUrls( } if (data != null) { - data = data.replace("." + fromDnsPrefix + ".", - "." + toDnsPrefix + "."); + data = data.replace(fromDnsPrefix, toDnsPrefix); } return data; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java index 5b6e29aa0d7bd..72fa10e0d8bfc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCheckAccess.java @@ -22,6 +22,7 @@ import java.lang.reflect.Field; import java.util.List; +import org.apache.hadoop.fs.azurebfs.services.PrefixMode; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.junit.Assume; import org.junit.Test; @@ -44,6 +45,8 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_BLOB_FS_CHECKACCESS_TEST_USER_GUID; @@ -84,8 +87,15 @@ private void setTestUserFs() throws Exception { if (this.testUserFs != null) { return; } + AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); + String accountName = this.getAccountName(); + if (abfsStore.getPrefixMode() == PrefixMode.BLOB) { + if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) { + accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX); + } + } checkIfConfigIsSet(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT - + "." + getAccountName()); + + "." + accountName); Configuration conf = getRawConfiguration(); setTestFsConf(FS_AZURE_BLOB_FS_CLIENT_ID, FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_ID); @@ -93,7 +103,7 @@ private void setTestUserFs() throws Exception { FS_AZURE_BLOB_FS_CHECKACCESS_TEST_CLIENT_SECRET); conf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth.name()); conf.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + "." - + getAccountName(), ClientCredsTokenProvider.class.getName()); + + accountName, ClientCredsTokenProvider.class.getName()); conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); this.testUserFs = FileSystem.newInstance(getRawConfiguration()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index 41d4d1d759d0b..8fd619aa03fa8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -74,6 +74,8 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_BLOB_MKDIR_OVERWRITE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_META_HDI_ISFOLDER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -758,7 +760,14 @@ public void testMkdirOnExistingFileWithImplicitParentDir() throws Exception { public void testCPKOverBlob() throws Exception { Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB); Configuration configuration = Mockito.spy(getRawConfiguration()); - configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + getAccountName(), "abcd"); + AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); + String accountName = getAccountName(); + if (abfsStore.getPrefixMode() == PrefixMode.BLOB) { + if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) { + accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX); + } + } + configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, "abcd"); intercept(InvalidConfigurationValueException.class, () -> (AzureBlobFileSystem) FileSystem.newInstance(configuration)); } @@ -769,7 +778,14 @@ public void testCPKOverBlob() throws Exception { public void testCPKOverBlobEmptyKey() throws Exception { Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB); Configuration configuration = Mockito.spy(getRawConfiguration()); - configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + getAccountName(), ""); + AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); + String accountName = getAccountName(); + if (abfsStore.getPrefixMode() == PrefixMode.BLOB) { + if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) { + accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX); + } + } + configuration.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, ""); intercept(InvalidConfigurationValueException.class, () -> (AzureBlobFileSystem) FileSystem.newInstance(configuration)); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java index 7d17ab65cc403..63e1b45ce144c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestSharedKeyAuth.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azurebfs; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.services.PrefixMode; import org.junit.Assume; import org.junit.Test; @@ -28,6 +29,8 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DNS_PREFIX; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.WASB_DNS_PREFIX; import static org.apache.hadoop.test.LambdaTestUtils.intercept; public class ITestSharedKeyAuth extends AbstractAbfsIntegrationTest { @@ -42,7 +45,13 @@ public void testWithWrongSharedKey() throws Exception { Configuration config = this.getRawConfiguration(); config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); + AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); String accountName = this.getAccountName(); + if (abfsStore.getPrefixMode() == PrefixMode.BLOB) { + if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) { + accountName = getAccountName().replace(ABFS_DNS_PREFIX, WASB_DNS_PREFIX); + } + } String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName; // a wrong sharedKey String secret = "XjUjsGherkDpljuyThd7RpljhR6uhsFjhlxRpmhgD12lnj7lhfRn8kgPt5" From 0e76fa384c8091e7ca593b37b4674b41e1c201d1 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 11 May 2023 08:21:50 -0700 Subject: [PATCH 2/2] Addressing PR comments --- .../fs/azurebfs/constants/FileSystemUriSchemes.java | 2 +- .../fs/azurebfs/ITestAzureBlobFileSystemCreate.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java index 3219d6235cd70..573c819fc29f2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemUriSchemes.java @@ -39,4 +39,4 @@ public final class FileSystemUriSchemes { public static final String WASB_DNS_PREFIX = ".blob."; private FileSystemUriSchemes() {} -} \ No newline at end of file +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index 8fd619aa03fa8..dd291649cca2d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -758,9 +758,9 @@ public void testMkdirOnExistingFileWithImplicitParentDir() throws Exception { */ @Test public void testCPKOverBlob() throws Exception { - Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB); - Configuration configuration = Mockito.spy(getRawConfiguration()); AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); + Assume.assumeTrue(abfsStore.getPrefixMode() == PrefixMode.BLOB); + Configuration configuration = Mockito.spy(getRawConfiguration()); String accountName = getAccountName(); if (abfsStore.getPrefixMode() == PrefixMode.BLOB) { if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) { @@ -776,9 +776,9 @@ public void testCPKOverBlob() throws Exception { */ @Test public void testCPKOverBlobEmptyKey() throws Exception { - Assume.assumeTrue(getFileSystem().getAbfsStore().getPrefixMode() == PrefixMode.BLOB); - Configuration configuration = Mockito.spy(getRawConfiguration()); AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); + Assume.assumeTrue(abfsStore.getPrefixMode() == PrefixMode.BLOB); + Configuration configuration = Mockito.spy(getRawConfiguration()); String accountName = getAccountName(); if (abfsStore.getPrefixMode() == PrefixMode.BLOB) { if (abfsStore.getAbfsConfiguration().shouldEnableBlobEndPoint()) {