Skip to content

Commit

Permalink
Resolved Commentss
Browse files Browse the repository at this point in the history
  • Loading branch information
@anujmodi2021
anujmodi2021 committed Oct 15, 2024
1 parent 7ba0a41 commit 0964df6
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 20 deletions.
20 changes: 12 additions & 8 deletions ...op-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public void testCustomTokenFetchRetryCount() throws Exception {

public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception {
AzureBlobFileSystem fs = this.getFileSystem();
Configuration config = getEntries(numOfRetries);
Configuration config = getCustomAuthConfiguration(numOfRetries);
try (final AzureBlobFileSystem fs1 =
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
config)) {
Expand All @@ -167,7 +167,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce

@Test
public void testAuthFailException() throws Exception {
Configuration config = getEntries(0);
Configuration config = getCustomAuthConfiguration(0);
final AzureBlobFileSystem fs = getFileSystem(config);
AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> {
fs.getFileStatus(new Path("/"));
Expand All @@ -184,19 +184,23 @@ public void testAuthFailException() throws Exception {
.contains("Auth failure: ");
}

private Configuration getEntries(final int numOfRetries) {
/**
* Returns a configuration with a custom token provider configured. {@link RetryTestTokenProvider}
* @param numOfRetries Number of retries to be configured for token fetch.
* @return Configuration
*/
private Configuration getCustomAuthConfiguration(final int numOfRetries) {
Configuration config = new Configuration(this.getRawConfiguration());
String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
// Setup to configure custom token provider.
config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom");
config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName),
RETRY_TEST_TOKEN_PROVIDER);
config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(
numOfRetries));
config.setInt(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, numOfRetries);
// Stop filesystem creation as it will lead to calls to store.
config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + "");
config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false);
config.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true));
return config;
}
}
18 changes: 9 additions & 9 deletions ...re/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@ public ITestAzureBlobFileSystemAuthorization() throws Exception {

@Override
public void setup() throws Exception {
boolean isHNSEnabled = this.getConfiguration().getBoolean(
boolean isHNSEnabled = getConfiguration().getBoolean(
TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
Assume.assumeTrue(isHNSEnabled);
loadConfiguredFileSystem();
this.getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
this.getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS");
getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.toString());
super.setup();
}

Expand All @@ -79,9 +79,9 @@ public void testSASTokenProviderInitializeException() throws Exception {
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true");
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_FAIL_INIT, true);
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));

intercept(SASTokenProviderException.class,
()-> {
Expand All @@ -96,9 +96,9 @@ public void testSASTokenProviderEmptySASToken() throws Exception {
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));

testFs.initialize(fs.getUri(), testConfig);
intercept(SASTokenProviderException.class,
Expand All @@ -114,9 +114,9 @@ public void testSASTokenProviderNullSASToken() throws Exception {
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));

testFs.initialize(fs.getUri(), testConfig);
intercept(SASTokenProviderException.class,
Expand Down
5 changes: 2 additions & 3 deletions ...src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.utils.AbfsTestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
Expand Down Expand Up @@ -68,9 +69,7 @@ protected Configuration createConfiguration() {
protected AbstractFSContract createContract(final Configuration conf) {
conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE);
conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE);
// Disabling cache to make sure new configs are picked up.
conf.set("fs.abfss.impl.disable.cache", "true");
conf.set("fs.abfs.impl.disable.cache", "true");
disableFilesystemCaching(conf);
return new AbfsFileSystemContract(conf, isSecure);
}

Expand Down
14 changes: 14 additions & 0 deletions ...p-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;

import org.apache.hadoop.conf.Configuration;
import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -29,6 +30,8 @@
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.services.AuthType;

import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_SCHEME;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_SECURE_SCHEME;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX;

/**
Expand Down Expand Up @@ -82,4 +85,15 @@ public void deleteContainers() throws Throwable {
}
LOG.info("Deleted {} test containers", count);
}

/**
* Turn off FS Caching: use if a filesystem with different options from
* the default is required.
* @param conf configuration to patch
*/
public static void disableFilesystemCaching(Configuration conf) {
// Disabling cache to make sure new configs are picked up.
conf.setBoolean(String.format("fs.%s.impl.disable.cache", ABFS_SCHEME), true);
conf.setBoolean(String.format("fs.%s.impl.disable.cache", ABFS_SECURE_SCHEME), true);
}
}

0 comments on commit 0964df6

Please sign in to comment.