Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HADOOP-18960: [ABFS] Making Contract tests run in sequential and Other Test Fixes #7104

Open
wants to merge 8 commits into
base: trunk
Choose a base branch
from
+123 −50
Open

HADOOP-18960: [ABFS] Making Contract tests run in sequential and Other Test Fixes #7104

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
19e9aa2
Fixing Failing Tests
anujmodi2021 Oct 9, 2024
acca3fb
Missed Chages
anujmodi2021 Oct 9, 2024
d755255
Follow Up Changes
anujmodi2021 Oct 9, 2024
7ba0a41
Merge branch 'trunk' into testFixes
anujmodi2021 Oct 15, 2024
0964df6
Resolved Commentss
anujmodi2021 Oct 15, 2024
1630291
Yetus Rerun
anujmodi2021 Oct 16, 2024
9419e36
Merge branch 'trunk' into testFixes
anujmodi2021 Oct 18, 2024
0dc4ce0
Fixed Import Ordering
anujmodi2021 Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions ...op-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,8 @@ public void testCustomTokenFetchRetryCount() throws Exception {

public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception {
AzureBlobFileSystem fs = this.getFileSystem();
Configuration config = getEntries(numOfRetries);
try (final AzureBlobFileSystem fs1 =
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
Configuration config = getCustomAuthConfiguration(numOfRetries);
try (AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
config)) {
RetryTestTokenProvider retryTestTokenProvider
= RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
Expand All @@ -167,7 +166,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce

@Test
public void testAuthFailException() throws Exception {
Configuration config = getEntries(0);
Configuration config = getCustomAuthConfiguration(0);
final AzureBlobFileSystem fs = getFileSystem(config);
AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> {
fs.getFileStatus(new Path("/"));
Expand All @@ -184,19 +183,23 @@ public void testAuthFailException() throws Exception {
.contains("Auth failure: ");
}

private Configuration getEntries(final int numOfRetries) {
/**
* Returns a configuration with a custom token provider configured. {@link RetryTestTokenProvider}
* @param numOfRetries Number of retries to be configured for token fetch.
* @return Configuration
*/
private Configuration getCustomAuthConfiguration(final int numOfRetries) {
Configuration config = new Configuration(this.getRawConfiguration());
String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
// Setup to configure custom token provider.
config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom");
config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName),
RETRY_TEST_TOKEN_PROVIDER);
config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(
numOfRetries));
config.setInt(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, numOfRetries);
// Stop filesystem creation as it will lead to calls to store.
config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + "");
config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false);
config.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true));
return config;
}
}
18 changes: 9 additions & 9 deletions ...re/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@ public ITestAzureBlobFileSystemAuthorization() throws Exception {

@Override
public void setup() throws Exception {
boolean isHNSEnabled = this.getConfiguration().getBoolean(
boolean isHNSEnabled = getConfiguration().getBoolean(
TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
Assume.assumeTrue(isHNSEnabled);
loadConfiguredFileSystem();
this.getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
this.getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS");
getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.toString());
super.setup();
}

Expand All @@ -79,9 +79,9 @@ public void testSASTokenProviderInitializeException() throws Exception {
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true");
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_FAIL_INIT, true);
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));

intercept(SASTokenProviderException.class,
()-> {
Expand All @@ -96,9 +96,9 @@ public void testSASTokenProviderEmptySASToken() throws Exception {
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));

testFs.initialize(fs.getUri(), testConfig);
intercept(SASTokenProviderException.class,
Expand All @@ -114,9 +114,9 @@ public void testSASTokenProviderNullSASToken() throws Exception {
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + "");
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));

testFs.initialize(fs.getUri(), testConfig);
intercept(SASTokenProviderException.class,
Expand Down
5 changes: 2 additions & 3 deletions ...src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.utils.AbfsTestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
Expand Down Expand Up @@ -68,9 +69,7 @@ protected Configuration createConfiguration() {
protected AbstractFSContract createContract(final Configuration conf) {
conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE);
conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE);
// Disabling cache to make sure new configs are picked up.
conf.set("fs.abfss.impl.disable.cache", "true");
conf.set("fs.abfs.impl.disable.cache", "true");
disableFilesystemCaching(conf);
return new AbfsFileSystemContract(conf, isSecure);
}

Expand Down
14 changes: 14 additions & 0 deletions ...p-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;

import org.apache.hadoop.conf.Configuration;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: import ordering

import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -29,6 +30,8 @@
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.services.AuthType;

import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_SCHEME;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_SECURE_SCHEME;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX;

/**
Expand Down Expand Up @@ -82,4 +85,15 @@ public void deleteContainers() throws Throwable {
}
LOG.info("Deleted {} test containers", count);
}

/**
* Turn off FS Caching: use if a filesystem with different options from
* the default is required.
* @param conf configuration to patch
*/
public static void disableFilesystemCaching(Configuration conf) {
// Disabling cache to make sure new configs are picked up.
conf.setBoolean(String.format("fs.%s.impl.disable.cache", ABFS_SCHEME), true);
conf.setBoolean(String.format("fs.%s.impl.disable.cache", ABFS_SECURE_SCHEME), true);
}
}
Loading