Skip to content

Commit 487727a

Browse files
anujmodi2021anujmodi2021
authored
HADOOP-18960: [ABFS] Making Contract tests run in sequential and Other Test Fixes (#7104)
 Contributed by: Anuj Modi
1 parent f7651e2 commit 487727a

File tree

9 files changed

+123
-50
lines changed

9 files changed

+123
-50
lines changed

‎hadoop-tools/hadoop-azure/pom.xml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -662,7 +662,6 @@
662662
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
663663
</systemPropertyVariables>
664664
<includes>
665-
<include>**/azurebfs/contract/ITest*.java</include>
666665
<include>**/azurebfs/ITestAzureBlobFileSystemE2EScale.java</include>
667666
<include>**/azurebfs/ITestAbfsReadWriteAndSeek.java</include>
668667
<include>**/azurebfs/ITestAzureBlobFileSystemListStatus.java</include>
@@ -674,6 +673,33 @@
674673
</includes>
675674
</configuration>
676675
</execution>
676+
<execution>
677+
<id>integration-test-abfs-sequential-contract-classes</id>
678+
<goals>
679+
<goal>integration-test</goal>
680+
<goal>verify</goal>
681+
</goals>
682+
<configuration>
683+
<!--NOTICE: hadoop contract tests methods can not be run in parallel-->
684+
<forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
685+
<trimStackTrace>false</trimStackTrace>
686+
<systemPropertyVariables>
687+
<!-- Tell tests that they are being executed sequentially -->
688+
<test.parallel.execution>false</test.parallel.execution>
689+
<!-- Propagate scale parameters -->
690+
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
691+
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
692+
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
693+
<fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
694+
<fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
695+
<!-- http connection pool size -->
696+
<http.maxConnections>${http.maxConnections}</http.maxConnections>
697+
</systemPropertyVariables>
698+
<includes>
699+
<include>**/azurebfs/contract/ITest*.java</include>
700+
</includes>
701+
</configuration>
702+
</execution>
677703
</executions>
678704
</plugin>
679705
</plugins>

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,9 +400,11 @@ private AzureBlobFileSystem getAzureBlobFileSystem(final Configuration conf) {
400400
AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(
401401
conf);
402402
fileSystemsOpenedInTest.add(fs);
403+
// Default for this config should be true here as FNS Accounts would have failed initialization.
404+
// This is needed to make sure test runs even if test config is missing.
403405
Assertions.assertThat(
404406
getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT,
405-
false))
407+
true))
406408
.describedAs("Encryption tests should run only on namespace enabled account")
407409
.isTrue();
408410
return fs;

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java‎

Lines changed: 27 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,14 @@
3030
import org.apache.hadoop.fs.FileSystem;
3131
import org.apache.hadoop.fs.Path;
3232

33-
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT;
3433
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
34+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT;
3535
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
36+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
3637
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
38+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
3739
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME;
40+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
3841
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
3942

4043
/**
@@ -140,19 +143,8 @@ public void testCustomTokenFetchRetryCount() throws Exception {
140143

141144
public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception {
142145
AzureBlobFileSystem fs = this.getFileSystem();
143-
144-
Configuration config = new Configuration(this.getRawConfiguration());
145-
String accountName = config.get("fs.azure.abfs.account.name");
146-
// Setup to configure custom token provider.
147-
config.set("fs.azure.account.auth.type." + accountName, "Custom");
148-
config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs"
149-
+ ".azurebfs.oauth2.RetryTestTokenProvider");
150-
config.set("fs.azure.custom.token.fetch.retry.count", Integer.toString(numOfRetries));
151-
// Stop filesystem creation as it will lead to calls to store.
152-
config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false");
153-
154-
try (final AzureBlobFileSystem fs1 =
155-
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
146+
Configuration config = getCustomAuthConfiguration(numOfRetries);
147+
try (AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
156148
config)) {
157149
RetryTestTokenProvider retryTestTokenProvider
158150
= RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
@@ -174,18 +166,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
174166

175167
@Test
176168
public void testAuthFailException() throws Exception {
177-
Configuration config = new Configuration(getRawConfiguration());
178-
String accountName = config
179-
.get(FS_AZURE_ABFS_ACCOUNT_NAME);
180-
// Setup to configure custom token provider
181-
config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT
182-
+ accountName, "Custom");
183-
config.set(
184-
FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName,
185-
RETRY_TEST_TOKEN_PROVIDER);
186-
// Stop filesystem creation as it will lead to calls to store.
187-
config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false");
188-
169+
Configuration config = getCustomAuthConfiguration(0);
189170
final AzureBlobFileSystem fs = getFileSystem(config);
190171
AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> {
191172
fs.getFileStatus(new Path("/"));
@@ -201,4 +182,24 @@ public void testAuthFailException() throws Exception {
201182
.describedAs("Incorrect error message: " + errorDesc)
202183
.contains("Auth failure: ");
203184
}
185+
186+
/**
187+
* Returns a configuration with a custom token provider configured. {@link RetryTestTokenProvider}
188+
* @param numOfRetries Number of retries to be configured for token fetch.
189+
* @return Configuration
190+
*/
191+
private Configuration getCustomAuthConfiguration(final int numOfRetries) {
192+
Configuration config = new Configuration(this.getRawConfiguration());
193+
String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME);
194+
// Setup to configure custom token provider.
195+
config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom");
196+
config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName),
197+
RETRY_TEST_TOKEN_PROVIDER);
198+
config.setInt(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, numOfRetries);
199+
// Stop filesystem creation as it will lead to calls to store.
200+
config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false);
201+
config.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean(
202+
FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true));
203+
return config;
204+
}
204205
}

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,16 @@
2828
import org.apache.hadoop.conf.Configuration;
2929

3030
import org.apache.hadoop.fs.Path;
31-
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
3231
import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
3332
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException;
3433
import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider;
3534
import org.apache.hadoop.fs.azurebfs.services.AuthType;
3635
import org.apache.hadoop.fs.permission.FsAction;
3736
import org.apache.hadoop.fs.permission.FsPermission;
3837

38+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
39+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
40+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
3941
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_FAIL_INIT;
4042
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN;
4143
import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry;
@@ -61,12 +63,12 @@ public ITestAzureBlobFileSystemAuthorization() throws Exception {
6163

6264
@Override
6365
public void setup() throws Exception {
64-
boolean isHNSEnabled = this.getConfiguration().getBoolean(
66+
boolean isHNSEnabled = getConfiguration().getBoolean(
6567
TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
6668
Assume.assumeTrue(isHNSEnabled);
6769
loadConfiguredFileSystem();
68-
this.getConfiguration().set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
69-
this.getConfiguration().set(ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS");
70+
getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS);
71+
getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.toString());
7072
super.setup();
7173
}
7274

@@ -75,13 +77,15 @@ public void testSASTokenProviderInitializeException() throws Exception {
7577
final AzureBlobFileSystem fs = this.getFileSystem();
7678

7779
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
78-
Configuration testConfig = this.getConfiguration().getRawConfiguration();
79-
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
80-
testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true");
80+
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
81+
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
82+
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_FAIL_INIT, true);
83+
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
84+
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));
8185

8286
intercept(SASTokenProviderException.class,
8387
()-> {
84-
testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration());
88+
testFs.initialize(fs.getUri(), testConfig);
8589
});
8690
}
8791

@@ -90,12 +94,13 @@ public void testSASTokenProviderEmptySASToken() throws Exception {
9094
final AzureBlobFileSystem fs = this.getFileSystem();
9195

9296
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
93-
Configuration testConfig = this.getConfiguration().getRawConfiguration();
94-
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
95-
testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true");
97+
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
98+
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
99+
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
100+
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
101+
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));
96102

97-
testFs.initialize(fs.getUri(),
98-
this.getConfiguration().getRawConfiguration());
103+
testFs.initialize(fs.getUri(), testConfig);
99104
intercept(SASTokenProviderException.class,
100105
() -> {
101106
testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close();
@@ -107,10 +112,13 @@ public void testSASTokenProviderNullSASToken() throws Exception {
107112
final AzureBlobFileSystem fs = this.getFileSystem();
108113

109114
final AzureBlobFileSystem testFs = new AzureBlobFileSystem();
110-
Configuration testConfig = this.getConfiguration().getRawConfiguration();
111-
testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
115+
Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration());
116+
testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS);
117+
testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true);
118+
// Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check.
119+
testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs));
112120

113-
testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration());
121+
testFs.initialize(fs.getUri(), testConfig);
114122
intercept(SASTokenProviderException.class,
115123
()-> {
116124
testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close();

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@
3737
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN;
3838
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
3939
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
40+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_ID;
41+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET;
42+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID;
43+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID;
4044
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
4145

4246
/**
@@ -62,8 +66,8 @@ public ITestAzureBlobFileSystemChooseSAS() throws Exception {
6266

6367
@Override
6468
public void setup() throws Exception {
65-
createFilesystemWithTestFileForSASTests(new Path(TEST_PATH));
6669
super.setup();
70+
createFilesystemWithTestFileForSASTests(new Path(TEST_PATH));
6771
generateAccountSAS();
6872
}
6973

@@ -95,6 +99,11 @@ public void testBothProviderFixedTokenConfigured() throws Exception {
9599
// Configuring a SASTokenProvider class which provides a user delegation SAS.
96100
testAbfsConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE,
97101
MockDelegationSASTokenProvider.class.getName());
102+
// Make sure test configs required by MockDelegationSASTokenProvider are set.
103+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_ID);
104+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SECRET);
105+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID);
106+
assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID);
98107

99108
// configuring the Fixed SAS token which is an Account SAS.
100109
testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
4545
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
4646
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty;
47+
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY;
4748
import static org.mockito.ArgumentMatchers.any;
4849
import static org.mockito.ArgumentMatchers.anyString;
4950
import static org.mockito.Mockito.doReturn;
@@ -286,12 +287,15 @@ public void testAccountSpecificConfig() throws Exception {
286287
this.getAccountName()));
287288
String testAccountName = "testAccount.dfs.core.windows.net";
288289
String otherAccountName = "otherAccount.dfs.core.windows.net";
290+
String dummyAcountKey = "dummyKey";
289291
String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName);
290292
String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName);
291293

292294
// Set both account specific and account agnostic config for test account
293295
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR);
294296
rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR);
297+
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, testAccountName), dummyAcountKey);
298+
rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, otherAccountName), dummyAcountKey);
295299
// Assert that account specific config takes precedence
296300
rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri);
297301
assertFileSystemInitWithExpectedHNSSettings(rawConfig, false);

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE;
3636
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
3737
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
38+
import static org.apache.hadoop.fs.azurebfs.utils.AbfsTestUtils.disableFilesystemCaching;
3839
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
3940
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
4041
import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
@@ -68,6 +69,7 @@ protected Configuration createConfiguration() {
6869
protected AbstractFSContract createContract(final Configuration conf) {
6970
conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE);
7071
conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE);
72+
disableFilesystemCaching(conf);
7173
return new AbfsFileSystemContract(conf, isSecure);
7274
}
7375

0 commit comments

Comments
 (0)