Skip to content

HADOOP-18861 ABFS: Fix failing tests for CPK #5979

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 9, 2023
Merged

HADOOP-18861 ABFS: Fix failing tests for CPK #5979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9b3d9d4
Fix failing tests for CPK
anmolanmol1234 Aug 23, 2023
d599fa2
Fix auth failure
anmolanmol1234 Aug 23, 2023
134c561
PR commnets
anmolanmol1234 Aug 23, 2023
f06eefa
Merge branch 'apache:trunk' into HADOOP-18861
anmolanmol1234 Sep 4, 2023
318ec50
Common method
anmolanmol1234 Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 20 additions & 26 deletions ...ls/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,14 @@ public ITestCustomerProvidedKey() throws Exception {
.getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
}

private String getFileName() throws IOException {
return path("/" + methodName.getMethodName()).toUri().getPath();
}

@Test
public void testReadWithCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(true);
String fileName = path("/" + methodName.getMethodName()).toString();
String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE);

AbfsClient abfsClient = fs.getAbfsClient();
Expand Down Expand Up @@ -162,7 +166,7 @@ public void testReadWithCPK() throws Exception {
@Test
public void testReadWithoutCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(false);
String fileName = path("/" + methodName.getMethodName()).toString();
String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE);

AbfsClient abfsClient = fs.getAbfsClient();
Expand Down Expand Up @@ -201,7 +205,7 @@ public void testReadWithoutCPK() throws Exception {
@Test
public void testAppendWithCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(true);
final String fileName = path("/" + methodName.getMethodName()).toString();
final String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE);

// Trying to append with correct CPK headers
Expand Down Expand Up @@ -246,7 +250,7 @@ public void testAppendWithCPK() throws Exception {
@Test
public void testAppendWithoutCPK() throws Exception {
final AzureBlobFileSystem fs = getAbfs(false);
final String fileName = path("/" + methodName.getMethodName()).toString();
final String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE);

// Trying to append without CPK headers
Expand Down Expand Up @@ -282,7 +286,7 @@ public void testAppendWithoutCPK() throws Exception {
@Test
public void testSetGetXAttr() throws Exception {
final AzureBlobFileSystem fs = getAbfs(true);
final String fileName = path(methodName.getMethodName()).toString();
final String fileName = getFileName();
createFileAndGetContent(fs, fileName, FILE_SIZE);

String valSent = "testValue";
Expand Down Expand Up @@ -416,7 +420,7 @@ public void testListPathWithoutCPK() throws Exception {
private void testListPath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final Path testPath = path("/" + methodName.getMethodName());
String testDirName = testPath.toString();
String testDirName = testPath.toUri().getPath();
fs.mkdirs(testPath);
createFileAndGetContent(fs, testDirName + "/aaa", FILE_SIZE);
createFileAndGetContent(fs, testDirName + "/bbb", FILE_SIZE);
Expand Down Expand Up @@ -475,8 +479,7 @@ public void testCreatePathWithoutCPK() throws Exception {

private void testCreatePath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
createFileAndGetContent(fs, testFileName, FILE_SIZE);

AbfsClient abfsClient = fs.getAbfsClient();
Expand Down Expand Up @@ -519,8 +522,7 @@ public void testRenamePathWithoutCPK() throws Exception {

private void testRenamePath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
createFileAndGetContent(fs, testFileName, FILE_SIZE);

FileStatus fileStatusBeforeRename = fs
Expand Down Expand Up @@ -556,8 +558,7 @@ public void testFlushWithoutCPK() throws Exception {

private void testFlush(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
fs.create(new Path(testFileName)).close();
AbfsClient abfsClient = fs.getAbfsClient();
String expectedCPKSha = getCPKSha(fs);
Expand Down Expand Up @@ -617,8 +618,7 @@ public void testSetPathPropertiesWithoutCPK() throws Exception {

private void testSetPathProperties(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
createFileAndGetContent(fs, testFileName, FILE_SIZE);

AbfsClient abfsClient = fs.getAbfsClient();
Expand Down Expand Up @@ -648,8 +648,7 @@ public void testGetPathStatusFileWithoutCPK() throws Exception {

private void testGetPathStatusFile(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
createFileAndGetContent(fs, testFileName, FILE_SIZE);

AbfsClient abfsClient = fs.getAbfsClient();
Expand Down Expand Up @@ -686,8 +685,7 @@ public void testDeletePathWithoutCPK() throws Exception {

private void testDeletePath(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
createFileAndGetContent(fs, testFileName, FILE_SIZE);

FileStatus[] listStatuses = fs.listStatus(new Path(testFileName));
Expand Down Expand Up @@ -717,8 +715,7 @@ public void testSetPermissionWithoutCPK() throws Exception {

private void testSetPermission(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
Assume.assumeTrue(fs.getIsNamespaceEnabled(getTestTracingContext(fs, false)));
createFileAndGetContent(fs, testFileName, FILE_SIZE);
AbfsClient abfsClient = fs.getAbfsClient();
Expand All @@ -743,8 +740,7 @@ public void testSetAclWithoutCPK() throws Exception {

private void testSetAcl(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
TracingContext tracingContext = getTestTracingContext(fs, false);
Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext));
createFileAndGetContent(fs, testFileName, FILE_SIZE);
Expand Down Expand Up @@ -773,8 +769,7 @@ public void testGetAclWithoutCPK() throws Exception {

private void testGetAcl(final boolean isWithCPK) throws Exception {
final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
TracingContext tracingContext = getTestTracingContext(fs, false);
Assume.assumeTrue(fs.getIsNamespaceEnabled(tracingContext));
createFileAndGetContent(fs, testFileName, FILE_SIZE);
Expand Down Expand Up @@ -804,8 +799,7 @@ private void testCheckAccess(final boolean isWithCPK) throws Exception {
getAuthType() == AuthType.OAuth);

final AzureBlobFileSystem fs = getAbfs(isWithCPK);
final String testFileName = path("/" + methodName.getMethodName())
.toString();
final String testFileName = getFileName();
fs.create(new Path(testFileName)).close();
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
Expand Down
1 change: 1 addition & 0 deletions ...ls/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,7 @@ public void testConfigPropNotFound() throws Throwable {

for (String key : CONFIG_KEYS) {
setAuthConfig(abfsConf, true, AuthType.OAuth);
abfsConf.unset(key);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please share in description why we are unsetting it. Does this get ignored somehow in current tests.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a fix for this test failure :- TestAccountConfiguration.testConfigPropNotFound:386->testMissingConfigKey:399 Expected a org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException to be thrown, but got the result: : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"

abfsConf.unset(key + "." + accountName);
testMissingConfigKey(abfsConf, key);
}
Expand Down