fix failing tests

Author: ahmarsuhail

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java

@@ -24,6 +24,7 @@
 import software.amazon.awssdk.core.retry.RetryUtils;
 import software.amazon.awssdk.services.s3.model.S3Exception;
 import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.encryption.s3.S3EncryptionClientException;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -170,6 +171,12 @@ public static IOException translateException(@Nullable String operation,
         operation,
         StringUtils.isNotEmpty(path)? (" on " + path) : "",
         exception);
+
+    // Exceptions from encryption client are wrapped in S3EncryptionClientException, so unwrap.
+    if (exception instanceof S3EncryptionClientException) {
+      exception = (SdkException) exception.getCause();
+    }
+
     if (!(exception instanceof AwsServiceException)) {
       // exceptions raised client-side: connectivity, auth, network problems...
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java

@@ -332,8 +332,7 @@ private Map<String, byte[]> retrieveHeaders(
       maybeSetHeader(headers, XA_CONTENT_LENGTH,
           md.contentLength());
     }
-    maybeSetHeader(headers, XA_CONTENT_LENGTH,
-        md.contentLength());
+
     if (md.sdkHttpResponse() != null && md.sdkHttpResponse().headers() != null
         && md.sdkHttpResponse().headers().get("Content-Range") != null) {
       maybeSetHeader(headers, XA_CONTENT_RANGE,

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java

@@ -237,8 +237,8 @@ public void testEncryptionEnabledAndDisabledFS() throws Exception {
       assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE,
           encryptedFSFileStatus.getLen());
 
-      intercept(SecurityException.class, "",
-          "SecurityException should be thrown",
+      intercept(AWSClientIOException.class, "Instruction file not found!",
+          "AWSClientIOException should be thrown",
           () -> {
             in.read(new byte[SMALL_FILE_SIZE]);
             return "Exception should be raised if unencrypted data is read by "

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java

@@ -365,6 +365,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty()
       throws Exception {
 
     conf = new Configuration();
+    unsetClientSideConfiguration(conf);
     conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
     assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
 
@@ -403,6 +404,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty()
   @Test
   public void testDefaultUserAgent() throws Exception {
     conf = new Configuration();
+    unsetClientSideConfiguration(conf);
     fs = S3ATestUtils.createTestFileSystem(conf);
     assertNotNull(fs);
     S3Client s3 = getS3Client("User Agent");
@@ -416,6 +418,7 @@ public void testDefaultUserAgent() throws Exception {
   @Test
   public void testCustomUserAgent() throws Exception {
     conf = new Configuration();
+    unsetClientSideConfiguration(conf);
     conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
     fs = S3ATestUtils.createTestFileSystem(conf);
     assertNotNull(fs);
@@ -431,6 +434,7 @@ public void testCustomUserAgent() throws Exception {
   public void testRequestTimeout() throws Exception {
     conf = new Configuration();
     conf.set(REQUEST_TIMEOUT, "120");
+    unsetClientSideConfiguration(conf);
     fs = S3ATestUtils.createTestFileSystem(conf);
     S3Client s3 = getS3Client("Request timeout (ms)");
     SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
@@ -575,6 +579,12 @@ public void testS3SpecificSignerOverride() throws Exception {
         .describedAs("Custom STS signer not called").isTrue();
   }
 
+  private void unsetClientSideConfiguration(Configuration conf) {
+    removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_ALGORITHM);
+    conf.set(Constants.S3_ENCRYPTION_ALGORITHM,
+        S3AEncryptionMethods.NONE.getMethod());
+  }
+
   public static final class CustomS3Signer implements Signer {
 
     private static boolean s3SignerCalled = false;