HADOOP-16319 skip invalid tests when default encryption enabled

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -167,6 +167,13 @@ You can also force all the tests to run with a specific SSE encryption method
 by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a
 contract file.
 
+### <a name="default_encyption"></a> Default Encryption
+
+Buckets can be configured with [default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html)
+on the AWS side. Some S3AFileSystem tests are skipped when default encryption is
+enabled due to unpredictability in how [ETags](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html)
+are generated.
+
 ## <a name="running"></a> Running the Tests
 
 After completing the configuration, execute the test run through Maven.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java

@@ -23,6 +23,9 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.GetBucketEncryptionResult;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import org.junit.Assume;
@@ -33,9 +36,11 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.store.EtagChecksum;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.http.HttpStatus;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Tests of the S3A FileSystem which don't have a specific home and can share
@@ -134,10 +139,12 @@ Path mkFile(String name, byte[] data) throws IOException {
   /**
    * The assumption here is that 0-byte files uploaded in a single PUT
    * always have the same checksum, including stores with encryption.
+   * This will be skipped if the bucket has S3 default encryption enabled.
    * @throws Throwable on a failure
    */
   @Test
   public void testEmptyFileChecksums() throws Throwable {
+    Assume.assumeThat(getDefaultEncryption(), nullValue());
     final S3AFileSystem fs = getFileSystem();
     Path file1 = touchFile("file1");
     EtagChecksum checksum1 = fs.getFileChecksum(file1, 0);
@@ -186,12 +193,13 @@ public void testNonEmptyFileChecksums() throws Throwable {
   /**
    * Verify that on an unencrypted store, the checksum of two non-empty
    * (single PUT) files is the same if the data is the same.
-   * This will fail if the bucket has S3 default encryption enabled.
+   * This will be skipped if the bucket has S3 default encryption enabled.
    * @throws Throwable failure
    */
   @Test
   public void testNonEmptyFileChecksumsUnencrypted() throws Throwable {
     Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE));
+    Assume.assumeThat(getDefaultEncryption(), nullValue());
     final S3AFileSystem fs = getFileSystem();
     final EtagChecksum checksum1 =
         fs.getFileChecksum(mkFile("file5", HELLO), 0);
@@ -234,4 +242,24 @@ public void testS3AToStringUnitialized() throws Throwable {
     }
   }
 
+  /**
+   * Gets default encryption settings for the bucket or returns null if default
+   * encryption is disabled.
+   */
+  private GetBucketEncryptionResult getDefaultEncryption() {
+    S3AFileSystem fs = getFileSystem();
+    AmazonS3 s3 = fs.getAmazonS3ClientForTesting("check default encryption");
+    GetBucketEncryptionResult encryptionResult;
+    try {
+      encryptionResult = s3.getBucketEncryption(fs.getBucket());
+    } catch (AmazonS3Exception e) {
+      if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
+        return null;
+      }
+      // there was an unexpected problem with the API call
+      throw e;
+    }
+    return encryptionResult;
+  }
+
 }