AWS: bump aws sdk version to 2.20.18 (apache#7003)

aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIO.java

@@ -73,6 +73,7 @@
 import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
@@ -88,6 +89,7 @@ public class TestS3FileIO {
   private final String batchDeletionBucketPrefix = "batch-delete-";
   private final int batchDeletionSize = 5;
   private S3FileIO s3FileIO;
+
   private final Map<String, String> properties =
       ImmutableMap.of(
           "s3.write.tags.tagKey1",
@@ -99,11 +101,9 @@ public class TestS3FileIO {
   public void before() {
     s3FileIO = new S3FileIO(() -> s3mock);
     s3FileIO.initialize(properties);
-    s3.get().createBucket(CreateBucketRequest.builder().bucket("bucket").build());
+    createBucket("bucket");
     for (int i = 1; i <= numBucketsForBatchDeletion; i++) {
-      s3.get()
-          .createBucket(
-              CreateBucketRequest.builder().bucket(batchDeletionBucketPrefix + i).build());
+      createBucket(batchDeletionBucketPrefix + i);
     }
     StaticClientFactory.client = s3mock;
   }
@@ -369,4 +369,12 @@ private void createRandomObjects(String prefix, int count) {
                     builder -> builder.bucket(s3URI.bucket()).key(s3URI.key() + i).build(),
                     RequestBody.empty()));
   }
+
+  private void createBucket(String bucketName) {
+    try {
+      s3.get().createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+    } catch (BucketAlreadyExistsException e) {
+      // do nothing
+    }
+  }
 }

aws/src/test/java/org/apache/iceberg/aws/s3/TestS3InputStream.java

@@ -34,6 +34,7 @@
 import org.junit.Test;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 
@@ -45,7 +46,7 @@ public class TestS3InputStream {
 
   @Before
   public void before() {
-    s3.createBucket(CreateBucketRequest.builder().bucket("bucket").build());
+    createBucket("bucket");
   }
 
   @Test
@@ -185,4 +186,12 @@ private void writeS3Data(S3URI uri, byte[] data) throws IOException {
             .build(),
         RequestBody.fromBytes(data));
   }
+
+  private void createBucket(String bucketName) {
+    try {
+      s3.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+    } catch (BucketAlreadyExistsException e) {
+      // don't do anything
+    }
+  }
 }

aws/src/test/java/org/apache/iceberg/aws/s3/TestS3OutputStream.java

@@ -63,6 +63,7 @@
 import software.amazon.awssdk.core.sync.ResponseTransformer;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -105,7 +106,7 @@ public TestS3OutputStream() throws IOException {}
   @Before
   public void before() {
     properties.setS3ChecksumEnabled(false);
-    s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());
+    createBucket(BUCKET);
   }
 
   @After
@@ -335,4 +336,12 @@ private byte[] randomData(int size) {
   private S3URI randomURI() {
     return new S3URI(String.format("s3://%s/data/%s.dat", BUCKET, UUID.randomUUID()));
   }
+
+  private void createBucket(String bucketName) {
+    try {
+      s3.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+    } catch (BucketAlreadyExistsException e) {
+      // do nothing
+    }
+  }
 }

aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java

@@ -52,6 +52,7 @@
 import software.amazon.awssdk.http.SdkHttpFullRequest;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
 import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -109,6 +110,7 @@ public void before() throws Exception {
                 s3ClientBuilder ->
                     s3ClientBuilder.httpClientBuilder(
                         software.amazon.awssdk.http.apache.ApacheHttpClient.builder()))
+            .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
             .endpointOverride(minioContainer.getURI())
             .overrideConfiguration(
                 c -> c.putAdvancedOption(SdkAdvancedClientOption.SIGNER, validatingSigner))
@@ -176,11 +178,19 @@ public void validatedCreateMultiPartUpload() {
 
   @Test
   public void validatedUploadPart() {
+    String multipartUploadId =
+        s3.createMultipartUpload(
+                CreateMultipartUploadRequest.builder()
+                    .bucket(BUCKET)
+                    .key("some/multipart-key")
+                    .build())
+            .uploadId();
     s3.uploadPart(
         UploadPartRequest.builder()
             .bucket(BUCKET)
             .key("some/multipart-key")
-            .uploadId("1234")
+            .uploadId(multipartUploadId)
+            .partNumber(1)
             .build(),
         RequestBody.fromString("content"));
   }

aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3SignRequestParser.java

@@ -126,7 +126,7 @@ public void roundTripSerde() {
                     "Content-Type",
                     Arrays.asList("application/json"),
                     "User-Agent",
-                    Arrays.asList("aws-sdk-java/2.17.257", "Linux/5.4.0-126")))
+                    Arrays.asList("aws-sdk-java/2.20.18", "Linux/5.4.0-126")))
             .build();
 
     String json = S3SignRequestParser.toJson(s3SignRequest, true);
@@ -141,7 +141,7 @@ public void roundTripSerde() {
                 + "    \"amz-sdk-request\" : [ \"attempt=1\", \"max=4\" ],\n"
                 + "    \"Content-Length\" : [ \"191\" ],\n"
                 + "    \"Content-Type\" : [ \"application/json\" ],\n"
-                + "    \"User-Agent\" : [ \"aws-sdk-java/2.17.257\", \"Linux/5.4.0-126\" ]\n"
+                + "    \"User-Agent\" : [ \"aws-sdk-java/2.20.18\", \"Linux/5.4.0-126\" ]\n"
                 + "  }\n"
                 + "}");
   }
@@ -162,7 +162,7 @@ public void roundTripSerdeWithProperties() {
                     "Content-Type",
                     Arrays.asList("application/json"),
                     "User-Agent",
-                    Arrays.asList("aws-sdk-java/2.17.257", "Linux/5.4.0-126")))
+                    Arrays.asList("aws-sdk-java/2.20.18", "Linux/5.4.0-126")))
             .properties(ImmutableMap.of("k1", "v1"))
             .build();
 
@@ -178,7 +178,7 @@ public void roundTripSerdeWithProperties() {
                 + "    \"amz-sdk-request\" : [ \"attempt=1\", \"max=4\" ],\n"
                 + "    \"Content-Length\" : [ \"191\" ],\n"
                 + "    \"Content-Type\" : [ \"application/json\" ],\n"
-                + "    \"User-Agent\" : [ \"aws-sdk-java/2.17.257\", \"Linux/5.4.0-126\" ]\n"
+                + "    \"User-Agent\" : [ \"aws-sdk-java/2.20.18\", \"Linux/5.4.0-126\" ]\n"
                 + "  },\n"
                 + "  \"properties\" : {\n"
                 + "    \"k1\" : \"v1\"\n"

aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3SignResponseParser.java

@@ -74,7 +74,7 @@ public void roundTripSerde() {
                     "Content-Type",
                     Arrays.asList("application/json"),
                     "User-Agent",
-                    Arrays.asList("aws-sdk-java/2.17.257", "Linux/5.4.0-126")))
+                    Arrays.asList("aws-sdk-java/2.20.18", "Linux/5.4.0-126")))
             .build();
 
     String json = S3SignResponseParser.toJson(s3SignResponse, true);
@@ -87,7 +87,7 @@ public void roundTripSerde() {
                 + "    \"amz-sdk-request\" : [ \"attempt=1\", \"max=4\" ],\n"
                 + "    \"Content-Length\" : [ \"191\" ],\n"
                 + "    \"Content-Type\" : [ \"application/json\" ],\n"
-                + "    \"User-Agent\" : [ \"aws-sdk-java/2.17.257\", \"Linux/5.4.0-126\" ]\n"
+                + "    \"User-Agent\" : [ \"aws-sdk-java/2.20.18\", \"Linux/5.4.0-126\" ]\n"
                 + "  }\n"
                 + "}");
   }

build.gradle

@@ -473,6 +473,7 @@ project(':iceberg-aws') {
     testImplementation "org.apache.httpcomponents.client5:httpclient5"
     testImplementation 'org.mock-server:mockserver-netty'
     testImplementation 'org.mock-server:mockserver-client-java'
+    testImplementation 'javax.xml.bind:jaxb-api'
     testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
   }
 

docs/aws.md

@@ -48,15 +48,15 @@ Here are some examples.
 
 ### Spark
 
-For example, to use AWS features with Spark 3.3 (with scala 2.12) and AWS clients version 2.17.257, you can start the Spark SQL shell with:
+For example, to use AWS features with Spark 3.3 (with scala 2.12) and AWS clients version 2.20.18, you can start the Spark SQL shell with:
 
 ```sh
 # add Iceberg dependency
 ICEBERG_VERSION={{% icebergVersion %}}
 DEPENDENCIES="org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:$ICEBERG_VERSION"
 
 # add AWS dependnecy
-AWS_SDK_VERSION=2.17.257
+AWS_SDK_VERSION=2.20.18
 AWS_MAVEN_GROUP=software.amazon.awssdk
 AWS_PACKAGES=(
     "bundle"
@@ -75,7 +75,7 @@ spark-sql --packages $DEPENDENCIES \
     --conf spark.sql.catalog.my_catalog.io-impl=org.apache.iceberg.aws.s3.S3FileIO
 ```
 
-As you can see, In the shell command, we use `--packages` to specify the additional AWS bundle and HTTP client dependencies with their version as `2.17.257`.
+As you can see, In the shell command, we use `--packages` to specify the additional AWS bundle and HTTP client dependencies with their version as `2.20.18`.
 
 ### Flink
 
@@ -89,7 +89,7 @@ ICEBERG_MAVEN_URL=$MAVEN_URL/org/apache/iceberg
 wget $ICEBERG_MAVEN_URL/iceberg-flink-runtime/$ICEBERG_VERSION/iceberg-flink-runtime-$ICEBERG_VERSION.jar
 
 # download AWS dependnecy
-AWS_SDK_VERSION=2.17.257
+AWS_SDK_VERSION=2.20.18
 AWS_MAVEN_URL=$MAVEN_URL/software/amazon/awssdk
 AWS_PACKAGES=(
     "bundle"
@@ -566,7 +566,7 @@ The Glue, S3 and DynamoDB clients are then initialized with the assume-role cred
 Here is an example to start Spark shell with this client factory:
 
 ```shell
-spark-sql --packages org.apache.iceberg:iceberg-spark-runtime:{{% icebergVersion %}},software.amazon.awssdk:bundle:2.17.257 \
+spark-sql --packages org.apache.iceberg:iceberg-spark-runtime:{{% icebergVersion %}},software.amazon.awssdk:bundle:2.20.18 \
     --conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
     --conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket/my/key/prefix \    
     --conf spark.sql.catalog.my_catalog.catalog-impl=org.apache.iceberg.aws.glue.GlueCatalog \
@@ -645,7 +645,7 @@ For versions before 6.5.0, you can use a [bootstrap action](https://docs.aws.ama
 ```sh
 #!/bin/bash
 
-AWS_SDK_VERSION=2.17.257
+AWS_SDK_VERSION=2.20.18
 ICEBERG_VERSION={{% icebergVersion %}}
 MAVEN_URL=https://repo1.maven.org/maven2
 ICEBERG_MAVEN_URL=$MAVEN_URL/org/apache/iceberg

versions.props

@@ -21,7 +21,7 @@ com.aliyun.oss:aliyun-sdk-oss = 3.10.2
 javax.xml.bind:jaxb-api = 2.3.1
 javax.activation:activation = 1.1.1
 org.glassfish.jaxb:jaxb-runtime = 2.3.3
-software.amazon.awssdk:* = 2.17.257
+software.amazon.awssdk:* = 2.20.18
 org.projectnessie.nessie:* = 0.51.1
 com.google.cloud:libraries-bom = 24.1.0
 org.scala-lang.modules:scala-collection-compat_2.12 = 2.6.0
@@ -36,7 +36,7 @@ org.junit.vintage:junit-vintage-engine = 5.9.2
 org.junit.jupiter:* = 5.9.2
 org.mockito:* = 4.11.0
 org.apache.tez:* = 0.8.4
-com.adobe.testing:s3mock-junit4 = 2.1.28
+com.adobe.testing:s3mock-junit4 = 2.11.0
 org.assertj:assertj-core = 3.24.2
 org.xerial:sqlite-jdbc = 3.41.0.0
 com.fasterxml.jackson.dataformat:jackson-dataformat-xml = 2.9.9