Add an option to disable chunked encoding.

This is done to allow sha256 calculation of proper
payload in PutObjectRequest and UploadPartRequest.

Fixes aws#580

Author: Harshavardhana

aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java

@@ -2993,7 +2993,8 @@ protected Signer createSigner(final Request<?> request,
             final AmazonWebServiceRequest req = request.getOriginalRequest();
 
             if (!(signer instanceof AWSS3V4Signer) && ((upgradeToSigV4(req)))) {
-                final AWSS3V4Signer v4Signer = new AWSS3V4Signer();
+                boolean isChunkedEncodingDisabled = this.clientOptions.isChunkedEncodingDisabled();
+                final AWSS3V4Signer v4Signer = new AWSS3V4Signer(isChunkedEncodingDisabled);
                 // Always set the service name; if the user has overridden it via
                 // setEndpoint(String, String, String), this will return the right
                 // value. Otherwise it will return "s3", which is an appropriate
@@ -3609,8 +3610,13 @@ private <X, Y extends AmazonWebServiceRequest> X invoke(Request<Y> request,
         checkHttps(originalRequest);
         ExecutionContext executionContext = createExecutionContext(originalRequest);
         // Retry V4 auth errors
-        executionContext.setAuthErrorRetryStrategy(new S3V4AuthErrorRetryStrategy(buildDefaultEndpointResolver(
-                getProtocol(request), bucket, key)));
+        S3RequestEndpointResolver endpointResolver = buildDefaultEndpointResolver(getProtocol(request),
+                                                                                  bucket, key);
+        boolean isChunkedEncodingDisabled = this.clientOptions.isChunkedEncodingDisabled();
+        S3V4AuthErrorRetryStrategy authErrRetryStrategy = new S3V4AuthErrorRetryStrategy(endpointResolver,
+                                                                                         isChunkedEncodingDisabled);
+        executionContext.setAuthErrorRetryStrategy(authErrRetryStrategy);
+        // -- ends retry strategy.
         AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
         // Binds the request metrics to the current request.
         request.setAWSRequestMetrics(awsRequestMetrics);

aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/S3ClientOptions.java

@@ -22,13 +22,20 @@ public class S3ClientOptions {
     /** The default setting for use of path-style access */
     public static final boolean DEFAULT_PATH_STYLE_ACCESS = false;
 
+    /** The default setting for use of chunked encoding */
+    public static final boolean DEFAULT_CHUNKED_ENCODING_DISABLED = false;
+
     /** Flag for use of path-style access */
     private boolean pathStyleAccess = DEFAULT_PATH_STYLE_ACCESS;
 
+    /** Flag for user of chunked encoding */
+    private boolean chunkedEncodingDisabled = DEFAULT_CHUNKED_ENCODING_DISABLED;
+
     public S3ClientOptions() {}
 
     public S3ClientOptions( S3ClientOptions other ) {
         this.pathStyleAccess = other.pathStyleAccess;
+        this.chunkedEncodingDisabled = other.chunkedEncodingDisabled;
     }
 
     /**
@@ -53,6 +60,28 @@ public boolean isPathStyleAccess() {
         return pathStyleAccess;
     }
 
+    /**
+     * <p>
+     * Returns whether the client has chunked encoding disabled for all
+     * requests.
+     * </p>
+     * <p>
+     * The default behaviour is to detect API request which enables chunked
+     * encoding automatically for PutObjectRequest and UploadPartRequest.
+     * Setting this flag will result in disabling chunked encoding for all
+     * requests.
+     * Note: Enabling this option has performance implications since each
+     * payload will have to be checksum calculated. If your payload is large
+     * this will affect the overall time required to upload an object.
+     * Using this option is recommended only if your endpoint does not
+     * implement chunked uploading.
+     * </p>
+     * @return True is the client should always enable chunked encoding.
+     */
+    public boolean isChunkedEncodingDisabled() {
+        return chunkedEncodingDisabled;
+    }
+
     /**
      * <p>
      * Configures the client to use path-style access for all requests.
@@ -103,4 +132,27 @@ public S3ClientOptions withPathStyleAccess(boolean pathStyleAccess) {
       return this;
     }
 
-}
+    /**
+     * <p>
+     * Configures the client to disable chunked encoding for all requests.
+     * </p>
+     * <p>
+     * The default behaviour is to detect API request which enables chunked
+     * encoding automatically for PutObjectRequest and UploadPartRequest.
+     * Setting this flag will result in disabling chunked encoding for all
+     * requests.
+     * Note: Enabling this option has performance implications since each
+     * payload will have to be checksum calculated. If your payload is large
+     * this will affect the overall time required to upload an object.
+     * Using this option is recommended only if your endpoint does not
+     * implement chunked uploading.
+     * </p>
+     * @return The updated S3ClientOptions object with chunked encoding
+     *         disabled setting.
+     */
+    public S3ClientOptions disableChunkedEncoding() {
+      this.chunkedEncodingDisabled = true;
+      return this;
+    }
+
+}

aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/AWSS3V4Signer.java

@@ -36,13 +36,15 @@
  */
 public class AWSS3V4Signer extends AWS4Signer {
     private static final String CONTENT_SHA_256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+    private static boolean isChunkedEncodingDisabled;
 
     /**
      * Don't double-url-encode path elements; S3 expects path elements to be
      * encoded only once in the canonical URI.
      */
-    public AWSS3V4Signer() {
+    public AWSS3V4Signer(boolean isChunkedEncodingDisabled) {
         super(false);
+        this.isChunkedEncodingDisabled = isChunkedEncodingDisabled;
     }
 
     /**
@@ -120,6 +122,11 @@ protected String calculateContentHash(SignableRequest<?> request) {
     private static boolean useChunkEncoding(SignableRequest<?> request) {
         // Whether to use chunked encoding for signing the request
         boolean chunkedEncodingEnabled = false;
+        // if chunked encoding is explicitly disabled through client options
+        // return right here.
+        if (isChunkedEncodingDisabled) {
+          return chunkedEncodingEnabled;
+        }
         if (request.getOriginalRequestObject() instanceof PutObjectRequest
                 || request.getOriginalRequestObject() instanceof UploadPartRequest) {
             chunkedEncodingEnabled = true;

aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/S3V4AuthErrorRetryStrategy.java

@@ -46,20 +46,25 @@ public class S3V4AuthErrorRetryStrategy implements AuthErrorRetryStrategy {
             + "buckets located in regions that require V4 signing.";
 
     private final S3RequestEndpointResolver endpointResolver;
+    private final boolean isChunkedEncodingDisabled;
     private final SdkPredicate<AmazonServiceException> sigV4RetryPredicate;
 
-    public S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver) {
+  public S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver,
+                                    boolean isChunkedEncodingDisabled) {
         this.endpointResolver = endpointResolver;
+        this.isChunkedEncodingDisabled = isChunkedEncodingDisabled;
         this.sigV4RetryPredicate = new IsSigV4RetryablePredicate();
     }
 
     /**
      * Currently only used for testing
      */
     S3V4AuthErrorRetryStrategy(S3RequestEndpointResolver endpointResolver,
-            SdkPredicate<AmazonServiceException> isSigV4Retryable) {
+                               SdkPredicate<AmazonServiceException> isSigV4Retryable,
+                               boolean isChunkedEncodingDisabled) {
         this.endpointResolver = endpointResolver;
         this.sigV4RetryPredicate = isSigV4Retryable;
+        this.isChunkedEncodingDisabled = isChunkedEncodingDisabled;
     }
 
     @Override
@@ -92,7 +97,7 @@ private AuthRetryParameters redirectToRegionInHeader(Request<?> request, HttpRes
     /**
      * If the response doesn't have the x-amz-region header we have to resort to sending a request
      * to s3-external-1
-     * 
+     *
      * @return
      */
     private AuthRetryParameters redirectToS3External() {
@@ -107,7 +112,7 @@ private AuthRetryParameters redirectToS3External() {
     }
 
     private AWSS3V4Signer buildSigV4Signer(final String region) {
-        AWSS3V4Signer v4Signer = new AWSS3V4Signer();
+        AWSS3V4Signer v4Signer = new AWSS3V4Signer(isChunkedEncodingDisabled);
         v4Signer.setRegionName(region);
         v4Signer.setServiceName(AmazonS3Client.S3_SERVICE_NAME);
         return v4Signer;