HADOOP-19278. fix the build, tune PutObjectOptions use

Change-Id: I8f30359068e464eb49810bbe964e2048c5962455

Author: steveloughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -2140,8 +2140,7 @@ private FSDataOutputStream innerCreateFile(
         committerIntegration.createTracker(path, key, outputStreamStatistics);
     String destKey = putTracker.getDestKey();
 
-    // put options are derived from the path and the
-    // option builder.
+    // put options are derived from the option builder.
     final PutObjectOptions putOptions =
         new PutObjectOptions(null, options.getHeaders());
 
@@ -3578,7 +3577,7 @@ private void createFakeDirectoryIfNecessary(Path f)
     // is mostly harmless to create a new one.
     if (!key.isEmpty() && !s3Exists(f, StatusProbeEnum.DIRECTORIES)) {
       LOG.debug("Creating new fake directory at {}", f);
-      createFakeDirectory(key, putOptionsForPath(f));
+      createFakeDirectory(key, PutObjectOptions.defaultOptions());
     }
   }
 
@@ -3806,7 +3805,7 @@ public void createFakeDirectory(final Path dir)
         throws IOException {
       S3AFileSystem.this.createFakeDirectory(
           pathToKey(dir),
-          PutObjectOptions.keepingDirs());
+          PutObjectOptions.defaultOptions());
     }
   }
 
@@ -4210,7 +4209,8 @@ public void copyLocalFileFromTo(File file, Path from, Path to) throws IOExceptio
             newPutObjectRequestBuilder(key, file.length(), false);
         final String dest = to.toString();
         S3AFileSystem.this.invoker.retry("putObject(" + dest + ")", dest, true, () ->
-            executePut(putObjectRequestBuilder.build(), null, putOptionsForPath(to), file));
+            executePut(putObjectRequestBuilder.build(), null,
+                PutObjectOptions.defaultOptions(), file));
         return null;
       });
     }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java

@@ -565,7 +565,7 @@ public CompleteMultipartUploadResponse commitUpload(
         uploadId,
         partETags,
         length,
-        PutObjectOptions.keepingDirs(),
+        PutObjectOptions.defaultOptions(),
         Invoker.NO_OP);
   }
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java

@@ -547,7 +547,7 @@ public SinglePendingCommit uploadFileToPendingCommit(File localFile,
 
       statistics.commitCreated();
       uploadId = writeOperations.initiateMultiPartUpload(destKey,
-          PutObjectOptions.keepingDirs());
+          PutObjectOptions.defaultOptions());
       long length = localFile.length();
 
       SinglePendingCommit commitData = new SinglePendingCommit();

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/S3MagicCommitTracker.java

@@ -117,7 +117,7 @@ public boolean aboutToComplete(String uploadId,
   @Retries.RetryTranslated
   private void upload(PutObjectRequest request, byte[] bytes) throws IOException {
     trackDurationOfInvocation(getTrackerStatistics(), COMMITTER_MAGIC_MARKER_PUT.getSymbol(),
-        () -> getWriter().putObject(request, PutObjectOptions.keepingDirs(),
+        () -> getWriter().putObject(request, PutObjectOptions.defaultOptions(),
             new S3ADataBlocks.BlockUploadData(bytes, null), null));
   }
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java

@@ -63,25 +63,18 @@ public String toString() {
         '}';
   }
 
-  private static final PutObjectOptions KEEP_DIRS = new PutObjectOptions(
-      null, null);
-  private static final PutObjectOptions DELETE_DIRS = new PutObjectOptions(
-      null, null);
-
   /**
-   * Get the options to keep directories.
-   * @return an instance which keeps dirs
+   * Empty options.
    */
-  public static PutObjectOptions keepingDirs() {
-    return KEEP_DIRS;
-  }
+  private static final PutObjectOptions EMPTY_OPTIONS = new PutObjectOptions(
+      null, null);
 
   /**
-   * Get the options to delete directory markers.
-   * @return an instance which deletes dirs
+   * Get the default options.
+   * @return an instance with no storage class or headers.
    */
-  public static PutObjectOptions deletingDirs() {
-    return DELETE_DIRS;
+  public static PutObjectOptions defaultOptions() {
+    return EMPTY_OPTIONS;
   }
 
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java

@@ -128,7 +128,7 @@ public CompletableFuture<UploadHandle> startUpload(
     return context.submit(new CompletableFuture<>(),
         trackDurationOfCallable(statistics, OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(), () -> {
           String uploadId = writeOperations.initiateMultiPartUpload(key,
-              PutObjectOptions.keepingDirs());
+              PutObjectOptions.defaultOptions());
           statistics.uploadStarted();
           return BBUploadHandle.from(ByteBuffer.wrap(
               uploadId.getBytes(StandardCharsets.UTF_8)));

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java

@@ -26,17 +26,13 @@
 import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
 
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.assertj.core.api.Assertions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.EnumSet;
 
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java

@@ -108,7 +108,7 @@ public void testPutObjectDirect() throws Throwable {
       LambdaTestUtils.intercept(IllegalStateException.class,
           () -> fs.putObjectDirect(
               putObjectRequestBuilder.build(),
-              PutObjectOptions.keepingDirs(),
+              PutObjectOptions.defaultOptions(),
               new S3ADataBlocks.BlockUploadData("PUT".getBytes(), null),
               null));
       assertPathDoesNotExist("put object was created", path);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java

@@ -41,7 +41,6 @@
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
 import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase;
 import org.apache.hadoop.fs.s3a.impl.ClientManager;
-import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
 import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java

@@ -90,7 +90,7 @@ public static IdKey createPartUpload(S3AFileSystem fs, String key, int len,
       WriteOperationHelper writeHelper = fs.getWriteOperationHelper();
       byte[] data = dataset(len, 'a', 'z');
       InputStream in = new ByteArrayInputStream(data);
-      String uploadId = writeHelper.initiateMultiPartUpload(key, PutObjectOptions.keepingDirs());
+      String uploadId = writeHelper.initiateMultiPartUpload(key, PutObjectOptions.defaultOptions());
       UploadPartRequest req = writeHelper.newUploadPartRequestBuilder(key, uploadId,
           partNo, len).build();
       RequestBody body = RequestBody.fromInputStream(in, len);