HADOOP-13887. review comments

Author: Mehakmeet Singh

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -426,7 +426,7 @@ private Constants() {
    *
    * {@value}
    */
-  public static final String S3_ENCRYPTION_ALGORITHM =
+  public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM =
       "fs.s3a.server-side-encryption-algorithm";
 
   /**
@@ -442,14 +442,14 @@ private Constants() {
 
   /**
    * Used to specify which AWS KMS key to use if
-   * {@link #S3_ENCRYPTION_ALGORITHM} is
+   * {@link #SERVER_SIDE_ENCRYPTION_ALGORITHM} is
    * {@code SSE-KMS} (will default to aws/s3
    * master key if left blank).
    * With with {@code SSE_C}, the base-64 encoded AES 256 key.
    * May be set within a JCEKS file.
    * Value: "{@value}".
    */
-  public static final String S3_ENCRYPTION_KEY =
+  public static final String SERVER_SIDE_ENCRYPTION_KEY =
       "fs.s3a.server-side-encryption.key";
 
   /**

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java

@@ -58,8 +58,8 @@
 import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CENTRAL_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING;
 import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
+import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
 
 /**
@@ -125,7 +125,7 @@ public AmazonS3 createS3Client(
 
     try {
       if (S3AEncryptionMethods.getMethod(S3AUtils.
-          lookupPassword(conf, S3_ENCRYPTION_ALGORITHM, null))
+          lookupPassword(conf, SERVER_SIDE_ENCRYPTION_ALGORITHM, null))
           .equals(S3AEncryptionMethods.CSE_KMS)) {
         return buildAmazonS3EncryptionClient(
             awsConf,
@@ -149,6 +149,7 @@ public AmazonS3 createS3Client(
    * @param parameters parameters.
    *
    * @return new AmazonS3 client.
+   * @throws IOException if lookupPassword() has any problem.
    */
   protected AmazonS3 buildAmazonS3EncryptionClient(
       final ClientConfiguration awsConf,
@@ -161,10 +162,10 @@ protected AmazonS3 buildAmazonS3EncryptionClient(
 
     //CSE-KMS Method
     String kmsKeyId = S3AUtils.lookupPassword(conf,
-        S3_ENCRYPTION_KEY, null);
+        SERVER_SIDE_ENCRYPTION_KEY, null);
     // Check if kmsKeyID is not null
     Preconditions.checkArgument(kmsKeyId != null, "CSE-KMS method "
-        + "requires KMS key ID. Use " + S3_ENCRYPTION_KEY
+        + "requires KMS key ID. Use " + SERVER_SIDE_ENCRYPTION_KEY
         + " property to set it. ");
 
     EncryptionMaterialsProvider materialsProvider =

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java

@@ -329,7 +329,8 @@ public synchronized void write(byte[] source, int offset, int len)
    * @param isLast true, if part being uploaded is last and client side
    *               encryption is enabled.
    * @throws IOException Problems opening the destination for upload,
-   *                     initializing the upload, or if a previous operation has failed.
+   *                     initializing the upload, or if a previous operation
+   *                     has failed.
    */
   private synchronized void uploadCurrentBlock(boolean isLast)
       throws IOException {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -428,7 +428,7 @@ public void initialize(URI name, Configuration originalConf)
       initializeStatisticsBinding();
       // If CSE-KMS method is set then CSE is enabled.
       isCSEEnabled = S3AUtils.lookupPassword(conf,
-          S3_ENCRYPTION_ALGORITHM, null) != null;
+          SERVER_SIDE_ENCRYPTION_ALGORITHM, null) != null;
       LOG.debug("Client Side Encryption enabled: {}", isCSEEnabled);
       setCSEGauge();
       // Username is the current user at the time the FS was instantiated.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java

@@ -125,14 +125,14 @@ public final class S3AUtils {
   public static final String SSE_C_NO_KEY_ERROR =
       S3AEncryptionMethods.SSE_C.getMethod()
           + " is enabled but no encryption key was declared in "
-          + S3_ENCRYPTION_KEY;
+          + SERVER_SIDE_ENCRYPTION_KEY;
   /**
    * Encryption SSE-S3 is used but the caller also set an encryption key.
    */
   public static final String SSE_S3_WITH_KEY_ERROR =
       S3AEncryptionMethods.SSE_S3.getMethod()
           + " is enabled but an encryption key was set in "
-          + S3_ENCRYPTION_KEY;
+          + SERVER_SIDE_ENCRYPTION_KEY;
   private static final String EOF_MESSAGE_IN_XML_PARSER
       = "Failed to sanitize XML document destined for handler class";
 
@@ -518,6 +518,7 @@ public static String stringify(AmazonS3Exception e) {
    * @param owner owner of the file
    * @param eTag S3 object eTag or null if unavailable
    * @param versionId S3 object versionId or null if unavailable
+   * @param isCSEEnabled is client side encryption enabled?
    * @return a status entry
    */
   public static S3AFileStatus createFileStatus(Path keyPath,
@@ -1576,9 +1577,9 @@ static void patchSecurityCredentialProviders(Configuration conf) {
   public static String getS3EncryptionKey(String bucket,
       Configuration conf) {
     try {
-      return lookupPassword(bucket, conf, S3_ENCRYPTION_KEY);
+      return lookupPassword(bucket, conf, SERVER_SIDE_ENCRYPTION_KEY);
     } catch (IOException e) {
-      LOG.error("Cannot retrieve " + S3_ENCRYPTION_KEY, e);
+      LOG.error("Cannot retrieve " + SERVER_SIDE_ENCRYPTION_KEY, e);
       return "";
     }
   }
@@ -1598,7 +1599,7 @@ public static S3AEncryptionMethods getEncryptionAlgorithm(String bucket,
       Configuration conf) throws IOException {
     S3AEncryptionMethods encryptionMethod = S3AEncryptionMethods.getMethod(
         lookupPassword(bucket, conf,
-            S3_ENCRYPTION_ALGORITHM));
+            SERVER_SIDE_ENCRYPTION_ALGORITHM));
     String encryptionKey = getS3EncryptionKey(bucket, conf);
     int encryptionKeyLen =
         StringUtils.isBlank(encryptionKey) ? 0 : encryptionKey.length();

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java

@@ -1348,7 +1348,7 @@ public int run(String[] args, PrintStream out)
           ENDPOINT,
           StringUtils.isNotEmpty(endpoint) ? endpoint : "(unset)");
       String encryption =
-          printOption(out, "\tEncryption", S3_ENCRYPTION_ALGORITHM,
+          printOption(out, "\tEncryption", SERVER_SIDE_ENCRYPTION_ALGORITHM,
               "none");
       printOption(out, "\tInput seek policy", INPUT_FADVISE, INPUT_FADV_NORMAL);
       printOption(out, "\tChange Detection Source", CHANGE_DETECT_SOURCE,

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md

@@ -60,7 +60,7 @@ to encrypt the data as it saved to S3. It remains encrypted on S3 until deleted:
 clients cannot change the encryption attributes of an object once uploaded.
 
 The Amazon AWS SDK also offers client-side encryption, in which all the encoding
-and decoding of data is performed on the client. 
+and decoding of data is performed on the client.
 
 The server-side "SSE" encryption is performed with symmetric AES256 encryption;
 S3 offers different mechanisms for actually defining the key to use.
@@ -113,7 +113,7 @@ This encrypts the data on the client, before transmitting to S3, where it is
 stored encrypted. The data is unencrypted after downloading when it is being
 read back.
 
-In CSE-KMS, the ID of an AWS-KMS key is provided to the S3A client; 
+In CSE-KMS, the ID of an AWS-KMS key is provided to the S3A client;
 the client communicates with AWS-KMS to request a new encryption key, which
 KMS returns along with the same key encrypted with the KMS key.
 The S3 client encrypts the payload *and* attaches the KMS-encrypted version
@@ -508,7 +508,7 @@ Analysis
 1. The WARN commands are the AWS SDK warning that because the S3A client uses
 an encryption algorithm which seek() requires, the SDK considers it less
 secure than the most recent algorithm(s). Ignore.
-   
+
 * `header.x-amz-server-side-encryption="AES256"` : the file has been encrypted with S3-SSE. This is set up as the S3 default encryption,
 so even when CSE is enabled, the data is doubly encrypted.
 * `header.x-amz-cek-alg="AES/GCM/NoPadding`: client-side encrypted with the `"AES/GCM/NoPadding` algorithm.
@@ -569,11 +569,11 @@ Use `distCp`for this, with per-bucket encryption policies.
 Amazon S3 Client Side Encryption(S3-CSE), is used to encrypt data on the
 client-side and then transmit it over to S3 storage. The same encrypted data
 is then transmitted over to client while reading and then
-decrypted on the client-side. 
+decrypted on the client-side.
 
 S3-CSE, uses `AmazonS3EncryptionClientV2.java`  as the AmazonS3 client. The
 encryption and decryption is done by AWS SDK. As of July 2021, Only CSE-KMS
-method is supported. 
+method is supported.
 
 A key reason this feature (HADOOP-13887) has been unavailable for a long time
 is that the AWS S3 client pads uploaded objects with a 16 byte footer. This
@@ -585,7 +585,7 @@ footer, as ORC and Parquet do.
 There is now a workaround: compensate for the footer in listings when CSE is enabled.
 
 - When listing files and directories, 16 bytes are subtracted from the length
-of all non-empty objects( greater than or equal to 16 bytes). 
+of all non-empty objects( greater than or equal to 16 bytes).
 - Directory markers MAY be longer than 0 bytes long.
 
 This "appears" to work; secondly it does in the testing as of July 2021. However
@@ -605,11 +605,11 @@ clients where S3-CSE has not been enabled.
  client.
 - Writing files may be slower, as only a single block can be encrypted and
  uploaded at a time.
-- Multipart Uploader API disabled. 
+- Multipart Uploader API disabled.
 - S3 Select is not supported.
 - Multipart uploads would be serial, and partSize must be a multiple of 16
  bytes.
-- maximum message size in bytes that can be encrypted under this mode is 
+- maximum message size in bytes that can be encrypted under this mode is
  2^36-32, or ~64G, due to the security limitation of AES/GCM as recommended by
  NIST.
 
@@ -625,15 +625,15 @@ KMS_KEY_ID:
 Identifies the symmetric CMK that encrypts the data key.
 To specify a CMK, use its key ID, key ARN, alias name, or alias ARN. When
 using an alias name, prefix it with "alias/". To specify a CMK in a
-different AWSaccount, you must use the key ARN or alias ARN.
+different AWS account, you must use the key ARN or alias ARN.
 
 For example:
-- Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab
-- Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab
-- Alias name: alias/ExampleAlias
-- Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias
+- Key ID: `1234abcd-12ab-34cd-56ef-1234567890ab`
+- Key ARN: `arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab`
+- Alias name: `alias/ExampleAlias`
+- Alias ARN: `arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias`
 
-*Note:* If `fs.s3a.server-side-encryption-algorithm=CSE-KMS` is set, 
+*Note:* If `fs.s3a.server-side-encryption-algorithm=CSE-KMS` is set,
 `fs.s3a.server-side-encryption.key=<KMS_KEY_ID>` property must be set for
 S3-CSE to work.
 

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md

@@ -1161,7 +1161,7 @@ file using configured SSE-C keyB into that structure.
 
 ### Instruction file not found for S3 object
 
-Reading an unencrypted file would fail when read through CSE enabled client. 
+Reading an unencrypted file would fail when read through CSE enabled client.
 ```
 java.lang.SecurityException: Instruction file not found for S3 object with bucket name: ap-south-cse, key: unencryptedData.txt
 	at com.amazonaws.services.s3.internal.crypto.v2.S3CryptoModuleAE.decipher(S3CryptoModuleAE.java:190)
@@ -1199,13 +1199,13 @@ java.lang.SecurityException: Instruction file not found for S3 object with bucke
 	at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:95)
 	at org.apache.hadoop.fs.FsShell.main(FsShell.java:390)
 ```
-CSE enabled client should read encrypted data only. 
+CSE enabled client should read encrypted data only.
 
-### CSE-KMS method requires KMS key ID 
+### CSE-KMS method requires KMS key ID
 
 KMS key ID is required for CSE-KMS to encrypt data, not providing one leads
- to failure. 
- 
+ to failure.
+
 ```
 2021-07-07 11:33:04,550 WARN fs.FileSystem: Failed to initialize fileystem
 s3a://ap-south-cse/: java.lang.IllegalArgumentException: CSE-KMS
@@ -1214,7 +1214,7 @@ method requires KMS key ID. Use fs.s3a.server-side-encryption.key property to se
  set it.
 ```
 
-set `fs.s3a.server-side-encryption.key=<KMS_KEY_ID>` generated through AWS console. 
+set `fs.s3a.server-side-encryption.key=<KMS_KEY_ID>` generated through AWS console.
 
 ### `com.amazonaws.services.kms.model.IncorrectKeyException` The key ID in the request does not identify a CMK that can perform this operation.
 
@@ -1231,7 +1231,7 @@ The key ID in the request does not identify a CMK that can perform this
 operation. (Service: AWSKMS ; Status Code: 400; Error Code: IncorrectKeyException;
 Request ID: da21aa8a-f00d-467c-94a0-32b627d32bc0; Proxy: null)
 ```
-Use the same KMS key ID used to upload data to download and read it as well. 
+Use the same KMS key ID used to upload data to download and read it as well.
 
 ### `com.amazonaws.services.kms.model.NotFoundException` key/<KMS_KEY_ID> does not exist
 
@@ -1249,8 +1249,8 @@ does not exist(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException;
 Request ID: 279db85d-864d-4a38-9acd-d892adb504c0; Proxy: null)
 ```
 While generating the KMS Key ID make sure to generate it in the same region
- as your bucket. 
- 
+ as your bucket.
+
 ### Unable to perform range get request: Range get support has been disabled
 
 If Range get is not supported for a CSE algorithm or is disabled:
@@ -1276,7 +1276,7 @@ java.lang.SecurityException: Unable to perform range get request: Range get supp
 	at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:408)
 	at java.io.DataInputStream.readByte(DataInputStream.java:265)
 ```
-Range gets msut be enabled for CSE to work. 
+Range gets must be enabled for CSE to work.
 
 ### WARNING: Range gets do not provide authenticated encryption properties even when used with an authenticated mode (AES-GCM).
 
@@ -1297,7 +1297,7 @@ get data.
 
 The S3 Encryption Client is configured to read encrypted data with legacy
 encryption modes through the CryptoMode setting, and we would see this
-warning for all S3-CSE request. 
+warning for all S3-CSE request.
 
 ```
 2021-07-14 12:54:09,519 [main] WARN  s3.AmazonS3EncryptionClientV2
@@ -1307,16 +1307,18 @@ encryption modes through the CryptoMode setting. If you don't have objects
 encrypted with these legacy modes, you should disable support for them to
 enhance security. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html
 ```
-We can ignore this, since this CryptoMode setting(CryptoMode.AuthenticatedEncryption) 
-is required for range gets to work. 
+We can ignore this, since this CryptoMode setting(CryptoMode.AuthenticatedEncryption)
+is required for range gets to work.
 
 ### com.amazonaws.services.kms.model.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK
 
 If you generated an Asymmetric CMK from AWS console then CSE-KMS won't be
-able to generate unique data key for encryption. 
+able to generate unique data key for encryption.
 
 ```
-Caused by: com.amazonaws.services.kms.model.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK (Service: AWSKMS; Status Code: 400; Error Code: InvalidKeyUsageException; Request ID: 93609c15-e490-4035-8390-f4396f0d90bf; Proxy: null)
+Caused by: com.amazonaws.services.kms.model.InvalidKeyUsageException:
+You cannot generate a data key with an asymmetric CMK
+(Service: AWSKMS; Status Code: 400; Error Code: InvalidKeyUsageException; Request ID: 93609c15-e490-4035-8390-f4396f0d90bf; Proxy: null)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372)
@@ -1348,15 +1350,16 @@ Caused by: com.amazonaws.services.kms.model.InvalidKeyUsageException: You cannot
 ```
 
 Generate a Symmetric Key in the same region as your S3 storage for CSE-KMS to
-work. 
+work.
 
 ### com.amazonaws.services.kms.model.NotFoundException: Invalid keyId
 
 If the value in `fs.s3a.server-side-encryption.key` property, does not exist
 /valid in AWS KMS CMK(Customer managed keys), then this error would be seen.
 
 ```
-Caused by: com.amazonaws.services.kms.model.NotFoundException: Invalid keyId abc (Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: 9d53552a-3d1b-47c8-984c-9a599d5c2391; Proxy: null)
+Caused by: com.amazonaws.services.kms.model.NotFoundException: Invalid keyId abc
+(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: 9d53552a-3d1b-47c8-984c-9a599d5c2391; Proxy: null)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372)
@@ -1392,12 +1395,11 @@ same on AWS console.
 
 ### com.amazonaws.services.kms.model.AWSKMSException: User: <User_ARN> is not authorized to perform : kms :GenerateDataKey on resource: <KEY_ID>
 
-User doesn't have authorisation to the specific AWS KMS Key ID. 
+User doesn't have authorization to the specific AWS KMS Key ID.
 ```
-Caused by: com.amazonaws.services.kms.model.AWSKMSException: User: arn:aws
-:iam::152813717728:user/<user> is not authorized to perform: kms
-:GenerateDataKey on resource: <key_ID> (Service: AWSKMS; Status Code: 400
-; Error Code: AccessDeniedException; Request ID: 4ded9f1f-b245-4213-87fc-16cba7a1c4b9; Proxy: null)
+Caused by: com.amazonaws.services.kms.model.AWSKMSException:
+User: arn:aws:iam::152813717728:user/<user> is not authorized to perform: kms:GenerateDataKey on resource: <key_ID>
+(Service: AWSKMS; Status Code: 400; Error Code: AccessDeniedException; Request ID: 4ded9f1f-b245-4213-87fc-16cba7a1c4b9; Proxy: null)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1819)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1403)
 	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1372)
@@ -1431,8 +1433,8 @@ Caused by: com.amazonaws.services.kms.model.AWSKMSException: User: arn:aws
 The user trying to use the KMS Key ID should have the right permissions to access
 (encrypt/decrypt) using the AWS KMS Key used via `fs.s3a.server-side-encryption.key`.
 If not, then add permission(or IAM role) in "Key users" section by selecting the
-AWS-KMS CMK Key on AWS console. 
- 
+AWS-KMS CMK Key on AWS console.
+
 ### <a name="not_all_bytes_were_read"></a> Message appears in logs "Not all bytes were read from the S3ObjectInputStream"