docs and small fixes

Change-Id: Ief4db248f46be18d1838743dc0b212d6a4bbfd83

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -176,7 +176,7 @@ private Constants() {
 
   // number of times we should retry errors
   public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
-  public static final int DEFAULT_MAX_ERROR_RETRIES = 20;
+  public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
 
   // seconds until we give up trying to establish a connection to s3
   public static final String ESTABLISH_TIMEOUT =

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.fs.s3a.s3guard;
 
 import javax.annotation.Nullable;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.URI;
@@ -295,7 +294,7 @@ public class DynamoDBMetadataStore implements MetadataStore,
    * This policy is mostly for batched writes, not for processing
    * exceptions in invoke() calls.
    * It also has a role purpose in
-   * {@link DynamoDBMetadataStoreTableHandler#getVersionMarkerItem()};
+   * {@link DynamoDBMetadataStoreTableManager#getVersionMarkerItem()};
    * look at that method for the details.
    */
   private RetryPolicy batchWriteRetryPolicy;
@@ -342,7 +341,7 @@ public class DynamoDBMetadataStore implements MetadataStore,
    */
   private ITtlTimeProvider ttlTimeProvider;
 
-  private DynamoDBMetadataStoreTableHandler tableHandler;
+  private DynamoDBMetadataStoreTableManager tableHandler;
 
   /**
    * A utility function to create DynamoDB instance.
@@ -423,7 +422,7 @@ public void initialize(FileSystem fs, ITtlTimeProvider ttlTp)
 
     this.ttlTimeProvider = ttlTp;
 
-    tableHandler = new DynamoDBMetadataStoreTableHandler(
+    tableHandler = new DynamoDBMetadataStoreTableManager(
         dynamoDB, tableName, region, amazonDynamoDB, conf, readOp,
         batchWriteRetryPolicy);
     this.table = tableHandler.initTable();
@@ -509,7 +508,7 @@ public void initialize(Configuration config,
     initDataAccessRetries(conf);
     this.ttlTimeProvider = ttlTp;
 
-    tableHandler = new DynamoDBMetadataStoreTableHandler(
+    tableHandler = new DynamoDBMetadataStoreTableManager(
         dynamoDB, tableName, region, amazonDynamoDB, conf, readOp,
         batchWriteRetryPolicy);
     this.table = tableHandler.initTable();
@@ -2300,7 +2299,7 @@ private static String stateAsString(@Nullable AncestorState state) {
     }
   }
 
-  protected DynamoDBMetadataStoreTableHandler getTableHandler() {
+  protected DynamoDBMetadataStoreTableManager getTableHandler() {
     Preconditions.checkNotNull(tableHandler, "Not initialized");
     return tableHandler;
   }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableHandler.java → hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java

@@ -84,13 +84,13 @@
  * Table handling for dynamo tables, factored out from DynamoDBMetadataStore
  * Mainly
  */
-public class DynamoDBMetadataStoreTableHandler {
+public class DynamoDBMetadataStoreTableManager {
   public static final Logger LOG = LoggerFactory.getLogger(
-      DynamoDBMetadataStoreTableHandler.class);
+      DynamoDBMetadataStoreTableManager.class);
 
   /** Error: version marker not found in table but the table is not empty. */
   public static final String E_NO_VERSION_MARKER_AND_NOT_EMPTY
-      = "S3Guard table lacks version marker, and not empty.";
+      = "S3Guard table lacks version marker, and it is not empty.";
 
   /** Error: version mismatch. */
   public static final String E_INCOMPATIBLE_TAG_VERSION
@@ -116,7 +116,7 @@ public class DynamoDBMetadataStoreTableHandler {
   private Table table;
   private String tableArn;
 
-  public DynamoDBMetadataStoreTableHandler(DynamoDB dynamoDB,
+  public DynamoDBMetadataStoreTableManager(DynamoDB dynamoDB,
       String tableName,
       String region,
       AmazonDynamoDB amazonDynamoDB,
@@ -249,7 +249,7 @@ protected static Item getVersionMarkerFromTags(Table table,
               .withResourceArn(description.getTableArn());
       tags = addb.listTagsOfResource(listTagsOfResourceRequest).getTags();
     } catch (ResourceNotFoundException e) {
-      LOG.error("Table: {} not found.");
+      LOG.error("Table: {} not found.", table.getTableName());
       throw e;
     } catch (AmazonDynamoDBException e) {
       LOG.warn("Exception while getting tags from the dynamo table: {}",
@@ -286,7 +286,7 @@ protected static Item getVersionMarkerFromTags(Table table,
    * @throws IOException on any failure.
    * @throws InterruptedIOException if the wait was interrupted
    */
-  @Retries.OnceRaw
+  @Retries.OnceMixed
   private void createTable(ProvisionedThroughput capacity) throws IOException {
     try {
       String mode;
@@ -432,7 +432,7 @@ protected void verifyVersionCompatibility() throws IOException {
       throwExceptionOnVersionMismatch(itemVersionMarker, tableName,
           E_INCOMPATIBLE_ITEM_VERSION);
 
-      LOG.info("Table {} contains correct version marker TAG and ITEM.",
+      LOG.debug("Table {} contains correct version marker TAG and ITEM.",
           tableName);
     }
   }

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -951,9 +951,36 @@ logged.
 
 ### Versioning
 
-S3Guard tables are created with a version marker, an entry with the primary
-key and child entry of `../VERSION`; the use of a relative path guarantees
-that it will not be resolved.
+S3Guard tables are created with a version marker entry and table tag.
+The entry is created with the primary key and child entry of `../VERSION`; 
+the use of a relative path guarantees that it will not be resolved.
+Table tag key is named `s3guard_version`.
+
+When the table is initialized by S3Guard, the table will be tagged during the 
+creating and the version marker entry will be created in the table.
+If the table lacks the version marker entry or tag, S3Guard will try to create
+it according to the following rules:
+
+1. If the table lacks both version markers AND it's empty, both markers will be added. 
+If the table is not empty the check throws IOException
+1. If there's no version marker ITEM, the compatibility with the TAG
+will be checked, and the version marker ITEM will be added if the
+TAG version is compatible.
+If the TAG version is not compatible, the check throws OException
+1. If there's no version marker TAG, the compatibility with the ITEM
+version marker will be checked, and the version marker ITEM will be
+added if the ITEM version is compatible.
+If the ITEM version is not compatible, the check throws IOException
+1. If the TAG and ITEM versions are both present then both will be checked
+for compatibility. If the ITEM or TAG version marker is not compatible,
+the check throws IOException
+
+*Note*: If the user does not have sufficient rights to tag the table the 
+initialization of S3Guard will not fail, but there will be no version marker tag
+on the dynamo table and the following message will be logged on WARN level:
+```
+Exception during tagging table: {AmazonDynamoDBException exception message}
+```
 
 *Versioning policy*
 
@@ -974,6 +1001,8 @@ in an incompatible manner. The version marker in tables exists to support
 such an option if it ever becomes necessary, by ensuring that all S3Guard
 client can recognise any version mismatch.
 
+* Table versionin
+
 ## Security
 
 All users of the DynamoDB table must have write access to it. This

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java

@@ -78,10 +78,10 @@
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableHandler.E_INCOMPATIBLE_ITEM_VERSION;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableHandler.E_INCOMPATIBLE_TAG_VERSION;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableHandler.E_NO_VERSION_MARKER_AND_NOT_EMPTY;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableHandler.getVersionMarkerFromTags;
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.E_INCOMPATIBLE_ITEM_VERSION;
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.E_INCOMPATIBLE_TAG_VERSION;
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.E_NO_VERSION_MARKER_AND_NOT_EMPTY;
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.getVersionMarkerFromTags;
 import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
 import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
 import static org.apache.hadoop.test.LambdaTestUtils.*;
@@ -122,7 +122,7 @@ public ITestDynamoDBMetadataStore() {
 
   private S3AFileSystem fileSystem;
   private S3AContract s3AContract;
-  private DynamoDBMetadataStoreTableHandler tableHandler;
+  private DynamoDBMetadataStoreTableManager tableHandler;
 
   private URI fsUri;
 
@@ -652,7 +652,7 @@ public void testTableVersioning() throws Exception {
     DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
     try {
       ddbms.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-      DynamoDBMetadataStoreTableHandler localTableHandler =
+      DynamoDBMetadataStoreTableManager localTableHandler =
           ddbms.getTableHandler();
 
       Table table = verifyTableInitialized(tableName, ddbms.getDynamoDB());
@@ -668,7 +668,7 @@ public void testTableVersioning() throws Exception {
   }
 
   private void checkVerifyVersionMarkerCompatibility(
-      DynamoDBMetadataStoreTableHandler localTableHandler, Table table)
+      DynamoDBMetadataStoreTableManager localTableHandler, Table table)
       throws Exception {
     final AmazonDynamoDB addb
         = getDynamoMetadataStore().getAmazonDynamoDB();

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreScale.java

@@ -91,7 +91,7 @@ public class ITestDynamoDBMetadataStoreScale
   private static final long MAXIMUM_WRITE_CAPACITY = 15;
 
   private DynamoDBMetadataStore ddbms;
-  private DynamoDBMetadataStoreTableHandler tableHandler;
+  private DynamoDBMetadataStoreTableManager tableHandler;
 
   private DynamoDB ddb;
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java

@@ -32,7 +32,7 @@
 import org.apache.hadoop.test.HadoopTestBase;
 import org.apache.hadoop.fs.Path;
 
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableHandler.translateTableWaitFailure;
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.translateTableWaitFailure;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;