HADOOP-13502. Split fs.contract.is-blobstore flag into more descripti…

…ve flags for use by contract tests. Contributed by Chris Nauroth.

.gitignore

@@ -30,6 +30,7 @@ yarnregistry.pdf
 hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
 hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
 hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
+hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
 patchprocess/
 hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
 hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java

@@ -121,7 +121,7 @@ public void testOverwriteNonEmptyDirectory() throws Throwable {
     try {
       assertIsDirectory(path);
     } catch (AssertionError failure) {
-      if (isSupported(IS_BLOBSTORE)) {
+      if (isSupported(CREATE_OVERWRITES_DIRECTORY)) {
         // file/directory hack surfaces here
         throw new AssumptionViolatedException(failure.toString(), failure);
       }
@@ -137,10 +137,10 @@ public void testOverwriteNonEmptyDirectory() throws Throwable {
       FileStatus status = getFileSystem().getFileStatus(path);
 
       boolean isDir = status.isDirectory();
-      if (!isDir && isSupported(IS_BLOBSTORE)) {
-        // object store: downgrade to a skip so that the failure is visible
-        // in test results
-        skip("Object store allows a file to overwrite a directory");
+      if (!isDir && isSupported(CREATE_OVERWRITES_DIRECTORY)) {
+        // For some file systems, downgrade to a skip so that the failure is
+        // visible in test results.
+        skip("This Filesystem allows a file to overwrite a directory");
       }
       fail("write of file over dir succeeded");
     } catch (FileAlreadyExistsException expected) {
@@ -170,10 +170,10 @@ public void testCreatedFileIsImmediatelyVisible() throws Throwable {
                                    1024)) {
       if (!getFileSystem().exists(path)) {
 
-        if (isSupported(IS_BLOBSTORE)) {
-          // object store: downgrade to a skip so that the failure is visible
-          // in test results
-          skip("Filesystem is an object store and newly created files are not immediately visible");
+        if (isSupported(CREATE_VISIBILITY_DELAYED)) {
+          // For some file systems, downgrade to a skip so that the failure is
+          // visible in test results.
+          skip("This Filesystem delays visibility of newly created files");
         }
         assertPathExists("expected path to be visible before anything written",
                          path);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java

@@ -36,17 +36,34 @@ public interface ContractOptions {
    */
   String FS_CONTRACT_KEY = "fs.contract.";
 
+  /**
+   * Flag to indicate that a newly created file may overwrite a pre-existing
+   * directory.
+   * {@value}
+   */
+  String CREATE_OVERWRITES_DIRECTORY = "create-overwrites-directory";
+
+  /**
+   * Flag to indicate that a newly created file is not made visible in the
+   * namespace immediately.  Instead, the file becomes visible at a later point
+   * in the file creation lifecycle, such as when the client closes it.
+   * {@value}
+   */
+  String CREATE_VISIBILITY_DELAYED = "create-visibility-delayed";
+
   /**
    * Is a filesystem case sensitive.
    * Some of the filesystems that say "no" here may mean
    * that it varies from platform to platform -the localfs being the key
    * example.
+   * {@value}
    */
   String IS_CASE_SENSITIVE = "is-case-sensitive";
 
   /**
    * Blobstore flag. Implies it's not a real directory tree and
    * consistency is below that which Hadoop expects
+   * {@value}
    */
   String IS_BLOBSTORE = "is-blobstore";
 
@@ -196,6 +213,7 @@ public interface ContractOptions {
   /**
    * Limit for #of random seeks to perform.
    * Keep low for remote filesystems for faster tests
+   * {@value}
    */
   String TEST_RANDOM_SEEK_COUNT = "test.random-seek-count";
 

hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml

@@ -26,11 +26,6 @@
     <value>false</value>
   </property>
 
-  <property>
-    <name>fs.contract.is-blobstore</name>
-    <value>false</value>
-  </property>
-
   <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
@@ -81,4 +76,4 @@
     <value>false</value>
   </property>
 
-</configuration>
+</configuration>

hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml

@@ -37,6 +37,11 @@
     <value>true</value>
   </property>
 
+  <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
   <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>

hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml

@@ -37,6 +37,16 @@
     <value>true</value>
   </property>
 
+  <property>
+    <name>fs.contract.create-overwrites-directory</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
   <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>

hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml

@@ -37,6 +37,16 @@
     <value>true</value>
   </property>
 
+  <property>
+    <name>fs.contract.create-overwrites-directory</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.create-visibility-delayed</name>
+    <value>true</value>
+  </property>
+
   <property>
     <name>fs.contract.is-case-sensitive</name>
     <value>true</value>