HDDS-1543. Implement addAcl,removeAcl,setAcl,getAcl for Prefix. Contr…

…ibuted by Xiaoyu Yao.

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java

@@ -46,6 +46,9 @@
  *   <tr>
  *     <td> 2 </td> <td> Bucket Lock </td>
  *   </tr>
+ *   <tr>
+ *     <td> 3 </td> <td> Prefix Lock </td>
+ *    </tr>
  * </table>
  *
  * One cannot obtain a lower weight lock while holding a lock with higher
@@ -66,6 +69,7 @@ public final class OzoneManagerLock {
 
   private static final String VOLUME_LOCK = "volumeLock";
   private static final String BUCKET_LOCK = "bucketLock";
+  private static final String PREFIX_LOCK = "prefixLock";
   private static final String S3_BUCKET_LOCK = "s3BucketLock";
   private static final String S3_SECRET_LOCK = "s3SecretetLock";
 
@@ -77,6 +81,7 @@ public final class OzoneManagerLock {
           () -> ImmutableMap.of(
               VOLUME_LOCK, new AtomicInteger(0),
               BUCKET_LOCK, new AtomicInteger(0),
+              PREFIX_LOCK, new AtomicInteger(0),
               S3_BUCKET_LOCK, new AtomicInteger(0),
               S3_SECRET_LOCK, new AtomicInteger(0)
           )
@@ -241,4 +246,24 @@ public void releaseS3SecretLock(String awsAccessId) {
     manager.unlock(awsAccessId);
     myLocks.get().get(S3_SECRET_LOCK).decrementAndGet();
   }
+
+  public void acquirePrefixLock(String prefixPath) {
+    if (hasAnyPrefixLock()) {
+      throw new RuntimeException(
+          "Thread '" + Thread.currentThread().getName() +
+              "' cannot acquire prefix path lock while holding prefix " +
+              "path lock(s) for path: " + prefixPath + ".");
+    }
+    manager.lock(prefixPath);
+    myLocks.get().get(PREFIX_LOCK).incrementAndGet();
+  }
+
+  private boolean hasAnyPrefixLock() {
+    return myLocks.get().get(PREFIX_LOCK).get() != 0;
+  }
+
+  public void releasePrefixLock(String prefixPath) {
+    manager.unlock(prefixPath);
+    myLocks.get().get(PREFIX_LOCK).decrementAndGet();
+  }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java

@@ -199,6 +199,9 @@ public enum ResultCodes {
 
     PERMISSION_DENIED, // Error codes used during acl validation
 
-    TIMEOUT // Error codes used during acl validation
+    TIMEOUT, // Error codes used during acl validation
+
+    PREFIX_NOT_FOUND,
+
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java

@@ -71,6 +71,19 @@ public StoreType getStoreType() {
 
   public abstract String getKeyName();
 
+  /**
+   * Get PrefixName.
+   * A prefix name is like a key name under the bucket but
+   * are mainly used for ACL for now and persisted into a separate prefix table.
+   *
+   * @return prefix name.
+   */
+  public abstract String getPrefixName();
+
+  /**
+   * Get full path of a key or prefix including volume and bucket.
+   * @return full path of a key or prefix.
+   */
   public abstract String getPath();
 
   /**
@@ -79,7 +92,8 @@ public StoreType getStoreType() {
   public enum ResourceType {
     VOLUME(OzoneConsts.VOLUME),
     BUCKET(OzoneConsts.BUCKET),
-    KEY(OzoneConsts.KEY);
+    KEY(OzoneConsts.KEY),
+    PREFIX(OzoneConsts.PREFIX);
 
     /**
      * String value for this Enum.

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java

@@ -23,32 +23,51 @@
 
 /**
  * Class representing an ozone object.
+ * It can be a volume with non-null volumeName (bucketName=null & name=null)
+ * or a bucket with non-null volumeName and bucketName (name=null)
+ * or a key with non-null volumeName, bucketName and key name
+ * (via getKeyName)
+ * or a prefix with non-null volumeName, bucketName and prefix name
+ * (via getPrefixName)
  */
 public final class OzoneObjInfo extends OzoneObj {
 
   private final String volumeName;
   private final String bucketName;
-  private final String keyName;
-
+  private final String name;
 
+  /**
+   *
+   * @param resType
+   * @param storeType
+   * @param volumeName
+   * @param bucketName
+   * @param name - keyName/PrefixName
+   */
   private OzoneObjInfo(ResourceType resType, StoreType storeType,
-      String volumeName, String bucketName, String keyName) {
+      String volumeName, String bucketName, String name) {
     super(resType, storeType);
     this.volumeName = volumeName;
     this.bucketName = bucketName;
-    this.keyName = keyName;
+    this.name = name;
   }
 
   @Override
   public String getPath() {
     switch (getResourceType()) {
     case VOLUME:
-      return getVolumeName();
+      return OZONE_URI_DELIMITER + getVolumeName();
     case BUCKET:
-      return getVolumeName() + OZONE_URI_DELIMITER + getBucketName();
+      return OZONE_URI_DELIMITER + getVolumeName()
+          + OZONE_URI_DELIMITER + getBucketName();
     case KEY:
-      return getVolumeName() + OZONE_URI_DELIMITER + getBucketName()
+      return OZONE_URI_DELIMITER + getVolumeName()
+          + OZONE_URI_DELIMITER + getBucketName()
           + OZONE_URI_DELIMITER + getKeyName();
+    case PREFIX:
+      return OZONE_URI_DELIMITER + getVolumeName()
+          + OZONE_URI_DELIMITER + getBucketName()
+          + OZONE_URI_DELIMITER + getPrefixName();
     default:
       throw new IllegalArgumentException("Unknown resource " +
         "type" + getResourceType());
@@ -67,9 +86,15 @@ public String getBucketName() {
 
   @Override
   public String getKeyName() {
-    return keyName;
+    return name;
   }
 
+  @Override
+  public String getPrefixName() {
+    return name;
+  }
+
+
   public static OzoneObjInfo fromProtobuf(OzoneManagerProtocolProtos.OzoneObj
       proto) {
     Builder builder = new Builder()
@@ -88,7 +113,7 @@ public static OzoneObjInfo fromProtobuf(OzoneManagerProtocolProtos.OzoneObj
     case BUCKET:
       if (tokens.length < 2) {
         throw new IllegalArgumentException("Unexpected argument for " +
-            "Ozone key. Path:" + proto.getPath());
+            "Ozone bucket. Path:" + proto.getPath());
       }
       builder.setVolumeName(tokens[0]);
       builder.setBucketName(tokens[1]);
@@ -102,6 +127,15 @@ public static OzoneObjInfo fromProtobuf(OzoneManagerProtocolProtos.OzoneObj
       builder.setBucketName(tokens[1]);
       builder.setKeyName(tokens[2]);
       break;
+    case PREFIX:
+      if (tokens.length < 3) {
+        throw new IllegalArgumentException("Unexpected argument for " +
+            "Ozone Prefix. Path:" + proto.getPath());
+      }
+      builder.setVolumeName(tokens[0]);
+      builder.setBucketName(tokens[1]);
+      builder.setPrefixName(tokens[2]);
+      break;
     default:
       throw new IllegalArgumentException("Unexpected type for " +
           "Ozone key. Type:" + proto.getResType());
@@ -118,7 +152,7 @@ public static class Builder {
     private OzoneObj.StoreType storeType;
     private String volumeName;
     private String bucketName;
-    private String keyName;
+    private String name;
 
     public static Builder newBuilder() {
       return new Builder();
@@ -145,14 +179,17 @@ public Builder setBucketName(String bucket) {
     }
 
     public Builder setKeyName(String key) {
-      this.keyName = key;
+      this.name = key;
+      return this;
+    }
+
+    public Builder setPrefixName(String prefix) {
+      this.name = prefix;
       return this;
     }
 
     public OzoneObjInfo build() {
-      return new OzoneObjInfo(resType, storeType, volumeName, bucketName,
-          keyName);
+      return new OzoneObjInfo(resType, storeType, volumeName, bucketName, name);
     }
   }
-
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/RadixTree.java

@@ -202,9 +202,15 @@ public String getLongestPrefix(String path) {
         break;
       }
     }
-    return level >= 1 ?
-        Paths.get(root.getName()).resolve(p.subpath(0, level)).toString() :
-        root.getName();
+
+    if (level >= 1) {
+      Path longestMatch =
+          Paths.get(root.getName()).resolve(p.subpath(0, level));
+      String ret = longestMatch.toString();
+      return path.endsWith("/") ?  ret + "/" : ret;
+    } else {
+      return root.getName();
+    }
   }
 
   // root of a radix tree has a name of "/" and may optionally has it value.

hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto

@@ -276,6 +276,7 @@ enum Status {
     NOT_A_FILE = 47;
     PERMISSION_DENIED = 48;
     TIMEOUT = 49;
+    PREFIX_NOT_FOUND=50;
 }
 
 
@@ -507,15 +508,15 @@ message OzoneAclInfo {
     }
 
     enum OzoneAclRights {
-      READ = 1;
-      WRITE = 2;
-      CREATE = 3;
-      LIST = 4;
-      DELETE = 5;
-      READ_ACL = 6;
-      WRITE_ACL  = 7;
-      ALL = 8;
-      NONE = 9;
+        READ = 1;
+        WRITE = 2;
+        CREATE = 3;
+        LIST = 4;
+        DELETE = 5;
+        READ_ACL = 6;
+        WRITE_ACL = 7;
+        ALL = 8;
+        NONE = 9;
     }
     required OzoneAclType type = 1;
     required string name = 2;

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/util/TestRadixTree.java

@@ -84,7 +84,6 @@ public void testGetLongestPrefixPath() {
     assertEquals("g", lpn.getName());
     lpn.setValue(100);
 
-
     List<RadixNode<Integer>> lpq =
         ROOT.getLongestPrefixPath("/a/b/c/d/g/q");
     RadixNode<Integer> lqn = lpp.get(lpq.size()-1);
@@ -93,7 +92,6 @@ public void testGetLongestPrefixPath() {
     assertEquals("g", lqn.getName());
     assertEquals(100, (int)lqn.getValue());
 
-
     assertEquals("/a/", RadixTree.radixPathToString(
         ROOT.getLongestPrefixPath("/a/g")));
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java

@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -2200,6 +2201,66 @@ public void testNativeAclsForKey() throws Exception {
     validateOzoneAcl(ozObj);
   }
 
+  @Test
+  public void testNativeAclsForPrefix() throws Exception {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+
+    String prefix1 = "PF" + UUID.randomUUID().toString() + "/";
+    String key1 = prefix1 + "KEY" + UUID.randomUUID().toString();
+
+    String prefix2 = "PF" + UUID.randomUUID().toString() + "/";
+    String key2 = prefix2 + "KEY" + UUID.randomUUID().toString();
+
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+    assertNotNull("Bucket creation failed", bucket);
+
+    writeKey(key1, bucket);
+    writeKey(key2, bucket);
+
+    OzoneObj ozObj = new OzoneObjInfo.Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setPrefixName(prefix1)
+        .setResType(OzoneObj.ResourceType.PREFIX)
+        .setStoreType(OzoneObj.StoreType.OZONE)
+        .build();
+
+    // add acl
+    BitSet aclRights1 = new BitSet();
+    aclRights1.set(ACLType.READ.ordinal());
+    OzoneAcl user1Acl = new OzoneAcl(ACLIdentityType.USER,
+        "user1", aclRights1);
+    assertTrue(store.addAcl(ozObj, user1Acl));
+
+    // get acl
+    List<OzoneAcl> aclsGet = store.getAcl(ozObj);
+    Assert.assertEquals(1, aclsGet.size());
+    Assert.assertEquals(user1Acl, aclsGet.get(0));
+
+    // remove acl
+    Assert.assertTrue(store.removeAcl(ozObj, user1Acl));
+    aclsGet = store.getAcl(ozObj);
+    Assert.assertEquals(0, aclsGet.size());
+
+    // set acl
+    BitSet aclRights2 = new BitSet();
+    aclRights2.set(ACLType.ALL.ordinal());
+    OzoneAcl group1Acl = new OzoneAcl(ACLIdentityType.GROUP,
+        "group1", aclRights2);
+    List<OzoneAcl> acls = new ArrayList<>();
+    acls.add(user1Acl);
+    acls.add(group1Acl);
+    Assert.assertTrue(store.setAcl(ozObj, acls));
+
+    // get acl
+    aclsGet = store.getAcl(ozObj);
+    Assert.assertEquals(2, aclsGet.size());
+  }
+
   /**
    * Helper function to get default acl list for current user.
    *
@@ -2218,8 +2279,7 @@ private List<OzoneAcl> getAclList(OzoneConfiguration conf)
     listOfAcls.add(new OzoneAcl(ACLIdentityType.USER,
         ugi.getUserName(), userRights));
     //Group ACLs of the User
-    List<String> userGroups = Arrays.asList(UserGroupInformation
-        .createRemoteUser(ugi.getUserName()).getGroupNames());
+    List<String> userGroups = Arrays.asList(ugi.getGroupNames());
     userGroups.stream().forEach((group) -> listOfAcls.add(
         new OzoneAcl(ACLIdentityType.GROUP, group, groupRights)));
     return listOfAcls;