HDFS-8831. Trash Support for deletion in HDFS encryption zone. Contributed by Xiaoyu Yao.

Author: xiaoyuyao

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -105,6 +105,8 @@ public abstract class FileSystem extends Configured implements Closeable {
    */
   public static final int SHUTDOWN_HOOK_PRIORITY = 10;
 
+  public static final String TRASH_PREFIX = ".Trash";
+
   /** FileSystem cache */
   static final Cache CACHE = new Cache();
 
@@ -2663,6 +2665,53 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
         + " doesn't support getAllStoragePolicies");
   }
 
+  /**
+   * Get the root directory of Trash for current user when the path specified
+   * is deleted.
+   *
+   * @param path the trash root of the path to be determined.
+   * @return the default implementation returns "/user/$USER/.Trash".
+   * @throws IOException
+   */
+  public Path getTrashRoot(Path path) throws IOException {
+    return this.makeQualified(new Path(getHomeDirectory().toUri().getPath(),
+        TRASH_PREFIX));
+  }
+
+  /**
+   * Get all the trash roots for current user or all users.
+   *
+   * @param allUsers return trash roots for all users if true.
+   * @return all the trash root directories.
+   *         Default FileSystem returns .Trash under users' home directories if
+   *         /user/$USER/.Trash exists.
+   * @throws IOException
+   */
+  public Collection<FileStatus> getTrashRoots(boolean allUsers)
+      throws IOException {
+    Path userHome = new Path(getHomeDirectory().toUri().getPath());
+    List<FileStatus> ret = new ArrayList<FileStatus>();
+    if (!allUsers) {
+      Path userTrash = new Path(userHome, TRASH_PREFIX);
+      if (exists(userTrash)) {
+        ret.add(getFileStatus(userTrash));
+      }
+    } else {
+      Path homeParent = userHome.getParent();
+      if (exists(homeParent)) {
+        FileStatus[] candidates = listStatus(homeParent);
+        for (FileStatus candidate : candidates) {
+          Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX);
+          if (exists(userTrash)) {
+            candidate.setPath(userTrash);
+            ret.add(candidate);
+          }
+        }
+      }
+    }
+    return ret;
+  }
+
   // making it volatile to be able to do a double checked locking
   private volatile static boolean FILE_SYSTEMS_LOADED = false;
 
@@ -3183,7 +3232,7 @@ public void incrementWriteOps(int count) {
      * For each StatisticsData object, we will call accept on the visitor.
      * Finally, at the end, we will call aggregate to get the final total. 
      *
-     * @param         The visitor to use.
+     * @param         visitor to use.
      * @return        The total.
      */
     private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -643,4 +643,15 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
       throws IOException {
     return fs.getAllStoragePolicies();
   }
+
+  @Override
+  public Path getTrashRoot(Path path) throws IOException {
+    return fs.getTrashRoot(path);
+  }
+
+  @Override
+  public Collection<FileStatus> getTrashRoots(boolean allUsers)
+      throws IOException {
+    return fs.getTrashRoots(allUsers);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java

@@ -121,6 +121,16 @@ public Path getCurrentTrashDir() throws IOException {
     return getTrash().getCurrentTrashDir();
   }
 
+  /**
+   * Returns the current trash location for the path specified
+   * @param path to be deleted
+   * @return path to the trash
+   * @throws IOException
+   */
+  public Path getCurrentTrashDir(Path path) throws IOException {
+    return getTrash().getCurrentTrashDir(path);
+  }
+
   protected String getUsagePrefix() {
     return usagePrefix;
   }
@@ -129,7 +139,7 @@ protected String getUsagePrefix() {
   // that access commandFactory
 
   /**
-   *  Display help for commands with their short usage and long description
+   *  Display help for commands with their short usage and long description.
    */
    protected class Usage extends FsCommand {
     public static final String NAME = "usage";

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java

@@ -54,7 +54,7 @@ public Trash(Configuration conf) throws IOException {
    */
   public Trash(FileSystem fs, Configuration conf) throws IOException {
     super(conf);
-    trashPolicy = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory());
+    trashPolicy = TrashPolicy.getInstance(conf, fs);
   }
 
   /**
@@ -92,11 +92,7 @@ public static boolean moveToAppropriateTrash(FileSystem fs, Path p,
       throw new IOException("Failed to get server trash configuration", e);
     }
     Trash trash = new Trash(fullyResolvedFs, conf);
-    boolean success = trash.moveToTrash(fullyResolvedPath);
-    if (success) {
-      LOG.info("Moved: '" + p + "' to trash at: " + trash.getCurrentTrashDir());
-    }
-    return success;
+    return trash.moveToTrash(fullyResolvedPath);
   }
 
   /**
@@ -124,7 +120,7 @@ public void expunge() throws IOException {
   }
 
   /** get the current working directory */
-  Path getCurrentTrashDir() {
+  Path getCurrentTrashDir() throws IOException {
     return trashPolicy.getCurrentTrashDir();
   }
 
@@ -139,4 +135,8 @@ TrashPolicy getTrashPolicy() {
   public Runnable getEmptier() throws IOException {
     return trashPolicy.getEmptier();
   }
+
+  public Path getCurrentTrashDir(Path path) throws IOException {
+    return trashPolicy.getCurrentTrashDir(path);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java

@@ -38,15 +38,27 @@ public abstract class TrashPolicy extends Configured {
 
   /**
    * Used to setup the trash policy. Must be implemented by all TrashPolicy
-   * implementations
+   * implementations.
    * @param conf the configuration to be used
    * @param fs the filesystem to be used
    * @param home the home directory
    */
   public abstract void initialize(Configuration conf, FileSystem fs, Path home);
 
   /**
-   * Returns whether the Trash Policy is enabled for this filesystem
+   * Used to setup the trash policy. Must be implemented by all TrashPolicy
+   * implementations. Different from initialize(conf, fs, home), this one does
+   * not assume trash always under /user/$USER due to HDFS encryption zone.
+   * @param conf the configuration to be used
+   * @param fs the filesystem to be used
+   * @throws IOException
+   */
+  public void initialize(Configuration conf, FileSystem fs) throws IOException{
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Returns whether the Trash Policy is enabled for this filesystem.
    */
   public abstract boolean isEnabled();
 
@@ -68,8 +80,27 @@ public abstract class TrashPolicy extends Configured {
 
   /**
    * Get the current working directory of the Trash Policy
+   * This API does not work with files deleted from encryption zone when HDFS
+   * data encryption at rest feature is enabled as rename file between
+   * encryption zones or encryption zone and non-encryption zone is not allowed.
+   *
+   * The caller is recommend to use the new API
+   * TrashPolicy#getCurrentTrashDir(Path path).
+   * It returns the trash location correctly for the path specified no matter
+   * the path is in encryption zone or not.
+   */
+  public abstract Path getCurrentTrashDir() throws IOException;
+
+  /**
+   * Get the current trash directory for path specified based on the Trash
+   * Policy
+   * @param path path to be deleted
+   * @return current trash directory for the path to be deleted
+   * @throws IOException
    */
-  public abstract Path getCurrentTrashDir();
+  public Path getCurrentTrashDir(Path path) throws IOException {
+    throw new UnsupportedOperationException();
+  }
 
   /** 
    * Return a {@link Runnable} that periodically empties the trash of all
@@ -78,7 +109,7 @@ public abstract class TrashPolicy extends Configured {
   public abstract Runnable getEmptier() throws IOException;
 
   /**
-   * Get an instance of the configured TrashPolicy based on the value 
+   * Get an instance of the configured TrashPolicy based on the value
    * of the configuration parameter fs.trash.classname.
    *
    * @param conf the configuration to be used
@@ -93,4 +124,21 @@ public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path ho
     trash.initialize(conf, fs, home); // initialize TrashPolicy
     return trash;
   }
+
+  /**
+   * Get an instance of the configured TrashPolicy based on the value
+   * of the configuration parameter fs.trash.classname.
+   *
+   * @param conf the configuration to be used
+   * @param fs the file system to be used
+   * @return an instance of TrashPolicy
+   */
+  public static TrashPolicy getInstance(Configuration conf, FileSystem fs)
+      throws IOException {
+    Class<? extends TrashPolicy> trashClass = conf.getClass(
+        "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
+    TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
+    trash.initialize(conf, fs); // initialize TrashPolicy
+    return trash;
+  }
 }