HDFS-16008. RBF: Tool to initialize ViewFS Mapping to Router

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java

@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.LinkedHashMap;
@@ -34,6 +35,10 @@
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.viewfs.Constants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -101,6 +106,9 @@ public class RouterAdmin extends Configured implements Tool {
   /** Pre-compiled regular expressions to detect duplicated slashes. */
   private static final Pattern SLASHES = Pattern.compile("/+");
 
+  // Parameter matching when initializing ViewFs mount point.
+  private static final String ALL_CLUSTERS = "allClusters";
+
   public static void main(String[] argv) throws Exception {
     Configuration conf = new HdfsConfiguration();
     RouterAdmin admin = new RouterAdmin(conf);
@@ -131,8 +139,8 @@ private String getUsage(String cmd) {
       String[] commands =
           {"-add", "-update", "-rm", "-ls", "-getDestination", "-setQuota",
               "-setStorageTypeQuota", "-clrQuota", "-clrStorageTypeQuota",
-              "-safemode", "-nameservice", "-getDisabledNameservices",
-              "-refresh", "-refreshRouterArgs",
+              "-initViewFsToMountTable", "-safemode", "-nameservice",
+              "-getDisabledNameservices", "-refresh", "-refreshRouterArgs",
               "-refreshSuperUserGroupsConfiguration"};
       StringBuilder usage = new StringBuilder();
       usage.append("Usage: hdfs dfsrouteradmin :\n");
@@ -171,7 +179,9 @@ private String getUsage(String cmd) {
       return "\t[-clrQuota <path>]";
     } else if (cmd.equals("-clrStorageTypeQuota")) {
       return "\t[-clrStorageTypeQuota <path>]";
-    } else if (cmd.equals("-safemode")) {
+    }  else if (cmd.equals("-initViewFsToMountTable")) {
+      return "\t[-initViewFsToMountTable <clusterName> | allClusters]";
+    }else if (cmd.equals("-safemode")) {
       return "\t[-safemode enter | leave | get]";
     } else if (cmd.equals("-nameservice")) {
       return "\t[-nameservice enable | disable <nameservice>]";
@@ -242,6 +252,10 @@ private boolean validateMin(String[] argv) {
       if (argv.length < 2) {
         return false;
       }
+    } else if ("-initViewFsToMountTable".equals(cmd)) {
+      if (argv.length < 2) {
+        return false;
+      }
     } else if ("-getDestination".equals(cmd)) {
       if (argv.length < 2) {
         return false;
@@ -384,6 +398,15 @@ public int run(String[] argv) throws Exception {
         getDisabledNameservices();
       } else if ("-refresh".equals(cmd)) {
         refresh(address);
+      } else if ("-initViewFsToMountTable".equals(cmd)) {
+        if (initViewFsToMountTable(argv[i])) {
+          System.out.println("Successfully init ViewFs mapping to router " +
+              argv[i]);
+        } else {
+          System.err.println(
+              "Failed when execute command initViewFsToMountTable");
+          exitCode = -1;
+        }
       } else if ("-refreshRouterArgs".equals(cmd)) {
         exitCode = genericRefresh(argv, i);
       } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
@@ -1036,6 +1059,83 @@ private boolean updateQuota(String mount, long nsQuota, long ssQuota)
     return updateResponse.getStatus();
   }
 
+  /**
+   * Initialize the ViewFS mount point to the Router,
+   * either to specify a cluster or to initialize it all.
+   * @param clusterName The specified cluster to initialize,
+   * AllCluster was then all clusters.
+   * @return If the quota was updated.
+   * @throws IOException Error adding the mount point.
+   */
+  public boolean initViewFsToMountTable(String clusterName)
+      throws IOException {
+    // fs.viewfs.mounttable.ClusterX.link./data
+    final String mountTablePrefix;
+    if (clusterName.equals(ALL_CLUSTERS)) {
+      mountTablePrefix =
+          Constants.CONFIG_VIEWFS_PREFIX + ".*" +
+              Constants.CONFIG_VIEWFS_LINK + ".";
+    } else {
+      mountTablePrefix =
+          Constants.CONFIG_VIEWFS_PREFIX + "." + clusterName + "." +
+              Constants.CONFIG_VIEWFS_LINK + ".";
+    }
+    final String rootPath = "/";
+    Map<String, String> viewFsMap = getConf().getValByRegex(
+        mountTablePrefix  + rootPath);
+    if (viewFsMap.isEmpty()) {
+      System.out.println("There is no ViewFs mapping to initialize.");
+      return true;
+    }
+    for (Entry<String, String> entry : viewFsMap.entrySet()) {
+      Path path = new Path(entry.getValue());
+      URI destUri = path.toUri();
+      String mountKey = entry.getKey();
+      DestinationOrder order = DestinationOrder.HASH;
+      String mount = mountKey.replaceAll(mountTablePrefix, "");
+      if (!destUri.getScheme().equals(HdfsConstants.HDFS_URI_SCHEME)) {
+        System.out.println("Only supports HDFS, " +
+            "added Mount Point failed , " + mountKey);
+      }
+      if (!mount.startsWith(rootPath) ||
+          !destUri.getPath().startsWith(rootPath)) {
+        System.out.println("Added Mount Point failed " + mountKey);
+        continue;
+      }
+      String[] nss = new String[]{destUri.getAuthority()};
+      boolean added = addMount(
+          mount, nss, destUri.getPath(), false,
+          false, order, getACLEntityFormHdfsPath(path, getConf()));
+      if (added) {
+        System.out.println("Added mount point " + mount);
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns ACLEntity according to a HDFS pat.
+   * @param path A path of HDFS.
+   */
+  static private ACLEntity getACLEntityFormHdfsPath(
+      Path path, Configuration conf) {
+    String owner = null;
+    String group = null;
+    FsPermission mode = null;
+    try {
+      FileSystem fs = path.getFileSystem(conf);
+      if (fs.exists(path)) {
+        FileStatus fileStatus = fs.getFileStatus(path);
+        owner = fileStatus.getOwner();
+        group = fileStatus.getGroup();
+        mode = fileStatus.getPermission();
+      }
+    } catch (IOException e) {
+      System.err.println("Exception encountered " + e);
+    }
+    return new ACLEntity(owner, group, mode);
+  }
+
   /**
    * Update storage type quota of specified mount table.
    *

hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md

@@ -241,6 +241,26 @@ Mount table permission can be set by following command:
 
 The option mode is UNIX-style permissions for the mount table. Permissions are specified in octal, e.g. 0755. By default, this is set to 0755.
 
+#### Init ViewFs To Router
+Router supports initializing the [ViewFs](../hadoop-hdfs/ViewFs.html) mount point to the Router. The mapping directory protocol of ViewFS must be HDFS, and the initializer only supports one-to-one mapping.
+
+For example, use the following [ViewFs](../hadoop-hdfs/ViewFs.html) to configure the initial mount table to the router.
+
+    <configuration>
+      <property>
+        <name>fs.viewfs.mounttable.ClusterX.link./data</name>
+        <value>hdfs://nn1-clusterx.example.com:8020/data</value>
+      </property>
+      <property>
+        <name>fs.viewfs.mounttable.ClusterY.link./project</name>
+        <value>hdfs://nn1-clustery.example.com:8020/project</value>
+      </property>
+    </configuration>
+
+The [ViewFs](../hadoop-hdfs/ViewFs.html) mount table can be initialized to the Router by using the following command:
+
+    [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -initViewFsToMountTable [<clusterName> | allClusters]
+
 #### Quotas
 Router-based federation supports global quota at mount table level. Mount table entries may spread multiple subclusters and the global quota will be
 accounted across these subclusters.