Add interface DataSetSubLockStrategy

Author: hfutatzhanghb

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -1744,6 +1744,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean
       DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT = false;
 
+  public static final String DFS_DATANODE_DATASET_SUBLOCK_COUNT_KEY =
+      "dfs.datanode.dataset.sublock.count";
+  public static final long DFS_DATANODE_DATASET_SUBLOCK_COUNT_DEFAULT = 1000L;
+
   // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
   @Deprecated
   public static final String  DFS_CLIENT_RETRY_POLICY_ENABLED_KEY

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java

@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import java.util.List;
+
+/**
+ * This interface is used to generate sub lock name for a blockid.
+ */
+public interface DataSetSubLockStrategy {
+
+  /**
+   * Generate sub lock name for the given blockid.
+   * @param blockid the block id.
+   * @return sub lock name for the input blockid.
+   */
+  String blockIdToSubLock(long blockid);
+
+  List<String> getAllSubLockName();
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java

@@ -21,8 +21,6 @@
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -129,31 +127,6 @@ public static File idToBlockDir(File root, long blockId) {
     return new File(root, path);
   }
 
-  /**
-   * Take an example. We hava a block with blockid mapping to:
-   * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
-   * We return "subdir0/subdir0"
-   * @param blockId blockId
-   * @return The two-level subdir name
-   */
-  public static String idToBlockDirSuffixName(long blockId) {
-    int d1 = (int) ((blockId >> 16) & 0x1F);
-    int d2 = (int) ((blockId >> 8) & 0x1F);
-    return DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
-        DataStorage.BLOCK_SUBDIR_PREFIX + d2;
-  }
-
-  public static List<String> getAllSubDirNameForDataSetLock() {
-    List<String> res = new ArrayList<>();
-    for (int d1 = 0; d1 <= 0x1F; d1++) {
-      for (int d2 = 0; d2 <= 0x1F; d2++) {
-        res.add(DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
-            DataStorage.BLOCK_SUBDIR_PREFIX + d2);
-      }
-    }
-    return res;
-  }
-
   /**
    * @return the FileInputStream for the meta data of the given block.
    * @throws FileNotFoundException

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ModDataSetSubLockStrategy implements DataSetSubLockStrategy {
+  public static final Logger LOG = LoggerFactory.getLogger(DataSetSubLockStrategy.class);
+
+  private static final String LOCK_NAME_PERFIX = "SubLock";
+  private long modFactor;
+
+  public ModDataSetSubLockStrategy(long mod) {
+    if (mod <= 0) {
+      mod = 1L;
+    }
+    this.modFactor = mod;
+  }
+
+  @Override
+  public String blockIdToSubLock(long blockid) {
+    return LOCK_NAME_PERFIX + String.valueOf(blockid % modFactor);
+  }
+
+  @Override
+  public List<String> getAllSubLockName() {
+    List<String> res = new ArrayList<>();
+    for (long i = 0L; i < modFactor; i++) {
+      res.add(LOCK_NAME_PERFIX + i);
+    }
+    return res;
+  }
+}