apache#1. namenode的启动流程

Author: Yifan122

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -973,6 +973,9 @@ private void loadFSImage(StartupOption startOpt) throws IOException {
       // We shouldn't be calling saveNamespace if we've come up in standby state.
       MetaRecoveryContext recovery = startOpt.createRecoveryContext();
       final boolean staleImage
+              // namenode 启动
+              // fsimage + editlog = new fsImage
+              // TODO (1)合并元数据
           = fsImage.recoverTransitionRead(startOpt, this, recovery);
       if (RollingUpgradeStartupOption.ROLLBACK.matches(startOpt) ||
           RollingUpgradeStartupOption.DOWNGRADE.matches(startOpt)) {
@@ -983,6 +986,7 @@ private void loadFSImage(StartupOption startOpt) throws IOException {
           + " (staleImage=" + staleImage + ", haEnabled=" + haEnabled
           + ", isRollingUpgrade=" + isRollingUpgrade() + ")");
       if (needToSave) {
+        // TODO (2) 把合并出来的新的fsimage写到我们的磁盘上面
         fsImage.saveNamespace(this);
       } else {
         updateStorageVersionForRollingUpgrade(fsImage.getLayoutVersion(),
@@ -996,6 +1000,7 @@ private void loadFSImage(StartupOption startOpt) throws IOException {
       // we shouldn't do it when coming up in standby state
       if (!haEnabled || (haEnabled && startOpt == StartupOption.UPGRADE)
           || (haEnabled && startOpt == StartupOption.UPGRADEONLY)) {
+        // TODO (3) 打开新的editlog开始写日志
         fsImage.openEditLogForWrite();
       }
       success = true;
@@ -1049,19 +1054,24 @@ private void stopSecretManager() {
 
   /** 
    * Start services common to both active and standby states
+   * 1） fsimage -> 目录，磁盘存储够不够（100M）
+   * 2）editlog -> 目录，磁盘存储够不够
    */
   void startCommonServices(Configuration conf, HAContext haContext) throws IOException {
     this.registerMBean(); // register the MBean for the FSNamesystemState
     writeLock();
     this.haContext = haContext;
     try {
+      // NameNode资源检查 通过hdfs-site.xml, core-site.xml 两个文件找到fsimage, editlog的目录
       nnResourceChecker = new NameNodeResourceChecker(conf);
+      // 判断路径是否有足够的可用空间
       checkAvailableResources();
       assert safeMode != null && !isPopulatingReplQueues();
       StartupProgress prog = NameNode.getStartupProgress();
       prog.beginPhase(Phase.SAFEMODE);
       prog.setTotal(Phase.SAFEMODE, STEP_AWAITING_REPORTED_BLOCKS,
         getCompleteBlocksTotal());
+      // TODO 设置安全模式
       setBlockTotal();
       blockManager.activate(conf);
     } finally {
@@ -5264,6 +5274,10 @@ private synchronized boolean canLeave() {
     /** 
      * There is no need to enter safe mode 
      * if DFS is empty or {@link #threshold} == 0
+     * TODO 触发安全模式的三个条件：
+     * 1. complete block的个数高于threshold
+     * 2， datanode的个数高于 DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY = "dfs.namenode.safemode.min.datanodes"
+     * 3. 存放fsimage editlog的目录下面的大小高于100m
      */
     private boolean needEnter() {
       return (threshold != 0 && blockSafe < blockThreshold) ||
@@ -5272,7 +5286,8 @@ private boolean needEnter() {
     }
 
     /**
-     * Check and trigger safe mode if needed. 
+     * Check and trigger safe mode if needed.
+     * TODO 检查是否进入安全模式
      */
     private void checkMode() {
       // Have to have write-lock since leaving safemode initializes
@@ -5283,6 +5298,7 @@ private void checkMode() {
       }
       // if smmthread is already running, the block threshold must have been 
       // reached before, there is no need to enter the safe mode again
+      // TODO needEnter =》 检查是否进入安全模式
       if (smmthread == null && needEnter()) {
         enter();
         // check if we are ready to initialize replication queues
@@ -5344,6 +5360,8 @@ private synchronized void setBlockTotal(int total) {
      */
     private synchronized void incrementSafeBlockCount(short replication) {
       if (replication == safeReplication) {
+        // datanode 会进行block的汇报
+        // 每次汇报后blocksafe就会 自增
         this.blockSafe++;
 
         // Report startup progress only if we haven't completed startup yet.
@@ -5679,6 +5697,7 @@ public void setBlockTotal() {
     SafeModeInfo safeMode = this.safeMode;
     if (safeMode == null)
       return;
+    // getCompleteBlocksTotal 获取所有正常的block的个数
     safeMode.setBlockTotal((int)getCompleteBlocksTotal());
   }
 
@@ -5694,6 +5713,10 @@ public long getBlocksTotal() {
   /**
    * Get the total number of COMPLETE blocks in the system.
    * For safe mode only complete blocks are counted.
+   *
+   * TODO 在HDFS集群里面block的状态分为两种类型
+   * 1）complete类型：正常的可用的block
+   * 2）underconstruction类型： 处于正在构建的block
    */
   private long getCompleteBlocksTotal() {
     // Calculate number of blocks under construction

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -137,8 +137,8 @@
  * is a second backup/failover NameNode, or when using federated NameNodes.)
  *
  * The NameNode controls two critical tables:
- *   1)  filename->blocksequence (namespace)
- *   2)  block->machinelist ("inodes")
+ *   1)  filename->blocksequence (namespace) 文件与文件块（block）的关系， 这张表存储在了磁盘上面
+ *   2)  block->machinelist ("inodes") ： 文件块与之间的关系，每次NameNode 重启后会重新构建出来
  *
  * The first table is stored on disk and is very precious.
  * The second table is rebuilt every time the NameNode comes up.
@@ -149,6 +149,17 @@
  * with exposing the IPC interface and the HTTP server to the outside world,
  * plus some configuration management.
  *
+ *  NameNode 服务是由三个重要的类支撑的：
+ *  1）NameNode类：
+ *    管理配置的参数：hdfs-site.xml, core-site.xml
+ *  2) NameNode Server:
+ *     IPC server:
+ *        NameNodeRPCServer: 开放端口，等待别人调用，比如：8020/9000
+ *     HTTP server：
+ *        NameNodeHttpServer: 开放50070界面，我们可以通过这个界面了解HDFS的情况
+ *   3）FSNameSystem:
+ *    这个类很重要，管理了HDFS的元数据
+ *
  * NameNode implements the
  * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} interface, which
  * allows clients to ask for DFS services.
@@ -635,13 +646,18 @@ protected void initialize(Configuration conf) throws IOException {
     StartupProgressMetrics.register(startupProgress);
 
     if (NamenodeRole.NAMENODE == role) {
+      // TODO 启动HttpServer
       startHttpServer(conf);
     }
 
     this.spanReceiverHost = SpanReceiverHost.getInstance(conf);
 
+    /**
+     * TODO 加载元数据
+     */
     loadNamesystem(conf);
 
+    // TODO 启动RPCserver
     rpcServer = createRpcServer(conf);
     if (clientNamenodeAddress == null) {
       // This is expected for MiniDFSCluster. Set it now using 
@@ -659,7 +675,9 @@ protected void initialize(Configuration conf) throws IOException {
     pauseMonitor = new JvmPauseMonitor(conf);
     pauseMonitor.start();
     metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
-    
+    // TODO 启动一些公共的服务。NameNode RPC的服务就是在里面启动的
+    // 1）进行资源检查，检查是否有磁盘足够存储元数据
+    // 2）进入安全模式检查，检查是否可以退出安全模式
     startCommonServices(conf);
   }
 
@@ -674,6 +692,7 @@ protected NameNodeRpcServer createRpcServer(Configuration conf)
 
   /** Start the services common to active and standby states */
   private void startCommonServices(Configuration conf) throws IOException {
+    // TODO 元数据管理
     namesystem.startCommonServices(conf, haContext);
     registerNNSMXBean();
     if (NamenodeRole.NAMENODE != role) {
@@ -807,6 +826,7 @@ protected NameNode(Configuration conf, NamenodeRole role)
     this.haContext = createHAContext();
     try {
       initializeGenericKeys(conf, nsId, namenodeId);
+      // TODO 初始化代码
       initialize(conf);
       try {
         haContext.writeLock();
@@ -1416,11 +1436,19 @@ public static NameNode createNameNode(String argv[], Configuration conf)
     LOG.info("createNameNode " + Arrays.asList(argv));
     if (conf == null)
       conf = new HdfsConfiguration();
+    /**
+     * 操作HDFS集群的时候会传进来如下的参数：
+     *
+     * hdfs namenode -format
+     *
+     * hadoop-daemon.sh start namenode
+     */
     StartupOption startOpt = parseArguments(argv);
     if (startOpt == null) {
       printUsage(System.err);
       return null;
     }
+
     setStartupOption(conf, startOpt);
 
     switch (startOpt) {
@@ -1550,6 +1578,7 @@ public static void main(String argv[]) throws Exception {
 
     try {
       StringUtils.startupShutdownMessage(NameNode.class, argv, LOG);
+      // TODO 创建NameNode 的核心代码
       NameNode namenode = createNameNode(argv, null);
       if (namenode != null) {
         namenode.join();