HBASE-23275: Track active master's address in ActiveMasterManager

Currently we just track whether an active master exists.
It helps to also track the address of the active master in
all the masters to help serve the client RPC requests to
know which master is active.

Author: bharathv

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -57,10 +58,16 @@ public class ActiveMasterManager extends ZKListener {
   final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false);
   final AtomicBoolean clusterShutDown = new AtomicBoolean(false);
 
+  // This server's information.
   private final ServerName sn;
   private int infoPort;
   private final Server master;
 
+  // Active master's server name. Non-null if cluster has an active master unless there
+  // is an issue fetching the active master's server name.
+  // ServerName is immutable, so we don't need heavy synchronization around it.
+  private final AtomicReference<ServerName> activeMasterServerName;
+
   /**
    * @param watcher
    * @param sn ServerName
@@ -71,6 +78,7 @@ public class ActiveMasterManager extends ZKListener {
     watcher.registerListener(this);
     this.sn = sn;
     this.master = master;
+    activeMasterServerName = new AtomicReference<>();
   }
 
   // will be set after jetty server is started
@@ -106,6 +114,31 @@ void handle(final String path) {
     }
   }
 
+  /*
+   * Fetches the active master's ServerName from zookeeper.
+   */
+  private void fetchAndSetActiveMasterServerName() {
+    LOG.debug("Attempting to fetch active master sn from zk");
+    try {
+      activeMasterServerName.set(MasterAddressTracker.getMasterAddress(watcher));
+    } catch (IOException | KeeperException e) {
+      // Log and ignore for now and re-fetch later if needed.
+      LOG.error("Error fetching active master information", e);
+    }
+  }
+
+  public ServerName getActiveMasterServerName() {
+    if (!clusterHasActiveMaster.get()) return null;
+    ServerName sname = activeMasterServerName.get();
+    if (sname != null) {
+      return sname;
+    }
+    // This happens if the data was not fetched earlier for some reason.
+    fetchAndSetActiveMasterServerName();
+    // It could still be null, but return whatever we have.
+    return activeMasterServerName.get();
+  }
+
   /**
    * Handle a change in the master node.  Doesn't matter whether this was called
    * from a nodeCreated or nodeDeleted event because there are no guarantees
@@ -134,6 +167,9 @@ private void handleMasterNodeChange() {
           // Notify any thread waiting to become the active master
           clusterHasActiveMaster.notifyAll();
         }
+        // Reset the active master sn. Will be re-fetched later if needed.
+        // We don't want to make a synchronous RPC under a monitor.
+        activeMasterServerName.set(null);
       }
     } catch (KeeperException ke) {
       master.abort("Received an unexpected KeeperException, aborting", ke);
@@ -178,10 +214,14 @@ boolean blockUntilBecomingActiveMaster(
           // We are the master, return
           startupStatus.setStatus("Successfully registered as active master.");
           this.clusterHasActiveMaster.set(true);
+          activeMasterServerName.set(sn);
           LOG.info("Registered as active master=" + this.sn);
           return true;
         }
 
+        // Invalidate the active master name so that subsequent requests do not get any stale
+        // master information. Will ne re-fetched if needed.
+        activeMasterServerName.set(null);
         // There is another active master running elsewhere or this is a restart
         // and the master ephemeral node has not expired yet.
         this.clusterHasActiveMaster.set(true);

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -3869,4 +3869,7 @@ public String getClusterId() {
     return cachedClusterId.getFromCacheOrFetch();
   }
 
+  public ServerName getActiveMaster() {
+    return activeMasterManager.getActiveMasterServerName();
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -91,6 +92,7 @@ public static void tearDownAfterClass() throws Exception {
     ActiveMasterManager activeMasterManager =
       dummyMaster.getActiveMasterManager();
     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
+    assertEquals(null, activeMasterManager.getActiveMasterServerName());
 
     // First test becoming the active master uninterrupted
     MonitoredTask status = Mockito.mock(MonitoredTask.class);
@@ -99,6 +101,7 @@ public static void tearDownAfterClass() throws Exception {
     activeMasterManager.blockUntilBecomingActiveMaster(100, status);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, master);
+    assertMaster(zk, activeMasterManager.getActiveMasterServerName());
 
     // Now pretend master restart
     DummyMaster secondDummyMaster = new DummyMaster(zk,master);
@@ -108,6 +111,8 @@ public static void tearDownAfterClass() throws Exception {
     activeMasterManager.blockUntilBecomingActiveMaster(100, status);
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, master);
+    assertMaster(zk, activeMasterManager.getActiveMasterServerName());
+    assertMaster(zk, secondActiveMasterManager.getActiveMasterServerName());
   }
 
   /**
@@ -135,6 +140,7 @@ public void testActiveMasterManagerFromZK() throws Exception {
     ActiveMasterManager activeMasterManager =
       ms1.getActiveMasterManager();
     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
+    assertEquals(activeMasterManager.getActiveMasterServerName(), null);
 
     // First test becoming the active master uninterrupted
     ClusterStatusTracker clusterStatusTracker =
@@ -144,6 +150,7 @@ public void testActiveMasterManagerFromZK() throws Exception {
         Mockito.mock(MonitoredTask.class));
     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
     assertMaster(zk, firstMasterAddress);
+    assertMaster(zk, activeMasterManager.getActiveMasterServerName());
 
     // New manager will now try to become the active master in another thread
     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
@@ -161,6 +168,8 @@ public void testActiveMasterManagerFromZK() throws Exception {
     assertTrue(t.manager.clusterHasActiveMaster.get());
     // But secondary one should not be the active master
     assertFalse(t.isActiveMaster);
+    // Verify the active master ServerName is populated in standby master.
+    assertEquals(firstMasterAddress, t.manager.getActiveMasterServerName());
 
     // Close the first server and delete it's master node
     ms1.stop("stopping first server");
@@ -189,6 +198,7 @@ public void testActiveMasterManagerFromZK() throws Exception {
 
     assertTrue(t.manager.clusterHasActiveMaster.get());
     assertTrue(t.isActiveMaster);
+    assertEquals(secondMasterAddress, t.manager.getActiveMasterServerName());
 
     LOG.info("Deleting master node");