YARN-11374. [Federation] Support refreshSuperUserGroupsConfiguration、refreshUserToGroupsMappings API's for Federation.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshSuperUserGroupsConfigurationRequest.java

@@ -33,4 +33,27 @@ public static RefreshSuperUserGroupsConfigurationRequest newInstance() {
         Records.newRecord(RefreshSuperUserGroupsConfigurationRequest.class);
     return request;
   }
+
+  @Public
+  @Stable
+  public static RefreshSuperUserGroupsConfigurationRequest newInstance(String subClusterId) {
+    RefreshSuperUserGroupsConfigurationRequest request =
+        Records.newRecord(RefreshSuperUserGroupsConfigurationRequest.class);
+    request.setSubClusterId(subClusterId);
+    return request;
+  }
+
+  /**
+   * Get the subClusterId.
+   *
+   * @return subClusterId.
+   */
+  public abstract String getSubClusterId();
+
+  /**
+   * Set the subClusterId.
+   *
+   * @param subClusterId subCluster Id.
+   */
+  public abstract void setSubClusterId(String subClusterId);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshUserToGroupsMappingsRequest.java

@@ -33,4 +33,27 @@ public static RefreshUserToGroupsMappingsRequest newInstance() {
         Records.newRecord(RefreshUserToGroupsMappingsRequest.class);
     return request;
   }
+
+  @Public
+  @Stable
+  public static RefreshUserToGroupsMappingsRequest newInstance(String subClusterId) {
+    RefreshUserToGroupsMappingsRequest request =
+        Records.newRecord(RefreshUserToGroupsMappingsRequest.class);
+    request.setSubClusterId(subClusterId);
+    return request;
+  }
+
+  /**
+   * Get the subClusterId.
+   *
+   * @return subClusterId.
+   */
+  public abstract String getSubClusterId();
+
+  /**
+   * Set the subClusterId.
+   *
+   * @param subClusterId subCluster Id.
+   */
+  public abstract void setSubClusterId(String subClusterId);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto

@@ -46,11 +46,13 @@ message RefreshNodesResponseProto {
 }
 
 message RefreshSuperUserGroupsConfigurationRequestProto {
+  optional string sub_cluster_id = 1;
 }
 message RefreshSuperUserGroupsConfigurationResponseProto {
 }
 
 message RefreshUserToGroupsMappingsRequestProto {
+  optional string sub_cluster_id = 1;
 }
 message RefreshUserToGroupsMappingsResponseProto {
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java

@@ -18,27 +18,31 @@
 
 package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationRequestProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationRequestProto;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
 
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
 @Private
 @Unstable
-public class RefreshSuperUserGroupsConfigurationRequestPBImpl 
-extends RefreshSuperUserGroupsConfigurationRequest {
+public class RefreshSuperUserGroupsConfigurationRequestPBImpl
+    extends RefreshSuperUserGroupsConfigurationRequest {
 
-  RefreshSuperUserGroupsConfigurationRequestProto proto = RefreshSuperUserGroupsConfigurationRequestProto.getDefaultInstance();
-  RefreshSuperUserGroupsConfigurationRequestProto.Builder builder = null;
-  boolean viaProto = false;
+  private RefreshSuperUserGroupsConfigurationRequestProto proto =
+      RefreshSuperUserGroupsConfigurationRequestProto.getDefaultInstance();
+  private RefreshSuperUserGroupsConfigurationRequestProto.Builder builder = null;
+  private boolean viaProto = false;
 
   public RefreshSuperUserGroupsConfigurationRequestPBImpl() {
     builder = RefreshSuperUserGroupsConfigurationRequestProto.newBuilder();
   }
 
-  public RefreshSuperUserGroupsConfigurationRequestPBImpl(RefreshSuperUserGroupsConfigurationRequestProto proto) {
+  public RefreshSuperUserGroupsConfigurationRequestPBImpl(
+      RefreshSuperUserGroupsConfigurationRequestProto proto) {
     this.proto = proto;
     viaProto = true;
   }
@@ -56,16 +60,46 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object other) {
-    if (other == null)
+
+    if (!(other instanceof RefreshSuperUserGroupsConfigurationRequest)) {
       return false;
-    if (other.getClass().isAssignableFrom(this.getClass())) {
-      return this.getProto().equals(this.getClass().cast(other).getProto());
     }
-    return false;
+
+    RefreshSuperUserGroupsConfigurationRequestPBImpl otherImpl = this.getClass().cast(other);
+    return new EqualsBuilder()
+        .append(this.getProto(), otherImpl.getProto())
+        .isEquals();
   }
 
   @Override
   public String toString() {
     return TextFormat.shortDebugString(getProto());
   }
+
+  private synchronized void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RefreshSuperUserGroupsConfigurationRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  @Override
+  public String getSubClusterId() {
+    RefreshSuperUserGroupsConfigurationRequestProtoOrBuilder p = viaProto ? proto : builder;
+    boolean hasSubClusterId = p.hasSubClusterId();
+    if (hasSubClusterId) {
+      return p.getSubClusterId();
+    }
+    return null;
+  }
+
+  @Override
+  public void setSubClusterId(String subClusterId) {
+    maybeInitBuilder();
+    if (subClusterId == null) {
+      builder.clearSubClusterId();
+      return;
+    }
+    builder.setSubClusterId(subClusterId);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshUserToGroupsMappingsRequestPBImpl.java

@@ -18,21 +18,23 @@
 
 package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
 
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
 @Private
 @Unstable
-public class RefreshUserToGroupsMappingsRequestPBImpl 
-extends RefreshUserToGroupsMappingsRequest {
+public class RefreshUserToGroupsMappingsRequestPBImpl extends RefreshUserToGroupsMappingsRequest {
 
-  RefreshUserToGroupsMappingsRequestProto proto = RefreshUserToGroupsMappingsRequestProto.getDefaultInstance();
-  RefreshUserToGroupsMappingsRequestProto.Builder builder = null;
-  boolean viaProto = false;
+  private RefreshUserToGroupsMappingsRequestProto proto =
+      RefreshUserToGroupsMappingsRequestProto.getDefaultInstance();
+  private RefreshUserToGroupsMappingsRequestProto.Builder builder = null;
+  private boolean viaProto = false;
 
   public RefreshUserToGroupsMappingsRequestPBImpl() {
     builder = RefreshUserToGroupsMappingsRequestProto.newBuilder();
@@ -56,16 +58,46 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object other) {
-    if (other == null)
+
+    if (!(other instanceof RefreshUserToGroupsMappingsRequest)) {
       return false;
-    if (other.getClass().isAssignableFrom(this.getClass())) {
-      return this.getProto().equals(this.getClass().cast(other).getProto());
     }
-    return false;
+
+    RefreshUserToGroupsMappingsRequestPBImpl otherImpl = this.getClass().cast(other);
+    return new EqualsBuilder()
+        .append(this.getProto(), otherImpl.getProto())
+        .isEquals();
   }
 
   @Override
   public String toString() {
     return TextFormat.shortDebugString(getProto());
   }
+
+  private synchronized void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RefreshUserToGroupsMappingsRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  @Override
+  public String getSubClusterId() {
+    RefreshUserToGroupsMappingsRequestProtoOrBuilder p = viaProto ? proto : builder;
+    boolean hasSubClusterId = p.hasSubClusterId();
+    if (hasSubClusterId) {
+      return p.getSubClusterId();
+    }
+    return null;
+  }
+
+  @Override
+  public void setSubClusterId(String subClusterId) {
+    maybeInitBuilder();
+    if (subClusterId == null) {
+      builder.clearSubClusterId();
+      return;
+    }
+    builder.setSubClusterId(subClusterId);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java

@@ -135,6 +135,10 @@ public final class RouterMetrics {
   private MutableGaugeInt numRenewDelegationTokenFailedRetrieved;
   @Metric("# of renewDelegationToken failed to be retrieved")
   private MutableGaugeInt numCancelDelegationTokenFailedRetrieved;
+  @Metric("# of refreshSuperUserGroupsConfiguration failed to be retrieved")
+  private MutableGaugeInt numRefreshSuperUserGroupsConfigurationFailedRetrieved;
+  @Metric("# of refreshUserToGroupsMappings failed to be retrieved")
+  private MutableGaugeInt numRefreshUserToGroupsMappingsFailedRetrieved;
 
   // Aggregate metrics are shared, and don't have to be looked up per call
   @Metric("Total number of successful Submitted apps and latency(ms)")
@@ -231,6 +235,10 @@ public final class RouterMetrics {
   private MutableRate totalSucceededRenewDelegationTokenRetrieved;
   @Metric("Total number of successful Retrieved CancelDelegationToken and latency(ms)")
   private MutableRate totalSucceededCancelDelegationTokenRetrieved;
+  @Metric("Total number of successful Retrieved RefreshSuperUserGroupsConfig and latency(ms)")
+  private MutableRate totalSucceededRefreshSuperUserGroupsConfigurationRetrieved;
+  @Metric("Total number of successful Retrieved RefreshUserToGroupsMappings and latency(ms)")
+  private MutableRate totalSucceededRefreshUserToGroupsMappingsRetrieved;
 
   /**
    * Provide quantile counters for all latencies.
@@ -282,6 +290,8 @@ public final class RouterMetrics {
   private MutableQuantiles getDelegationTokenLatency;
   private MutableQuantiles renewDelegationTokenLatency;
   private MutableQuantiles cancelDelegationTokenLatency;
+  private MutableQuantiles refreshSuperUserGroupsConfLatency;
+  private MutableQuantiles refreshUserToGroupsMappingsLatency;
 
   private static volatile RouterMetrics instance = null;
   private static MetricsRegistry registry;
@@ -456,6 +466,11 @@ private RouterMetrics() {
     cancelDelegationTokenLatency = registry.newQuantiles("cancelDelegationTokenLatency",
         "latency of cancel delegation token timeouts", "ops", "latency", 10);
 
+    refreshSuperUserGroupsConfLatency = registry.newQuantiles("refreshSuperUserGroupsConfLatency",
+        "latency of refresh superuser groups configuration timeouts", "ops", "latency", 10);
+
+    refreshUserToGroupsMappingsLatency = registry.newQuantiles("refreshUserToGroupsMappingsLatency",
+        "latency of refresh user to groups mappings timeouts", "ops", "latency", 10);
   }
 
   public static RouterMetrics getMetrics() {
@@ -712,6 +727,11 @@ public long getNumSucceededCancelDelegationTokenRetrieved() {
     return totalSucceededCancelDelegationTokenRetrieved.lastStat().numSamples();
   }
 
+  @VisibleForTesting
+  public long getNumSucceededRefreshSuperUserGroupsConfigurationRetrieved() {
+    return totalSucceededRefreshSuperUserGroupsConfigurationRetrieved.lastStat().numSamples();
+  }
+
   @VisibleForTesting
   public double getLatencySucceededAppsCreated() {
     return totalSucceededAppsCreated.lastStat().mean();
@@ -947,6 +967,11 @@ public double getLatencySucceededCancelDelegationTokenRetrieved() {
     return totalSucceededCancelDelegationTokenRetrieved.lastStat().mean();
   }
 
+  @VisibleForTesting
+  public double getLatencySucceededRefreshSuperUserGroupsConfigurationRetrieved() {
+    return totalSucceededRefreshSuperUserGroupsConfigurationRetrieved.lastStat().mean();
+  }
+
   @VisibleForTesting
   public int getAppsFailedCreated() {
     return numAppsFailedCreated.value();
@@ -1145,6 +1170,14 @@ public int getNumRefreshNodesFailedRetrieved() {
     return numRefreshNodesFailedRetrieved.value();
   }
 
+  public int getNumRefreshSuperUserGroupsConfigurationFailedRetrieved() {
+    return numRefreshSuperUserGroupsConfigurationFailedRetrieved.value();
+  }
+
+  public int getNumRefreshUserToGroupsMappingsFailedRetrieved() {
+    return numRefreshUserToGroupsMappingsFailedRetrieved.value();
+  }
+
   public int getDelegationTokenFailedRetrieved() {
     return numGetDelegationTokenFailedRetrieved.value();
   }
@@ -1392,6 +1425,16 @@ public void succeededCancelDelegationTokenRetrieved(long duration) {
     cancelDelegationTokenLatency.add(duration);
   }
 
+  public void succeededRefreshSuperUserGroupsConfRetrieved(long duration) {
+    totalSucceededRefreshSuperUserGroupsConfigurationRetrieved.add(duration);
+    refreshSuperUserGroupsConfLatency.add(duration);
+  }
+
+  public void succeededRefreshUserToGroupsMappingsRetrieved(long duration) {
+    totalSucceededRefreshUserToGroupsMappingsRetrieved.add(duration);
+    refreshUserToGroupsMappingsLatency.add(duration);
+  }
+
   public void incrAppsFailedCreated() {
     numAppsFailedCreated.incr();
   }
@@ -1568,6 +1611,14 @@ public void incrRefreshNodesFailedRetrieved() {
     numRefreshNodesFailedRetrieved.incr();
   }
 
+  public void incrRefreshSuperUserGroupsConfigurationFailedRetrieved() {
+    numRefreshSuperUserGroupsConfigurationFailedRetrieved.incr();
+  }
+
+  public void incrRefreshUserToGroupsMappingsFailedRetrieved() {
+    numRefreshUserToGroupsMappingsFailedRetrieved.incr();
+  }
+
   public void incrGetDelegationTokenFailedRetrieved() {
     numGetDelegationTokenFailedRetrieved.incr();
   }