Skip to content

Commit f5edc1b

Browse files
slfan1989slfan1989
committed
YARN-7707. BackPort [GPG] Policy generator framework.
1 parent 680af87 commit f5edc1b

File tree

18 files changed

+1776
-2
lines changed

18 files changed

+1776
-2
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4390,6 +4390,37 @@ public static boolean isAclEnabled(Configuration conf) {
43904390
public static final String GPG_KERBEROS_PRINCIPAL_HOSTNAME_KEY = FEDERATION_GPG_PREFIX +
43914391
"kerberos.principal.hostname";
43924392

4393+
public static final String FEDERATION_GPG_POLICY_PREFIX =
4394+
FEDERATION_GPG_PREFIX + "policy.generator.";
4395+
4396+
/** The interval at which the policy generator runs, default is one hour. */
4397+
public static final String GPG_POLICY_GENERATOR_INTERVAL_MS =
4398+
FEDERATION_GPG_POLICY_PREFIX + "interval-ms";
4399+
public static final long DEFAULT_GPG_POLICY_GENERATOR_INTERVAL_MS = TimeUnit.HOURS.toMillis(1);
4400+
4401+
/**
4402+
* The configured policy generator class, runs NoOpGlobalPolicy by
4403+
* default.
4404+
*/
4405+
public static final String GPG_GLOBAL_POLICY_CLASS = FEDERATION_GPG_POLICY_PREFIX + "class";
4406+
public static final String DEFAULT_GPG_GLOBAL_POLICY_CLASS =
4407+
"org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator." +
4408+
"NoOpGlobalPolicy";
4409+
4410+
/**
4411+
* Whether or not the policy generator is running in read only (won't modify
4412+
* policies), default is false.
4413+
*/
4414+
public static final String GPG_POLICY_GENERATOR_READONLY =
4415+
FEDERATION_GPG_POLICY_PREFIX + "readonly";
4416+
public static final boolean DEFAULT_GPG_POLICY_GENERATOR_READONLY = false;
4417+
4418+
/**
4419+
* Which sub-clusters the policy generator should blacklist.
4420+
*/
4421+
public static final String GPG_POLICY_GENERATOR_BLACKLIST =
4422+
FEDERATION_GPG_POLICY_PREFIX + "blacklist";
4423+
43934424
/**
43944425
* Connection and Read timeout from the Router to RM.
43954426
*/

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5408,6 +5408,14 @@
54085408
</description>
54095409
</property>
54105410

5411+
<property>
5412+
<description>
5413+
The interval at which the policy generator runs, default is one hour
5414+
</description>
5415+
<name>yarn.federation.gpg.policy.generator.interval-ms</name>
5416+
<value>1h</value>
5417+
</property>
5418+
54115419
<property>
54125420
<description>
54135421
The number of retry for Register UAM.
@@ -5419,6 +5427,30 @@
54195427

54205428
<property>
54215429
<description>
5430+
The configured policy generator class, runs NoOpGlobalPolicy by default
5431+
</description>
5432+
<name>yarn.federation.gpg.policy.generator.class</name>
5433+
<value>org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.NoOpGlobalPolicy</value>
5434+
</property>
5435+
5436+
<property>
5437+
<description>
5438+
Whether or not the policy generator is running in read only (won't modify policies), default is false
5439+
</description>
5440+
<name>yarn.federation.gpg.policy.generator.readonly</name>
5441+
<value>false</value>
5442+
</property>
5443+
5444+
<property>
5445+
<description>
5446+
Which subclusters the gpg should blacklist, default is none
5447+
</description>
5448+
<name>yarn.federation.gpg.policy.generator.blacklist</name>
5449+
<value></value>
5450+
</property>
5451+
5452+
<property>
5453+
<description>
54225454
Interval between retry for Register UAM.
54235455
The default value is 100ms.
54245456
</description>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
6666
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
6767
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
68+
import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
6869
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
6970
import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
7071
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -311,6 +312,18 @@ public SubClusterPolicyConfiguration getPolicyConfiguration(final String queue)
311312
}
312313
}
313314

315+
/**
316+
* Set a policy configuration into the state store.
317+
*
318+
* @param policyConf the policy configuration to set
319+
* @throws YarnException if the request is invalid/fails
320+
*/
321+
public void setPolicyConfiguration(SubClusterPolicyConfiguration policyConf)
322+
throws YarnException {
323+
stateStore.setPolicyConfiguration(
324+
SetSubClusterPolicyConfigurationRequest.newInstance(policyConf));
325+
}
326+
314327
/**
315328
* Get the policies that is represented as
316329
* {@link SubClusterPolicyConfiguration} for all currently active queues in

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@
6161
<scope>test</scope>
6262
</dependency>
6363

64+
<dependency>
65+
<groupId>org.apache.hadoop</groupId>
66+
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
67+
<scope>provided</scope>
68+
</dependency>
69+
6470
<dependency>
6571
<groupId>org.apache.hadoop</groupId>
6672
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
@@ -72,6 +78,12 @@
7278
<scope>test</scope>
7379
</dependency>
7480

81+
<dependency>
82+
<groupId>org.mockito</groupId>
83+
<artifactId>mockito-all</artifactId>
84+
<scope>test</scope>
85+
</dependency>
86+
7587
<dependency>
7688
<groupId>org.apache.hadoop</groupId>
7789
<artifactId>hadoop-yarn-server-common</artifactId>
@@ -106,6 +118,12 @@
106118
<plugin>
107119
<groupId>org.apache.rat</groupId>
108120
<artifactId>apache-rat-plugin</artifactId>
121+
<configuration>
122+
<excludes>
123+
<exclude>src/test/resources/schedulerInfo1.json</exclude>
124+
<exclude>src/test/resources/schedulerInfo2.json</exclude>
125+
</excludes>
126+
</configuration>
109127
</plugin>
110128
</plugins>
111129
</build>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,8 @@ public interface GPGContext {
2828
FederationStateStoreFacade getStateStoreFacade();
2929

3030
void setStateStoreFacade(FederationStateStoreFacade facade);
31+
32+
GPGPolicyFacade getPolicyFacade();
33+
34+
void setPolicyFacade(GPGPolicyFacade facade);
3135
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
public class GPGContextImpl implements GPGContext {
2727

2828
private FederationStateStoreFacade facade;
29+
private GPGPolicyFacade policyFacade;
2930

3031
@Override
3132
public FederationStateStoreFacade getStateStoreFacade() {
@@ -38,4 +39,13 @@ public void setStateStoreFacade(
3839
this.facade = federationStateStoreFacade;
3940
}
4041

42+
@Override
43+
public GPGPolicyFacade getPolicyFacade(){
44+
return policyFacade;
45+
}
46+
47+
@Override
48+
public void setPolicyFacade(GPGPolicyFacade gpgPolicyfacade){
49+
policyFacade = gpgPolicyfacade;
50+
}
4151
}

0 commit comments

Comments
 (0)