YARN-9546. Add configuration option for YARN Native services AM classpath. Contributed by Gergely Pollak.

Author: sunilgovind

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java

@@ -1234,11 +1234,15 @@ private String buildCommandLine(Service app, Configuration conf,
     return cmdStr;
   }
 
-  private Map<String, String> addAMEnv() throws IOException {
+  @VisibleForTesting
+  protected Map<String, String> addAMEnv() throws IOException {
     Map<String, String> env = new HashMap<>();
-    ClasspathConstructor classpath =
-        buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
-            .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
+    ClasspathConstructor classpath = buildClasspath(
+        YarnServiceConstants.SUBMITTED_CONF_DIR,
+        "lib",
+        fs,
+        getConfig().get(YarnServiceConf.YARN_SERVICE_CLASSPATH, ""),
+        getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
     env.put("CLASSPATH", classpath.buildClasspath());
     env.put("LANG", "en_US.UTF-8");
     env.put("LC_ALL", "en_US.UTF-8");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java

@@ -60,6 +60,8 @@ public class YarnServiceConf {
   public static final String ROLLING_LOG_INCLUSION_PATTERN = "yarn.service.rolling-log.include-pattern";
   public static final String ROLLING_LOG_EXCLUSION_PATTERN = "yarn.service.rolling-log.exclude-pattern";
 
+  public static final String YARN_SERVICE_CLASSPATH = "yarn.service.classpath";
+
   public static final String YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY =
       YARN_SERVICE_PREFIX + "system-service.dir";
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java

@@ -451,13 +451,15 @@ public static Path createLocalPath(File file) {
    * @param sliderConfDir relative path to the dir containing slider config
    *                      options to put on the classpath -or null
    * @param libdir directory containing the JAR files
+   * @param configClassPath extra class path configured in yarn-site.xml
    * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
    * (and hence the current classpath should be used, not anything built up)
    * @return a classpath
    */
   public static ClasspathConstructor buildClasspath(String sliderConfDir,
       String libdir,
       SliderFileSystem sliderFileSystem,
+      String configClassPath,
       boolean usingMiniMRCluster) {
 
     ClasspathConstructor classpath = new ClasspathConstructor();
@@ -479,6 +481,11 @@ public static ClasspathConstructor buildClasspath(String sliderConfDir,
       classpath.addRemoteClasspathEnvVar();
       classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
     }
+
+    if (!configClassPath.isEmpty()) {
+      classpath.appendAll(Arrays.asList(configClassPath.split(",")));
+    }
+
     return classpath;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java

@@ -76,6 +76,29 @@ public class TestServiceClient {
   public ServiceTestUtils.ServiceFSWatcher rule =
       new ServiceTestUtils.ServiceFSWatcher();
 
+  @Test
+  public void testAMEnvCustomClasspath() throws Exception {
+    Service service = createService();
+    service.getComponents().forEach(comp ->
+            comp.setRestartPolicy(Component.RestartPolicyEnum.NEVER));
+    ServiceClient client = MockServiceClient.create(rule, service, true);
+    //saving the original value of the param, for restoration purposes
+    String oldParam = client.getConfig().get("yarn.service.classpath", "");
+    String originalPath = client.addAMEnv().get("CLASSPATH");
+
+    client.getConfig().set("yarn.service.classpath", "{{VAR_1}},{{VAR_2}}");
+    String newPath = client.addAMEnv().get("CLASSPATH");
+
+    Assert.assertEquals(originalPath + "<CPS>{{VAR_1}}<CPS>{{VAR_2}}", newPath);
+    //restoring the original value for service classpath
+    client.getConfig().set("yarn.service.classpath", oldParam);
+
+    newPath = client.addAMEnv().get("CLASSPATH");
+    Assert.assertEquals(originalPath, newPath);
+
+    client.stop();
+  }
+
   @Test
   public void testUpgradeDisabledByDefault() throws Exception {
     Service service = createService();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -4211,4 +4211,14 @@
     <name>yarn.resourcemanager.activities-manager.app-activities.max-queue-length</name>
     <value>1000</value>
   </property>
+
+  <property>
+    <description>
+      Comma separated extra class path parameters for yarn services AM.
+      These path elements will be appended to the end of the YARN service AM
+      classpath.
+    </description>
+    <name>yarn.service.classpath</name>
+    <value></value>
+  </property>
 </configuration>