add livy-fairscheduler.xml based on test/client/cluster

apache · Mar 13, 2019 · 78c1c90 · 78c1c90
1 parent 97aa7c3
commit 78c1c90
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 4 deletions.
diff --git a/bin/livy-server b/bin/livy-server
@@ -21,7 +21,7 @@
 usage="Usage: livy-server (start|stop|status)"
 
 export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
-export LIVY_CONF_DIR=${LIVY_CONF_DIR:-"$LIVY_HOME/conf"}
+LIVY_CONF_DIR=${LIVY_CONF_DIR:-"$LIVY_HOME/conf"}
 
 if [ -f "${LIVY_CONF_DIR}/livy-env.sh" ]; then
   # Promote all variable declarations to environment (exported) variables

diff --git a/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java
@@ -202,7 +202,20 @@ private static ChildProcess startDriver(final RSCConf conf, Promise<?> promise)
     conf.set("spark.yarn.maxAppAttempts", "1");
 
     conf.set("spark.scheduler.mode", "FAIR");
-    conf.set("spark.scheduler.allocation.file", FAIR_SCHEDULER_FILENAME);
+    String master = conf.get(SparkLauncher.SPARK_MASTER);
+    String deployMode = conf.get(SparkLauncher.DEPLOY_MODE);
+    Boolean isCluster =  (master != null && master.endsWith("-cluster")) ||
+        (deployMode != null && deployMode.equals("cluster"));
+    String fairSchedulerXMLPath = System.getenv("LIVY_HOME")
+        + File.separator + "conf" + File.separator + FAIR_SCHEDULER_FILENAME;
+    if (conf.getBoolean(CLIENT_IN_PROCESS)) {
+      conf.set("spark.scheduler.allocation.file",
+          ClassLoader.getSystemClassLoader().getResource(FAIR_SCHEDULER_FILENAME).getPath());
+    } else if (isCluster) {
+      conf.set("spark.scheduler.allocation.file", FAIR_SCHEDULER_FILENAME);
+    } else if (master != null){
+      conf.set("spark.scheduler.allocation.file", fairSchedulerXMLPath);
+    }
 
     // Let the launcher go away when launcher in yarn cluster mode. This avoids keeping lots
     // of "small" Java processes lingering on the Livy server node.
@@ -243,8 +256,9 @@ public void run() {
     } else {
       final SparkLauncher launcher = new SparkLauncher();
       launcher.setSparkHome(System.getenv(SPARK_HOME_ENV));
-      launcher.addFile(
-        System.getenv().get("LIVY_CONF_DIR") + File.separator + FAIR_SCHEDULER_FILENAME);
+      if (isCluster) {
+        launcher.addFile(fairSchedulerXMLPath);
+      }
       launcher.setAppResource(SparkLauncher.NO_RESOURCE);
       launcher.setPropertiesFile(confFile.getAbsolutePath());
       launcher.setMainClass(RSCDriverBootstrapper.class.getName());

diff --git a/rsc/src/test/resources/livy-fairscheduler.xml b/rsc/src/test/resources/livy-fairscheduler.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<allocations>
+  <pool name="default">
+    <schedulingMode>FIFO</schedulingMode>
+    <weight>1</weight>
+    <minShare>0</minShare>
+  </pool>
+  <pool name="fair">
+    <schedulingMode>FAIR</schedulingMode>
+    <weight>1</weight>
+    <minShare>0</minShare>
+  </pool>
+</allocations>
diff --git a/server/src/test/resources/livy-fairscheduler.xml b/server/src/test/resources/livy-fairscheduler.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<allocations>
+  <pool name="default">
+    <schedulingMode>FIFO</schedulingMode>
+    <weight>1</weight>
+    <minShare>0</minShare>
+  </pool>
+  <pool name="fair">
+    <schedulingMode>FAIR</schedulingMode>
+    <weight>1</weight>
+    <minShare>0</minShare>
+  </pool>
+</allocations>