Skip to content

Commit 4fb25ed

Browse files
let thrift server take SPARK_DAEMON_MEMORY and SPARK_DAEMON_JAVA_OPTS
1 parent 1f2f723 commit 4fb25ed

File tree

1 file changed

+22
-2
lines changed

1 file changed

+22
-2
lines changed

launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,10 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env) throws IOE
190190
firstNonEmptyValue(SparkLauncher.DRIVER_EXTRA_CLASSPATH, conf, props) : null;
191191

192192
List<String> cmd = buildJavaCommand(extraClassPath);
193+
// Take Thrift Server as daemon
194+
if (isThriftServer(mainClass)) {
195+
addOptionString(cmd, System.getenv("SPARK_DAEMON_JAVA_OPTS"));
196+
}
193197
addOptionString(cmd, System.getenv("SPARK_SUBMIT_OPTS"));
194198
addOptionString(cmd, System.getenv("SPARK_JAVA_OPTS"));
195199

@@ -201,8 +205,16 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env) throws IOE
201205
// - SPARK_DRIVER_MEMORY env variable
202206
// - SPARK_MEM env variable
203207
// - default value (512m)
204-
String memory = firstNonEmpty(firstNonEmptyValue(SparkLauncher.DRIVER_MEMORY, conf, props),
205-
System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
208+
String memory;
209+
// Take Thrift Server as daemon
210+
if (isThriftServer(mainClass)) {
211+
memory = firstNonEmpty(System.getenv("SPARK_DAEMON_MEMORY"),
212+
firstNonEmptyValue(SparkLauncher.DRIVER_MEMORY, conf, props),
213+
System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
214+
} else {
215+
memory = firstNonEmpty(firstNonEmptyValue(SparkLauncher.DRIVER_MEMORY, conf, props),
216+
System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
217+
}
206218
cmd.add("-Xms" + memory);
207219
cmd.add("-Xmx" + memory);
208220
addOptionString(cmd, firstNonEmptyValue(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, conf, props));
@@ -292,6 +304,14 @@ private boolean isClientMode(Properties userProps) {
292304
(!userMaster.equals("yarn-cluster") && deployMode == null);
293305
}
294306

307+
/**
308+
* Return whether the given main class represents a thrift server.
309+
*/
310+
private boolean isThriftServer(String mainClass) {
311+
return mainClass.equals("org.apache.spark.sql.hive.thriftserver.HiveThriftServer2");
312+
}
313+
314+
295315
private class OptionParser extends SparkSubmitOptionParser {
296316

297317
@Override

0 commit comments

Comments
 (0)