Skip to content

Commit 567823e

Browse files
ashangitMridul Muralidharan
authored andcommitted
[SPARK-44242] Improve Max Heap not set check
### What changes were proposed in this pull request? Currently if any Xmx string is available in the driver java options even if not related to Max Heap setting the job sumission failed For Ex. this command failed `./bin/spark-submit --class org.apache.spark.examples.SparkPi --conf "spark.driver.extraJavaOptions=-Dtest=Xmx" examples/jars/spark-examples_2.12-3.4.1.jar 100` ### Why are the changes needed? The command should any failed if the -Xmx argument is set not if it is part of another parameter or string ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added a unitary test Closes #41806 from ashangit/nfraison/SPARK-44242. Authored-by: Nicolas Fraison <nicolas.fraison@datadoghq.com> Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
1 parent 1d02ca4 commit 567823e

File tree

2 files changed

+39
-5
lines changed

2 files changed

+39
-5
lines changed

launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -311,11 +311,15 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env)
311311
}
312312

313313
private void checkJavaOptions(String javaOptions) {
314-
if (!isEmpty(javaOptions) && javaOptions.contains("Xmx")) {
315-
String msg = String.format("Not allowed to specify max heap(Xmx) memory settings through " +
316-
"java options (was %s). Use the corresponding --driver-memory or " +
317-
"spark.driver.memory configuration instead.", javaOptions);
318-
throw new IllegalArgumentException(msg);
314+
if (!isEmpty(javaOptions)) {
315+
for (String javaOption: CommandBuilderUtils.parseOptionString(javaOptions)) {
316+
if (javaOption.startsWith("-Xmx")) {
317+
String msg = String.format("Not allowed to specify max heap(Xmx) memory settings " +
318+
"through java options (was %s). Use the corresponding --driver-memory or " +
319+
"spark.driver.memory configuration instead.", javaOptions);
320+
throw new IllegalArgumentException(msg);
321+
}
322+
}
319323
}
320324
}
321325

launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,36 @@ public void testCliHelpAndNoArg() throws Exception {
7272
cmd.contains("org.apache.spark.deploy.SparkSubmit"));
7373
}
7474

75+
@Test(expected = IllegalArgumentException.class)
76+
public void testCheckJavaOptionsThrowException() throws Exception {
77+
Map<String, String> env = new HashMap<>();
78+
List<String> sparkSubmitArgs = Arrays.asList(
79+
parser.MASTER,
80+
"local",
81+
parser.DRIVER_CLASS_PATH,
82+
"/driverCp",
83+
parser.DRIVER_JAVA_OPTIONS,
84+
"-Xmx64g -Dprop=Other -Dprop1=\"-Xmx -Xmx\" -Dprop2=\"-Xmx '-Xmx\" " +
85+
"-Dprop3='-Xmx -Xmx' -Dprop4='-Xmx \"-Xmx'",
86+
SparkLauncher.NO_RESOURCE);
87+
buildCommand(sparkSubmitArgs, env);
88+
}
89+
90+
@Test
91+
public void testCheckJavaOptions() throws Exception {
92+
Map<String, String> env = new HashMap<>();
93+
List<String> sparkSubmitArgs = Arrays.asList(
94+
parser.MASTER,
95+
"local",
96+
parser.DRIVER_CLASS_PATH,
97+
"/driverCp",
98+
parser.DRIVER_JAVA_OPTIONS,
99+
"-Dprop=-Xmx -Dprop1=\"-Xmx -Xmx\" -Dprop2=\"-Xmx '-Xmx\" " +
100+
"-Dprop3='-Xmx -Xmx' -Dprop4='-Xmx \"-Xmx'",
101+
SparkLauncher.NO_RESOURCE);
102+
buildCommand(sparkSubmitArgs, env);
103+
}
104+
75105
@Test
76106
public void testCliKillAndStatus() throws Exception {
77107
List<String> params = Arrays.asList("driver-20160531171222-0000");

0 commit comments

Comments
 (0)