From 6c5861b55db604920fcdd447076c831be3c9f7da Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Wed, 22 May 2024 13:15:40 -0700 Subject: [PATCH] initial commit --- .../apache/spark/deploy/SparkSubmitArguments.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 61235a7019070..15565e19b3d24 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -126,13 +126,22 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S */ private def mergeDefaultSparkProperties(): Unit = { // Use common defaults file, if not specified by user - propertiesFile = Option(propertiesFile).getOrElse(Utils.getDefaultPropertiesFile(env)) // Honor --conf before the defaults file defaultSparkProperties.foreach { case (k, v) => if (!sparkProperties.contains(k)) { sparkProperties(k) = v } } + + // Also load properties from `spark-defaults.conf` if they do not exist in the properties file + Option(Utils.getDefaultPropertiesFile(env)).foreach { filename => + val properties = Utils.getPropertiesFromFile(filename) + properties.foreach { case (k, v) => + if (!sparkProperties.contains(k)) { + sparkProperties(k) = v + } + } + } } /**