Skip to content

Commit 91765ab

Browse files
author
Marcelo Vanzin
committed
Make mainAppResource not optional.
1 parent 1a61f72 commit 91765ab

14 files changed

+62
-59
lines changed

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,15 @@ private[spark] sealed trait KubernetesRoleSpecificConf
3838
* Structure containing metadata for Kubernetes logic that builds a Spark driver.
3939
*/
4040
private[spark] case class KubernetesDriverSpecificConf(
41-
mainAppResource: Option[MainAppResource],
41+
mainAppResource: MainAppResource,
4242
mainClass: String,
4343
appName: String,
4444
appArgs: Seq[String],
45-
pyFiles: Seq[String] = Nil) extends KubernetesRoleSpecificConf
45+
pyFiles: Seq[String] = Nil) extends KubernetesRoleSpecificConf {
46+
47+
require(mainAppResource != null, "Main resource must be provided.")
48+
49+
}
4650

4751
/*
4852
* Structure containing metadata for Kubernetes logic that builds a Spark executor.
@@ -115,7 +119,7 @@ private[spark] object KubernetesConf {
115119
appName: String,
116120
appResourceNamePrefix: String,
117121
appId: String,
118-
mainAppResource: Option[MainAppResource],
122+
mainAppResource: MainAppResource,
119123
mainClass: String,
120124
appArgs: Array[String],
121125
maybePyFiles: Option[String],

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,17 +51,16 @@ private[spark] class BasicDriverFeatureStep(
5151

5252
// The memory overhead factor to use. If the user has not set it, then use a different
5353
// value for non-JVM apps. This value is propagated to executors.
54-
private val overheadFactor = conf.roleSpecificConf.mainAppResource match {
55-
case Some(_: NonJVMResource) =>
54+
private val overheadFactor =
55+
if (conf.roleSpecificConf.mainAppResource.isInstanceOf[NonJVMResource]) {
5656
if (conf.sparkConf.contains(MEMORY_OVERHEAD_FACTOR)) {
5757
conf.get(MEMORY_OVERHEAD_FACTOR)
5858
} else {
5959
NON_JVM_MEMORY_OVERHEAD_FACTOR
6060
}
61-
62-
case _ =>
61+
} else {
6362
conf.get(MEMORY_OVERHEAD_FACTOR)
64-
}
63+
}
6564

6665
private val memoryOverheadMiB = conf
6766
.get(DRIVER_MEMORY_OVERHEAD)

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,30 +39,27 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesConf[KubernetesDri
3939

4040
override def configurePod(pod: SparkPod): SparkPod = {
4141
driverConf.mainAppResource match {
42-
case Some(JavaMainAppResource(_)) | None =>
42+
case JavaMainAppResource(_) =>
4343
configureForJava(pod)
4444

45-
case Some(PythonMainAppResource(res)) =>
45+
case PythonMainAppResource(res) =>
4646
configureForPython(pod, res)
4747

48-
case Some(RMainAppResource(res)) =>
48+
case RMainAppResource(res) =>
4949
configureForR(pod, res)
5050
}
5151
}
5252

5353
override def getAdditionalPodSystemProperties(): Map[String, String] = {
5454
driverConf.mainAppResource match {
55-
case Some(JavaMainAppResource(res)) =>
56-
additionalJavaProperties(res)
55+
case JavaMainAppResource(res) =>
56+
res.map(additionalJavaProperties).getOrElse(Map.empty)
5757

58-
case Some(PythonMainAppResource(res)) =>
58+
case PythonMainAppResource(res) =>
5959
additionalPythonProperties(res)
6060

61-
case Some(RMainAppResource(res)) =>
61+
case RMainAppResource(res) =>
6262
additionalRProperties(res)
63-
64-
case None =>
65-
Map.empty
6663
}
6764
}
6865

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import org.apache.spark.util.Utils
4444
* @param maybePyFiles additional Python files via --py-files
4545
*/
4646
private[spark] case class ClientArguments(
47-
mainAppResource: Option[MainAppResource],
47+
mainAppResource: MainAppResource,
4848
mainClass: String,
4949
driverArgs: Array[String],
5050
maybePyFiles: Option[String],
@@ -53,18 +53,18 @@ private[spark] case class ClientArguments(
5353
private[spark] object ClientArguments {
5454

5555
def fromCommandLineArgs(args: Array[String]): ClientArguments = {
56-
var mainAppResource: Option[MainAppResource] = None
56+
var mainAppResource: MainAppResource = JavaMainAppResource(None)
5757
var mainClass: Option[String] = None
5858
val driverArgs = mutable.ArrayBuffer.empty[String]
5959
var maybePyFiles : Option[String] = None
6060

6161
args.sliding(2, 2).toList.foreach {
6262
case Array("--primary-java-resource", primaryJavaResource: String) =>
63-
mainAppResource = Some(JavaMainAppResource(primaryJavaResource))
63+
mainAppResource = JavaMainAppResource(Some(primaryJavaResource))
6464
case Array("--primary-py-file", primaryPythonResource: String) =>
65-
mainAppResource = Some(PythonMainAppResource(primaryPythonResource))
65+
mainAppResource = PythonMainAppResource(primaryPythonResource)
6666
case Array("--primary-r-file", primaryRFile: String) =>
67-
mainAppResource = Some(RMainAppResource(primaryRFile))
67+
mainAppResource = RMainAppResource(primaryRFile)
6868
case Array("--other-py-files", pyFiles: String) =>
6969
maybePyFiles = Some(pyFiles)
7070
case Array("--main-class", clazz: String) =>

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/MainAppResource.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ private[spark] sealed trait MainAppResource
2020

2121
private[spark] sealed trait NonJVMResource
2222

23-
private[spark] case class JavaMainAppResource(primaryResource: String) extends MainAppResource
23+
private[spark] case class JavaMainAppResource(primaryResource: Option[String])
24+
extends MainAppResource
2425

2526
private[spark] case class PythonMainAppResource(primaryResource: String)
2627
extends MainAppResource with NonJVMResource

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ class KubernetesConfSuite extends SparkFunSuite {
5656
APP_NAME,
5757
RESOURCE_NAME_PREFIX,
5858
APP_ID,
59-
mainAppResource = None,
59+
mainAppResource = JavaMainAppResource(None),
6060
MAIN_CLASS,
6161
APP_ARGS,
6262
maybePyFiles = None,
@@ -65,7 +65,6 @@ class KubernetesConfSuite extends SparkFunSuite {
6565
assert(conf.sparkConf.getAll.toMap === sparkConf.getAll.toMap)
6666
assert(conf.appResourceNamePrefix === RESOURCE_NAME_PREFIX)
6767
assert(conf.roleSpecificConf.appName === APP_NAME)
68-
assert(conf.roleSpecificConf.mainAppResource.isEmpty)
6968
assert(conf.roleSpecificConf.mainClass === MAIN_CLASS)
7069
assert(conf.roleSpecificConf.appArgs === APP_ARGS)
7170
}
@@ -94,7 +93,7 @@ class KubernetesConfSuite extends SparkFunSuite {
9493
APP_NAME,
9594
RESOURCE_NAME_PREFIX,
9695
APP_ID,
97-
mainAppResource = None,
96+
mainAppResource = JavaMainAppResource(None),
9897
MAIN_CLASS,
9998
APP_ARGS,
10099
maybePyFiles = None,

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
5252
new LocalObjectReferenceBuilder().withName(secret).build()
5353
}
5454
private val emptyDriverSpecificConf = KubernetesDriverSpecificConf(
55-
None,
55+
JavaMainAppResource(None),
5656
APP_NAME,
5757
MAIN_CLASS,
5858
APP_ARGS)
@@ -144,7 +144,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
144144
val javaKubernetesConf = KubernetesConf(
145145
javaSparkConf,
146146
KubernetesDriverSpecificConf(
147-
Some(JavaMainAppResource("")),
147+
JavaMainAppResource(None),
148148
APP_NAME,
149149
PY_MAIN_CLASS,
150150
APP_ARGS),
@@ -161,7 +161,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
161161
val pythonKubernetesConf = KubernetesConf(
162162
pythonSparkConf,
163163
KubernetesDriverSpecificConf(
164-
Some(PythonMainAppResource("")),
164+
PythonMainAppResource(""),
165165
APP_NAME,
166166
PY_MAIN_CLASS,
167167
APP_ARGS),
@@ -220,12 +220,10 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
220220
// Memory overhead tests. Tuples are:
221221
// test name, main resource, overhead factor, expected factor
222222
Seq(
223-
("java w/o resource", None, None, MEMORY_OVERHEAD_FACTOR.defaultValue.get),
224-
("java w/ resource", Some(JavaMainAppResource(null)), None,
225-
MEMORY_OVERHEAD_FACTOR.defaultValue.get),
226-
("python default", Some(PythonMainAppResource(null)), None, NON_JVM_MEMORY_OVERHEAD_FACTOR),
227-
("python w/ override", Some(PythonMainAppResource(null)), Some(0.9d), 0.9d),
228-
("r default", Some(RMainAppResource(null)), None, NON_JVM_MEMORY_OVERHEAD_FACTOR)
223+
("java", JavaMainAppResource(None), None, MEMORY_OVERHEAD_FACTOR.defaultValue.get),
224+
("python default", PythonMainAppResource(null), None, NON_JVM_MEMORY_OVERHEAD_FACTOR),
225+
("python w/ override", PythonMainAppResource(null), Some(0.9d), 0.9d),
226+
("r default", RMainAppResource(null), None, NON_JVM_MEMORY_OVERHEAD_FACTOR)
229227
).foreach { case (name, resource, factor, expectedFactor) =>
230228
test(s"memory overhead factor: $name") {
231229
// Choose a driver memory where the default memory overhead is > MEMORY_OVERHEAD_MIN_MIB

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
4343
test("java resource") {
4444
val mainResource = "local:///main.jar"
4545
val spec = applyFeatureStep(
46-
resource = Some(JavaMainAppResource(mainResource)),
46+
resource = JavaMainAppResource(Some(mainResource)),
4747
appArgs = Array("5", "7"))
4848
assert(spec.pod.container.getArgs.asScala === List(
4949
"driver",
@@ -62,7 +62,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
6262

6363
val spec = applyFeatureStep(
6464
conf = sparkConf,
65-
resource = Some(PythonMainAppResource(mainResource)))
65+
resource = PythonMainAppResource(mainResource))
6666
assert(spec.pod.container.getArgs.asScala === List(
6767
"driver",
6868
"--properties-file", SPARK_CONF_PATH,
@@ -90,7 +90,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
9090
.set(PYSPARK_MAJOR_PYTHON_VERSION, "2")
9191
val spec = applyFeatureStep(
9292
conf = sparkConf,
93-
resource = Some(PythonMainAppResource(mainResource)),
93+
resource = PythonMainAppResource(mainResource),
9494
appArgs = Array("5", "7", "9"),
9595
pyFiles = pyFiles)
9696

@@ -117,7 +117,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
117117
val mainResource = s"local://$expectedMainResource"
118118

119119
val spec = applyFeatureStep(
120-
resource = Some(RMainAppResource(mainResource)),
120+
resource = RMainAppResource(mainResource),
121121
appArgs = Array("5", "7", "9"))
122122

123123
assert(spec.pod.container.getArgs.asScala === List(
@@ -129,7 +129,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
129129

130130
private def applyFeatureStep(
131131
conf: SparkConf = new SparkConf(false),
132-
resource: Option[MainAppResource] = None,
132+
resource: MainAppResource = JavaMainAppResource(None),
133133
appArgs: Array[String] = Array(),
134134
pyFiles: Seq[String] = Nil): KubernetesDriverSpec = {
135135
val driverConf = new KubernetesDriverSpecificConf(

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
2626
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, SparkPod}
2727
import org.apache.spark.deploy.k8s.Config._
2828
import org.apache.spark.deploy.k8s.Constants._
29+
import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
2930
import org.apache.spark.util.Clock
3031

3132
class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
@@ -59,7 +60,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
5960
KubernetesConf(
6061
sparkConf,
6162
KubernetesDriverSpecificConf(
62-
None, "main", "app", Seq.empty),
63+
JavaMainAppResource(None), "main", "app", Seq.empty),
6364
SHORT_RESOURCE_NAME_PREFIX,
6465
"app-id",
6566
DRIVER_LABELS,
@@ -91,7 +92,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
9192
.set(org.apache.spark.internal.config.DRIVER_BLOCK_MANAGER_PORT, 8080)
9293
.set(KUBERNETES_NAMESPACE, "my-namespace"),
9394
KubernetesDriverSpecificConf(
94-
None, "main", "app", Seq.empty),
95+
JavaMainAppResource(None), "main", "app", Seq.empty),
9596
SHORT_RESOURCE_NAME_PREFIX,
9697
"app-id",
9798
DRIVER_LABELS,
@@ -113,7 +114,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
113114
KubernetesConf(
114115
sparkConf,
115116
KubernetesDriverSpecificConf(
116-
None, "main", "app", Seq.empty),
117+
JavaMainAppResource(None), "main", "app", Seq.empty),
117118
SHORT_RESOURCE_NAME_PREFIX,
118119
"app-id",
119120
DRIVER_LABELS,
@@ -144,7 +145,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
144145
KubernetesConf(
145146
sparkConf.set(KUBERNETES_NAMESPACE, "my-namespace"),
146147
KubernetesDriverSpecificConf(
147-
None, "main", "app", Seq.empty),
148+
JavaMainAppResource(None), "main", "app", Seq.empty),
148149
LONG_RESOURCE_NAME_PREFIX,
149150
"app-id",
150151
DRIVER_LABELS,
@@ -172,7 +173,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
172173
KubernetesConf(
173174
sparkConf.set(org.apache.spark.internal.config.DRIVER_BIND_ADDRESS, "host"),
174175
KubernetesDriverSpecificConf(
175-
None, "main", "app", Seq.empty),
176+
JavaMainAppResource(None), "main", "app", Seq.empty),
176177
LONG_RESOURCE_NAME_PREFIX,
177178
"app-id",
178179
DRIVER_LABELS,
@@ -198,7 +199,7 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
198199
KubernetesConf(
199200
sparkConf,
200201
KubernetesDriverSpecificConf(
201-
None, "main", "app", Seq.empty),
202+
JavaMainAppResource(None), "main", "app", Seq.empty),
202203
LONG_RESOURCE_NAME_PREFIX,
203204
"app-id",
204205
DRIVER_LABELS,

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/LocalDirsFeatureStepSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import org.scalatest.BeforeAndAfter
2424
import org.apache.spark.{SparkConf, SparkFunSuite}
2525
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, KubernetesRoleSpecificConf, SparkPod}
2626
import org.apache.spark.deploy.k8s.Config._
27+
import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
2728

2829
class LocalDirsFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
2930
private val defaultLocalDir = "/var/data/default-local-dir"
@@ -36,7 +37,7 @@ class LocalDirsFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
3637
kubernetesConf = KubernetesConf(
3738
sparkConf,
3839
KubernetesDriverSpecificConf(
39-
None,
40+
JavaMainAppResource(None),
4041
"app-name",
4142
"main",
4243
Seq.empty),

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,14 @@ package org.apache.spark.deploy.k8s.features
1818

1919
import org.apache.spark.{SparkConf, SparkFunSuite}
2020
import org.apache.spark.deploy.k8s._
21+
import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
2122

2223
class MountVolumesFeatureStepSuite extends SparkFunSuite {
2324
private val sparkConf = new SparkConf(false)
2425
private val emptyKubernetesConf = KubernetesConf(
2526
sparkConf = sparkConf,
2627
roleSpecificConf = KubernetesDriverSpecificConf(
27-
None,
28+
JavaMainAppResource(None),
2829
"app-name",
2930
"main",
3031
Seq.empty),

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import org.scalatest.BeforeAndAfter
2525

2626
import org.apache.spark.{SparkConf, SparkFunSuite}
2727
import org.apache.spark.deploy.k8s._
28+
import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
2829

2930
class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter {
3031
private var sparkConf: SparkConf = _
@@ -36,7 +37,7 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite with BeforeAndAfter {
3637
kubernetesConf = KubernetesConf(
3738
sparkConf,
3839
KubernetesDriverSpecificConf(
39-
None,
40+
JavaMainAppResource(None),
4041
"app-name",
4142
"main",
4243
Seq.empty),

‎resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
2828
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpec, KubernetesDriverSpecificConf, SparkPod}
2929
import org.apache.spark.deploy.k8s.Constants._
3030
import org.apache.spark.deploy.k8s.Fabric8Aliases._
31+
import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
3132

3233
class ClientSuite extends SparkFunSuite with BeforeAndAfter {
3334

@@ -133,7 +134,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
133134
sparkConf = new SparkConf(false)
134135
kubernetesConf = KubernetesConf[KubernetesDriverSpecificConf](
135136
sparkConf,
136-
KubernetesDriverSpecificConf(None, MAIN_CLASS, APP_NAME, APP_ARGS),
137+
KubernetesDriverSpecificConf(JavaMainAppResource(None), MAIN_CLASS, APP_NAME, APP_ARGS),
137138
KUBERNETES_RESOURCE_PREFIX,
138139
APP_ID,
139140
Map.empty,

0 commit comments

Comments
 (0)