From c3a462ce2966d42a3cebf238b809e2c2e2631c08 Mon Sep 17 00:00:00 2001 From: zhou-jiang Date: Fri, 3 May 2024 16:25:38 -0700 Subject: [PATCH] [SPARK-48121][K8S] Promote `KubernetesDriverConf` to `DeveloperApi` ### What changes were proposed in this pull request? This PR aims to promote `KubernetesDriverConf` to `DeveloperApi` ### Why are the changes needed? Since Apache Spark Kubernetes Operator requires this, we had better maintain it as a developer API officially from Apache Spark 4.0.0. https://github.com/apache/spark-kubernetes-operator/pull/10 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass the CIs ### Was this patch authored or co-authored using generative AI tooling? No Closes #46373 from jiangzho/driver_conf. Authored-by: zhou-jiang Signed-off-by: Dongjoon Hyun --- .../org/apache/spark/deploy/k8s/KubernetesConf.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala index fda772b737fed..f62204a8a9c0d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala @@ -22,6 +22,7 @@ import io.fabric8.kubernetes.api.model.{LocalObjectReference, LocalObjectReferen import org.apache.commons.lang3.StringUtils import org.apache.spark.{SPARK_VERSION, SparkConf} +import org.apache.spark.annotation.{DeveloperApi, Since, Unstable} import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.deploy.k8s.features.DriverServiceFeatureStep._ @@ -78,7 +79,15 @@ private[spark] abstract class KubernetesConf(val sparkConf: SparkConf) { def getOption(key: String): Option[String] = sparkConf.getOption(key) } -private[spark] class KubernetesDriverConf( +/** + * :: DeveloperApi :: + * + * Used for K8s operations internally and Spark K8s operator. + */ +@Unstable +@DeveloperApi +@Since("4.0.0") +class KubernetesDriverConf( sparkConf: SparkConf, val appId: String, val mainAppResource: MainAppResource,