Truncate k8s hostnames to be no longer than 63 characters (apache#102)

* Truncate k8s hostnames to be no longer than 63 characters * Use only executorId not executorKubernetesId
lins05 · Mar 8, 2017 · 7a4075f · 7a4075f
1 parent fe8b45c
commit 7a4075f
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/...ala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala b/...ala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala
@@ -155,9 +155,14 @@ private[spark] class KubernetesClusterSchedulerBackend(
   }
 
   private def allocateNewExecutorPod(): (String, Pod) = {
-    val executorKubernetesId = UUID.randomUUID().toString.replaceAll("-", "")
     val executorId = EXECUTOR_ID_COUNTER.incrementAndGet().toString
-    val name = s"${applicationId()}-exec-$executorKubernetesId"
+    val name = s"${applicationId()}-exec-$executorId"
+
+    // hostname must be no longer than 63 characters, so take the last 63 characters of the pod
+    // name as the hostname.  This preserves uniqueness since the end of name contains
+    // executorId and applicationId
+    val hostname = name.substring(Math.max(0, name.length - 63))
+
     val selectors = Map(SPARK_EXECUTOR_ID_LABEL -> executorId,
       SPARK_APP_ID_LABEL -> applicationId()).asJava
     val executorMemoryQuantity = new QuantityBuilder(false)
@@ -190,7 +195,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
           .build()
       })
     try {
-      (executorKubernetesId, kubernetesClient.pods().createNew()
+      (executorId, kubernetesClient.pods().createNew()
         .withNewMetadata()
           .withName(name)
           .withLabels(selectors)
@@ -204,6 +209,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
           .endOwnerReference()
         .endMetadata()
         .withNewSpec()
+          .withHostname(hostname)
           .addNewContainer()
             .withName(s"executor")
             .withImage(executorDockerImage)