Use launcher classes instead of jars in non-release mode.

This makes it easier for people who are used to building spark with
"sbt assembly" instead of "sbt package" to use the new launcher without
having to modify their workflow.

Author: Marcelo Vanzin

bin/spark-class

@@ -40,33 +40,38 @@ else
   fi
 fi
 
+# Look for the launcher. In non-release mode, add the compiled classes directly to the classpath
+# instead of looking for a jar file, since it's very common for people using sbt to use the
+# "assembly" target instead of "package".
 SPARK_LAUNCHER_CP=
 if [ -f $SPARK_HOME/RELEASE ]; then
   LAUNCHER_DIR="$SPARK_HOME/lib"
-else
-  LAUNCHER_DIR="$SPARK_HOME/launcher/target/scala-$SPARK_SCALA_VERSION"
-  if [ -n "$SPARK_PREPEND_CLASSES" ]; then
-    SPARK_LAUNCHER_CP="$LAUNCHER_DIR/classes:"
+  num_jars="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" | wc -l)"
+  if [ "$num_jars" -eq "0" -a -z "$SPARK_LAUNCHER_CP" ]; then
+    echo "Failed to find Spark launcher in $LAUNCHER_DIR." 1>&2
+    echo "You need to build Spark before running this program." 1>&2
+    exit 1
   fi
-fi
 
-num_jars="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" | wc -l)"
-if [ "$num_jars" -eq "0" ]; then
-  echo "Failed to find Spark launcher in $LAUNCHER_DIR." 1>&2
-  echo "You need to build Spark before running this program." 1>&2
-  exit 1
-fi
+  LAUNCHER_JARS="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" || true)"
+  if [ "$num_jars" -gt "1" ]; then
+    echo "Found multiple Spark launcher jars in $LAUNCHER_DIR:" 1>&2
+    echo "$LAUNCHER_JARS" 1>&2
+    echo "Please remove all but one jar." 1>&2
+    exit 1
+  fi
 
-LAUNCHER_JARS="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$")"
-if [ "$num_jars" -gt "1" ]; then
-  echo "Found multiple Spark launcher jars in $LAUNCHER_DIR:" 1>&2
-  echo "$LAUNCHER_JARS" 1>&2
-  echo "Please remove all but one jar." 1>&2
-  exit 1
+  SPARK_LAUNCHER_CP="${LAUNCHER_DIR}/${LAUNCHER_JARS}"
+else
+  LAUNCHER_DIR="$SPARK_HOME/launcher/target/scala-$SPARK_SCALA_VERSION"
+  if [ ! -d "$LAUNCHER_DIR/classes" ]; then
+    echo "Failed to find Spark launcher classes in $LAUNCHER_DIR." 1>&2
+    echo "You need to build Spark before running this program." 1>&2
+    exit 1
+  fi
+  SPARK_LAUNCHER_CP="$LAUNCHER_DIR/classes"
 fi
 
-SPARK_LAUNCHER_CP="${SPARK_LAUNCHER_CP}${LAUNCHER_DIR}/${LAUNCHER_JARS}"
-
 # The launcher library will print arguments separated by a NULL character. Read that in a while
 # loop, populating an array that will be used to exec the final command.
 CMD=()

bin/spark-class2.cmd

@@ -29,31 +29,30 @@ if "x%1"=="x" (
   exit /b 1
 )
 
-set LAUNCHER_JAR=0
-if exist "%SPARK_HOME%\RELEASE" goto find_release_launcher
+set LAUNCHER_CP=0
+if exist %SPARK_HOME%\RELEASE goto find_release_launcher
 
-rem Look for the Spark launcher in both Scala build directories. The launcher doesn't use
-rem Scala so it doesn't really matter which one is picked up.
-for %%d in (%SPARK_HOME%\launcher\target\scala-2.10\spark-launcher*.jar) do (
-  set LAUNCHER_JAR=%%d
-  set SPARK_SCALA_VERSION=2.10
+rem Look for the Spark launcher in both Scala build directories. The launcher doesn't use Scala so
+rem it doesn't really matter which one is picked up. Add the compiled classes directly to the
+rem classpath instead of looking for a jar file, since it's very common for people using sbt to use
+rem the "assembly" target instead of "package".
+set LAUNCHER_CLASSES=%SPARK_HOME%\launcher\target\scala-2.10\classes
+if exist %LAUNCHER_CLASSES% (
+  set LAUNCHER_CP=%LAUNCHER_CLASSES%
 )
-for %%d in (%SPARK_HOME%\launcher\target\scala-2.11\spark-launcher*.jar) do (
-  set LAUNCHER_JAR=%%d
-  set SPARK_SCALA_VERSION=2.11
-)
-if not "x%SPARK_PREPEND_CLASSES"=="x" (
-  set LAUNCHER_JAR=%SPARK_HOME%\launcher\target\scala-%SPARK_SCALA_VERSION%\classes;%LAUNCHER_JAR%
+set LAUNCHER_CLASSES=%SPARK_HOME%\launcher\target\scala-2.11\classes
+if exist %LAUNCHER_CLASSES% (
+  set LAUNCHER_CP=%LAUNCHER_CLASSES%
 )
 goto check_launcher
 
 :find_release_launcher
 for %%d in (%SPARK_HOME%\lib\spark-launcher*.jar) do (
-  set LAUNCHER_JAR=%%d
+  set LAUNCHER_CP=%%d
 )
 
 :check_launcher
-if "%LAUNCHER_JAR%"=="0" (
+if "%LAUNCHER_CP%"=="0" (
   echo Failed to find Spark launcher JAR.
   echo You need to build Spark before running this program.
   exit /b 1
@@ -63,7 +62,7 @@ rem Figure out where java is.
 set RUNNER=java
 if not "x%JAVA_HOME%"=="x" set RUNNER=%JAVA_HOME%\bin\java
 
-for /f "tokens=*" %%i in ('cmd /C ""%RUNNER%" -cp %LAUNCHER_JAR% org.apache.spark.launcher.Main %*"') do (
+for /f "tokens=*" %%i in ('cmd /C ""%RUNNER%" -cp %LAUNCHER_CP% org.apache.spark.launcher.Main %*"') do (
   set SPARK_CMD=%%i
 )
 %SPARK_CMD%

launcher/pom.xml

@@ -79,23 +79,5 @@
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-        <configuration>
-          <outputDirectory>target/scala-${scala.binary.version}</outputDirectory>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.scalatest</groupId>
-        <artifactId>scalatest-maven-plugin</artifactId>
-        <configuration>
-          <systemProperties>
-            <spark.test.home>${project.basedir}/..</spark.test.home>
-          </systemProperties>
-        </configuration>
-      </plugin>
-    </plugins>
   </build>
 </project>

pom.xml

@@ -1131,7 +1131,7 @@
             </environmentVariables>
             <systemProperties>
               <java.awt.headless>true</java.awt.headless>
-              <spark.test.home>${session.executionRootDirectory}</spark.test.home>
+              <spark.test.home>${spark.test.home}</spark.test.home>
               <spark.testing>1</spark.testing>
               <spark.ui.enabled>false</spark.ui.enabled>
               <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>