From 3ad4e75322b9f95818980f356fcd8cca2fb0a8eb Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Sun, 20 Jul 2014 22:10:55 +0800 Subject: [PATCH] Starts spark-sql shell with spark-submit --- bin/spark-sql | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/bin/spark-sql b/bin/spark-sql index 3798f83210604..756d62c408d78 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -17,8 +17,59 @@ # limitations under the License. # +SCALA_VERSION=2.10 + +cygwin=false +case "`uname`" in + CYGWIN*) cygwin=true;; +esac + +# Enter posix mode for bash +set -o posix + # Figure out where Spark is installed FWDIR="$(cd `dirname $0`/..; pwd)" +ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION" + +if [ -n "$JAVA_HOME" ]; then + JAR_CMD="$JAVA_HOME/bin/jar" +else + JAR_CMD="jar" +fi + +# Use spark-assembly jar from either RELEASE or assembly directory +if [ -f "$FWDIR/RELEASE" ]; then + assembly_folder="$FWDIR"/lib +else + assembly_folder="$ASSEMBLY_DIR" +fi + +num_jars=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar" | wc -l) +if [ "$num_jars" -eq "0" ]; then + echo "Failed to find Spark assembly in $assembly_folder" + echo "You need to build Spark before running this program." + exit 1 +fi +if [ "$num_jars" -gt "1" ]; then + jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar") + echo "Found multiple Spark assembly jars in $assembly_folder:" + echo "$jars_list" + echo "Please remove all but one jar." + exit 1 +fi + +ASSEMBLY_JAR=$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null) + +# Verify that versions of java used to build the jars and run Spark are compatible +jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1) +if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then + echo "Loading Spark jar with '$JAR_CMD' failed. " 1>&2 + echo "This is likely because Spark was compiled with Java 7 and run " 1>&2 + echo "with Java 6. (see SPARK-1703). Please use Java 7 to run Spark " 1>&2 + echo "or build Spark with Java 6." 1>&2 + exit 1 +fi + CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" -$FWDIR/bin/spark-class $CLASS $@ +exec "$FWDIR"/bin/spark-submit --class $CLASS $@ $ASSEMBLY_JAR