Skip to content

Commit

Permalink
Starts spark-sql shell with spark-submit
Browse files Browse the repository at this point in the history
  • Loading branch information
liancheng committed Jul 24, 2014
1 parent a5310d1 commit 3ad4e75
Showing 1 changed file with 52 additions and 1 deletion.
53 changes: 52 additions & 1 deletion bin/spark-sql
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,59 @@
# limitations under the License.
#

SCALA_VERSION=2.10

cygwin=false
case "`uname`" in
CYGWIN*) cygwin=true;;
esac

# Enter posix mode for bash
set -o posix

# Figure out where Spark is installed
FWDIR="$(cd `dirname $0`/..; pwd)"

ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION"

if [ -n "$JAVA_HOME" ]; then
JAR_CMD="$JAVA_HOME/bin/jar"
else
JAR_CMD="jar"
fi

# Use spark-assembly jar from either RELEASE or assembly directory
if [ -f "$FWDIR/RELEASE" ]; then
assembly_folder="$FWDIR"/lib
else
assembly_folder="$ASSEMBLY_DIR"
fi

num_jars=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar" | wc -l)
if [ "$num_jars" -eq "0" ]; then
echo "Failed to find Spark assembly in $assembly_folder"
echo "You need to build Spark before running this program."
exit 1
fi
if [ "$num_jars" -gt "1" ]; then
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar")
echo "Found multiple Spark assembly jars in $assembly_folder:"
echo "$jars_list"
echo "Please remove all but one jar."
exit 1
fi

ASSEMBLY_JAR=$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null)

# Verify that versions of java used to build the jars and run Spark are compatible
jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1)
if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then
echo "Loading Spark jar with '$JAR_CMD' failed. " 1>&2
echo "This is likely because Spark was compiled with Java 7 and run " 1>&2
echo "with Java 6. (see SPARK-1703). Please use Java 7 to run Spark " 1>&2
echo "or build Spark with Java 6." 1>&2
exit 1
fi

CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver"
$FWDIR/bin/spark-class $CLASS $@
exec "$FWDIR"/bin/spark-submit --class $CLASS $@ $ASSEMBLY_JAR

0 comments on commit 3ad4e75

Please sign in to comment.