Worked arount SPARK-2678

Author: liancheng

bin/spark-sql

@@ -28,19 +28,61 @@ CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver"
 # Figure out where Spark is installed
 FWDIR="$(cd `dirname $0`/..; pwd)"
 
-if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
+function usage {
   echo "Usage: ./sbin/spark-sql [options] [cli option]"
   pattern="usage"
   pattern+="\|Spark assembly has been built with Hive"
   pattern+="\|NOTE: SPARK_PREPEND_CLASSES is set"
   pattern+="\|Spark Command: "
+  pattern+="\|--help"
   pattern+="\|======="
 
   $FWDIR/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2
   echo
   echo "CLI options:"
   $FWDIR/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
+}
+
+function ensure_arg_number {
+  arg_number=$1
+  at_least=$2
+
+  if [[ $arg_number -lt $at_least ]]; then
+    usage
+    exit 1
+  fi
+}
+
+if [[ "$@" = --help ]] || [[ "$@" = -H ]]; then
+  usage
   exit 0
 fi
 
-exec "$FWDIR"/bin/spark-submit --class $CLASS spark-internal $@
+CLI_ARGS=""
+SUBMISSION_ARGS=""
+
+while (($#)); do
+  case $1 in 
+    -d | --define | --database | -e | -f | -h | --hiveconf | --hivevar | -i | -p)
+      ensure_arg_number $# 2
+      CLI_ARGS+=" $1"; shift
+      CLI_ARGS+=" $1"; shift
+      ;;
+
+    -s | --silent)
+      CLI_ARGS+=" $1"; shift
+      ;;
+
+    -v | --verbose)
+      # Both SparkSubmit and SparkSQLCLIDriver recognizes -v | --verbose
+      CLI_ARGS+=" $1"
+      SUBMISSION_ARGS+=" $1"; shift
+      ;;
+
+    *)
+      SUBMISSION_ARGS+=" $1"; shift
+      ;;
+  esac
+done
+
+exec "$FWDIR"/bin/spark-submit --class $CLASS $SUBMISSION_ARGS spark-internal $CLI_ARGS

core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala

@@ -220,6 +220,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
   /** Fill in values by parsing user options. */
   private def parseOpts(opts: Seq[String]): Unit = {
     var inSparkOpts = true
+    val EQ_SEPARATED_OPT="""(--[^=]+)=(.+)""".r
 
     // Delineates parsing of Spark options from parsing of user options.
     parse(opts)
@@ -322,33 +323,21 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
         verbose = true
         parse(tail)
 
+      case EQ_SEPARATED_OPT(opt, value) :: tail =>
+        parse(opt :: value :: tail)
+
+      case value :: tail if value.startsWith("-") =>
+        SparkSubmit.printErrorAndExit(s"Unrecognized option '$value'.")
+
       case value :: tail =>
-        if (inSparkOpts) {
-          value match {
-            // convert --foo=bar to --foo bar
-            case v if v.startsWith("--") && v.contains("=") && v.split("=").size == 2 =>
-              val parts = v.split("=")
-              parse(Seq(parts(0), parts(1)) ++ tail)
-            case v if v.startsWith("-") =>
-              val errMessage = s"Unrecognized option '$value'."
-              SparkSubmit.printErrorAndExit(errMessage)
-            case v =>
-              primaryResource =
-                if (!SparkSubmit.isShell(v) && !SparkSubmit.isInternal(v)) {
-                  Utils.resolveURI(v).toString
-                } else {
-                  v
-                }
-              inSparkOpts = false
-              isPython = SparkSubmit.isPython(v)
-              parse(tail)
+        primaryResource =
+          if (!SparkSubmit.isShell(value) && !SparkSubmit.isInternal(value)) {
+            Utils.resolveURI(value).toString
+          } else {
+            value
           }
-        } else {
-          if (!value.isEmpty) {
-            childArgs += value
-          }
-          parse(tail)
-        }
+        isPython = SparkSubmit.isPython(value)
+        childArgs ++= tail
 
       case Nil =>
     }

core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala

@@ -106,6 +106,18 @@ class SparkSubmitSuite extends FunSuite with Matchers {
     appArgs.childArgs should be (Seq("some", "--weird", "args"))
   }
 
+  test("handles arguments to user program with name collision") {
+    val clArgs = Seq(
+      "--name", "myApp",
+      "--class", "Foo",
+      "userjar.jar",
+      "--master", "local",
+      "some",
+      "--weird", "args")
+    val appArgs = new SparkSubmitArguments(clArgs)
+    appArgs.childArgs should be (Seq("--master", "local", "some", "--weird", "args"))
+  }
+
   test("handles YARN cluster mode") {
     val clArgs = Seq(
       "--deploy-mode", "cluster",

sbin/start-thriftserver.sh

@@ -28,7 +28,7 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
 
 CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"
 
-if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
+function usage {
   echo "Usage: ./sbin/start-thriftserver [options] [thrift server options]"
   pattern="usage"
   pattern+="\|Spark assembly has been built with Hive"
@@ -41,7 +41,38 @@ if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
   echo
   echo "Thrift server options:"
   $FWDIR/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
+}
+
+function ensure_arg_number {
+  arg_number=$1
+  at_least=$2
+
+  if [[ $arg_number -lt $at_least ]]; then
+    usage
+    exit 1
+  fi
+}
+
+if [[ "$@" = --help ]] || [[ "$@" = -h ]]; then
+  usage
   exit 0
 fi
 
-exec "$FWDIR"/bin/spark-submit --class $CLASS spark-internal $@
+THRIFT_SERVER_ARGS=""
+SUBMISSION_ARGS=""
+
+while (($#)); do
+  case $1 in
+    --hiveconf)
+      ensure_arg_number $# 2
+      THRIFT_SERVER_ARGS+=" $1"; shift
+      THRIFT_SERVER_ARGS+=" $1"; shift
+      ;;
+
+    *)
+      SUBMISSION_ARGS+=" $1"; shift
+      ;;
+  esac
+done
+
+exec "$FWDIR"/bin/spark-submit --class $CLASS $SUBMISSION_ARGS spark-internal $THRIFT_SERVER_ARGS