Fix propagation of config options to driver / executor.

ClientBase disagreed with itself about how to propagate config
options. Some places used the SparkConf object, others relied
on system properties. This lead to certain properties not being
propagated, mainly in cluster mode.

So standardize on using SparkConf for that. To maintain compatibility
with SPARK_JAVA_OPTS, remove the hack in ClientBase and just call
SparkConf.validateSettings(), which does the right thing and correctly
warns the user to stop using the env variable in the future.

Author: Marcelo Vanzin

yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

@@ -178,6 +178,7 @@ object Client {
     System.setProperty("SPARK_YARN_MODE", "true")
 
     val sparkConf = new SparkConf
+    sparkConf.validateSettings()
 
     try {
       val args = new ClientArguments(argStrings, sparkConf)

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala

@@ -351,21 +351,16 @@ trait ClientBase extends Logging {
       javaOpts += "-XX:CMSIncrementalDutyCycle=10"
     }
 
-    // SPARK_JAVA_OPTS is deprecated, but for backwards compatibility:
-    sys.env.get("SPARK_JAVA_OPTS").foreach { opts =>
-      sparkConf.set("spark.executor.extraJavaOptions", opts)
-      sparkConf.set("spark.driver.extraJavaOptions", opts)
-    }
-
     // Forward the Spark configuration to the application master / executors.
     // TODO: it might be nicer to pass these as an internal environment variable rather than
     // as Java options, due to complications with string parsing of nested quotes.
     for ((k, v) <- sparkConf.getAll) {
       javaOpts += "-D" + k + "=" + "\\\"" + v + "\\\""
     }
     if (args.amClass == classOf[ApplicationMaster].getName) {
-      sys.props.get("spark.driver.extraJavaOptions").foreach(opts => javaOpts += opts)
-      sys.props.get("spark.driver.libraryPath").foreach(p => javaOpts += s"-Djava.library.path=$p")
+      sparkConf.getOption("spark.driver.extraJavaOptions").foreach(opts => javaOpts += opts)
+      sparkConf.getOption("spark.driver.libraryPath")
+        .foreach(p => javaOpts += s"-Djava.library.path=$p")
     }
 
     // Command for the ApplicationMaster

yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

@@ -185,6 +185,7 @@ object Client {
     // see Client#setupLaunchEnv().
     System.setProperty("SPARK_YARN_MODE", "true")
     val sparkConf = new SparkConf()
+    sparkConf.validateSettings()
 
     try {
       val args = new ClientArguments(argStrings, sparkConf)