Merge branch 'master' into add-pyspark-tests

Author: susanxhuynh

cli/dcos_spark/version.py

@@ -1 +1 @@
-version = 'SNAPSHOT'
+version = '0.5.19'

conf/spark-env.sh

@@ -7,17 +7,24 @@
 # moves those config files into the standard directory.  In DCOS, the
 # CLI reads the "SPARK_HDFS_CONFIG_URL" marathon label in order to set
 # spark.mesos.uris
+
 mkdir -p "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/hdfs-site.xml" ] && cp "${MESOS_SANDBOX}/hdfs-site.xml" "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/core-site.xml" ] && cp "${MESOS_SANDBOX}/core-site.xml" "${HADOOP_CONF_DIR}"
 
-MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so
+cd $MESOS_SANDBOX
+
+MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so
 
 # Support environments without DNS
 if [ -n "$LIBPROCESS_IP" ]; then
   SPARK_LOCAL_IP=${LIBPROCESS_IP}
 fi
 
+# I first set this to MESOS_SANDBOX, as a Workaround for MESOS-5866
+# But this fails now due to MESOS-6391, so I'm setting it to /tmp
+MESOS_DIRECTORY=/tmp
+
 # Options read when launching programs locally with
 # ./bin/run-example or ./bin/spark-submit
 # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files

docker/Dockerfile

@@ -18,7 +18,7 @@
 # docker build -t spark:git-`git rev-parse --short HEAD` .
 
 # Basing from Mesos image so the Mesos native library is present.
-FROM mesosphere/mesos-modules-private:git-e348e3f
+FROM mesosphere/mesos-modules-private:dcos-ee-mesos-modules-1.8.5-rc2
 MAINTAINER Michael Gummelt <mgummelt@mesosphere.io>
 
 # Set environment variables.
@@ -27,24 +27,25 @@ ENV DEBCONF_NONINTERACTIVE_SEEN "true"
 
 # Upgrade package index and install basic commands.
 RUN apt-get update && \
-    apt-get install -y software-properties-common runit nginx
+    apt-get install -y \
+            software-properties-common \
+            runit \
+            nginx
+
 RUN add-apt-repository ppa:openjdk-r/ppa
 RUN apt-get update && \
     apt-get install -y openjdk-8-jdk curl
 
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV MESOS_NATIVE_JAVA_LIBRARY /usr/local/lib/libmesos.so
+ENV MESOS_NATIVE_JAVA_LIBRARY /usr/lib/libmesos.so
 ENV HADOOP_CONF_DIR /etc/hadoop
 
 RUN mkdir /etc/hadoop
 
-ADD dist /opt/spark/dist
 ADD runit/service /var/lib/runit/service
 ADD runit/init.sh /sbin/init.sh
 ADD nginx /etc/nginx
 
-#RUN ln -sf /usr/lib/libmesos.so /usr/lib/libmesos-0.23.1.so
-
 # The following symlinks are hacks to make spark-class work under the
 # restricted PATH (/usr/bin) set by the DCOS
 # --executor-environment-variables option
@@ -55,4 +56,13 @@ RUN ln -s /bin/grep /usr/bin/grep
 RUN ln -s /var/lib/runit/service/spark /etc/service/spark
 RUN ln -s /var/lib/runit/service/nginx /etc/service/nginx
 
+RUN chmod -R ugo+rw /etc/nginx
+RUN chmod -R ugo+rw /etc/service
+RUN chmod -R ugo+rw /var/lib/
+RUN chmod -R ugo+rw /var/run/
+RUN chmod -R ugo+rw /var/log/
+
+ADD dist /opt/spark/dist
+RUN chmod -R ugo+rw /opt/spark/dist
+
 WORKDIR /opt/spark/dist

docker/runit/service/spark/run

@@ -4,64 +4,83 @@ set -x
 
 exec 2>&1
 
-export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}"
-
-cd /opt/spark/dist
-
-export SPARK_DAEMON_JAVA_OPTS=""
-if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}"
-fi
-
-if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE"
-fi
-
-if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL"
-fi
-
-if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET"
-fi
-
-
-
-HISTORY_SERVER_CONF=""
-if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then
-    HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}"
-fi
-
-sed "s,<HISTORY_SERVER_CONF>,${HISTORY_SERVER_CONF}," \
-    conf/mesos-cluster-dispatcher.properties.template >conf/mesos-cluster-dispatcher.properties
+function export_daemon_opts() {
+    export SPARK_DAEMON_JAVA_OPTS=""
+    if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}"
+    fi
+
+    if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE"
+    fi
+
+    if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL"
+    fi
+
+    if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then
+        export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET"
+    fi
+}
 
-sed "s,<LOG_LEVEL>,${SPARK_LOG_LEVEL}," \
-    conf/log4j.properties.template >conf/log4j.properties
+function set_log_level() {
+    sed "s,<LOG_LEVEL>,${SPARK_LOG_LEVEL}," \
+        /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+}
 
 function add_if_non_empty() {
 	if [ -n "$2" ]; then
-		echo "$1=$2" >> conf/mesos-cluster-dispatcher.properties
+		echo "$1=$2" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
 	fi
 }
 
-if [ "${SPARK_SSL_KEYSTOREBASE64}" != "" ]; then
-    echo "${SPARK_SSL_KEYSTOREBASE64}" | base64 -d > /tmp/dispatcher-keystore.jks
-    add_if_non_empty spark.ssl.keyStore /tmp/dispatcher-keystore.jks
-fi
+function configure_properties() {
+    HISTORY_SERVER_CONF=""
+    if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then
+        HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}"
+    fi
+
+    sed "s,<HISTORY_SERVER_CONF>,${HISTORY_SERVER_CONF}," \
+        /opt/spark/dist/conf/mesos-cluster-dispatcher.properties.template >/opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+
+    if [ "${SPARK_SSL_KEYSTOREBASE64}" != "" ]; then
+        echo "${SPARK_SSL_KEYSTOREBASE64}" | base64 -d > /tmp/dispatcher-keystore.jks
+        add_if_non_empty spark.ssl.keyStore /tmp/dispatcher-keystore.jks
+    fi
+
+    if [ "${SPARK_SSL_TRUSTSTOREBASE64}" != "" ]; then
+        echo "${SPARK_SSL_TRUSTSTOREBASE64}" | base64 -d > /tmp/dispatcher-truststore.jks
+        add_if_non_empty spark.ssl.trustStore /tmp/dispatcher-truststore.jks
+    fi
+
+    add_if_non_empty spark.ssl.enabled "${SPARK_SSL_ENABLED}"
+    add_if_non_empty spark.ssl.keyPassword "${SPARK_SSL_KEYPASSWORD}"
+    add_if_non_empty spark.ssl.keyStorePassword "${SPARK_SSL_KEYSTOREPASSWORD}"
+    add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}"
+    add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}"
+    add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}"
+
+    # write defaults
+    if [ "${DCOS_SERVICE_ACCOUNT_CREDENTIAL}" != "" ]; then
+        # write defaults using both property names, since 2.0 uses one and 2.1 uses the other
+        echo "spark.mesos.dispatcher.driverDefault.spark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+        echo "spark.mesos.cluster.taskProperty.spark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+
+        echo "spark.mesos.dispatcher.driverDefault.spark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+        echo "spark.mesos.cluster.taskProperty.spark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+
+        echo "spark.mesos.dispatcher.driverDefault.spark.mesos.principal=${SPARK_DISPATCHER_MESOS_PRINCIPAL}" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+        echo "spark.mesos.cluster.taskProperty.spark.mesos.principal=${SPARK_DISPATCHER_MESOS_PRINCIPAL}" >> /opt/spark/dist/conf/mesos-cluster-dispatcher.properties
+    fi
+}
 
-if [ "${SPARK_SSL_TRUSTSTOREBASE64}" != "" ]; then
-    echo "${SPARK_SSL_TRUSTSTOREBASE64}" | base64 -d > /tmp/dispatcher-truststore.jks
-    add_if_non_empty spark.ssl.trustStore /tmp/dispatcher-truststore.jks
-fi
 
-add_if_non_empty spark.ssl.enabled "${SPARK_SSL_ENABLED}"
-add_if_non_empty spark.ssl.keyPassword "${SPARK_SSL_KEYPASSWORD}"
-add_if_non_empty spark.ssl.keyStorePassword "${SPARK_SSL_KEYSTOREPASSWORD}"
-add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}"
-add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}"
-add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}"
+export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}"
+set_log_level
+export_daemon_opts
+configure_properties
+ZK="master.mesos:2181"
 
-export ZK="master.mesos:2181"
 exec /opt/spark/dist/bin/spark-class \
     org.apache.spark.deploy.mesos.MesosClusterDispatcher \
     --port "${DISPATCHER_PORT}" \
@@ -70,4 +89,4 @@ exec /opt/spark/dist/bin/spark-class \
     --zk "${ZK}" \
     --host "${HOST}" \
     --name "${DCOS_SERVICE_NAME}" \
-    --properties-file "conf/mesos-cluster-dispatcher.properties"
+    --properties-file "/opt/spark/dist/conf/mesos-cluster-dispatcher.properties"

docs/user-docs.md

@@ -276,7 +276,114 @@ to the history server entry for that job.
 
 <a name="ssl"></a>
 
-### SSL
+### Security
+
+#### Mesos
+
+##### SSL
+
+<table class="table">
+  <tr>
+    <td>
+      `security.mesos.ssl.enabled`
+    </td>
+
+    <td>
+      Set to true to enable SSL on Mesos communication (default: false).
+    </td>
+  </tr>
+</table>
+
+
+##### Authentication
+
+When running in
+[DC/OS strict security mode](https://docs.mesosphere.com/latest/administration/id-and-access-mgt/),
+Both the dispatcher and jobs must authenticate to Mesos using a [DC/OS
+Service Account](https://docs.mesosphere.com/1.8/administration/id-and-access-mgt/service-auth/).
+Follow these instructions to authenticate in strict mode:
+
+1. Create a Service Account
+
+Instructions
+[here](https://docs.mesosphere.com/1.8/administration/id-and-access-mgt/service-auth/universe-service-auth/).
+
+2. Assign Permissions
+
+First, allow Spark to run tasks as root:
+
+```
+$ curl -k -L -X PUT \
+       -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \
+       "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:task:user:root" \
+       -d '{"description":"Allows root to execute tasks"}' \
+       -H 'Content-Type: application/json'
+
+$ curl -k -L -X PUT \
+     -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \
+     "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:task:user:root/users/${SERVICE_ACCOUNT_NAME}/create"
+```
+
+Now you must allow Spark to register under the desired role.  This is
+the value used for `service.role` when installing Spark (default:
+`*`):
+
+```
+$ export ROLE=<service.role value>
+$ curl -k -L -X PUT \
+       -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \
+       "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:framework:role:${ROLE}" \
+       -d '{"description":"Allows ${ROLE} to register as a framework with the Mesos master"}' \
+       -H 'Content-Type: application/json'
+
+$ curl -k -L -X PUT \
+       -H "Authorization: token=$(dcos config show core.dcos_acs_token)" \
+       "$(dcos config show core.dcos_url)/acs/api/v1/acls/dcos:mesos:master:framework:role:${ROLE}/users/${SERVICE_ACCOUNT_NAME}/create"
+```
+
+3. Install Spark
+
+```
+$ dcos package install spark --options=config.json
+```
+
+Where `config.json` contains the following JSON.  Replace
+`<principal>` with the name of your service account, and
+`<secret_name>` with the name of the DC/OS secret containing your
+service account's private key.  These values were created in Step #1
+above.
+
+```
+{
+    "service": {
+        "principal": "<principal>",
+        "user": "nobody"
+    },
+    "security": {
+        "mesos": {
+            "authentication": {
+                "secret_name": "<secret_name>"
+            }
+        }
+    }
+}
+```
+
+4. Submit a Job
+
+We've now installed the Spark Dispatcher, which is authenticating
+itself to the Mesos master.  Spark jobs are also frameworks which must
+authenticate.  The dispatcher will pass the secret along to the jobs,
+so all that's left to do is configure our jobs to use DC/OS authentication:
+
+```
+$ PROPS="-Dspark.mesos.driverEnv.MESOS_MODULES=file:///opt/mesosphere/etc/mesos-scheduler-modules/dcos_authenticatee_module.json "
+$ PROPS+="-Dspark.mesos.driverEnv.MESOS_AUTHENTICATEE=com_mesosphere_dcos_ClassicRPCAuthenticatee "
+$ PROPS+="-Dspark.mesos.principal=<principal>"
+$ dcos spark run --submit-args="${PROPS} ..."
+```
+
+#### Spark SSL
 
 SSL support in DC/OS Spark encrypts the following channels:
 

package/config.json

@@ -116,6 +116,22 @@
                         }
                     }
                 },
+                "mesos": {
+                    "description": "Mesos scheduler configuration properties.",
+                    "type": "object",
+                    "properties": {
+                        "authentication": {
+                            "description": "Mesos scheduler dcos-oauth configuration.",
+                            "type": "object",
+                            "properties": {
+                                "secret_name": {
+                                    "description": "Name of the secret used to authenticate with the Mesos Master.",
+                                    "type": "string"
+                                }
+                            }
+                        }
+                    }
+                },
                 "ssl": {
                     "description": "Spark SSL certificates and private key configuration.",
                     "type": "object",