Merge pull request apache#25 from mesosphere/spark-new-config

[SPARK-225] new config

Author: mgummelt

Makefile

@@ -0,0 +1,13 @@
+docker:
+	bin/make-docker.sh
+
+package:
+	bin/make-package.py
+
+universe:
+	bin/make-universe.sh
+
+test:
+	bin/test.sh
+
+.PHONY: package docker universe test

README.md

@@ -1,62 +1,47 @@
-Spark DCOS Package
-===
+# Spark DCOS Package
 
 This repo lets you configure, build, and test a new Spark DCOS package.
 It is the source for the Spark package in universe.  If you wish to modify
 that package, you should do so here, and generate a new package as
 described below.
 
-Configuring
----
+## Configuring
 
 edit `manifest.json`.
 
-Create a package
----
+## Push a docker image
+
+This will make a docker image from the distribution specified in `manifest.json`
 
 ```
-export DOCKER_IMAGE=...
-./bin/make-package.sh
+DOCKER_IMAGE=<name> make docker
 ```
 
-This command builds a docker image, pushes it, and writes a new
-package to `build/package`.  It uses the components listed in
-`manifest.json`.
+## Create a package
 
-Create a universe
----
+Write a package to `build/package`.  Use the `DOCKER_IMAGE` name you
+created above.
 
 ```
-./bin/make-universe.sh
+DOCKER_IMAGE=<name> make package
 ```
 
-This produces a new universe in `build/universe`.  You can then point your
-local `dcos` to this location via `dcos config set package.sources`.
+## Create a universe
 
-Create a docker image
----
+Write a universe to `build/universe`.  You can then upload this to
+e.g. S3, and point your DCOS cluster at it via `dcos package repo
+add`.
 
 ```
-./bin/make-docker.sh <spark-dist> <image>
+make universe
 ```
 
-* `<spark-dist>`: path to spark distribution
-* `<image>`: name of docker image
-
-This creates a new docker image from the given spark distribution.  It
-is called by `./bin/make-package.sh`.
 
-
-Test
----
+## Test
 
 ```
 ./bin/test.sh
 ```
 
-This performs every build step, including tests.  It builds spark, the docker image,
-the package, and the universe.  It spins up a CCM cluster and tests spark against that
-cluster.
-
-It requires several env variables.  Read the comment at the top of the file for a
-complete description.
+This requires several env variables, and is primarily used in CI.
+Read the comment at the top of the file for a complete description.

bin/make-docker.sh

@@ -1,14 +1,35 @@
 #!/usr/bin/env bash
 
-# Usage:
-# ./bin/make-docker.sh <spark-dist-dir> <image>
+set -e -o pipefail
 
+# ENV vars:
+#   DOCKER_IMAGE - <image>:<version>
+#   SPARK_DIST_URI (optional) - e.g. http://<domain>/spark-1.2.3.tgz
+
+if [ -z "${SPARK_DIST_URI}" ]; then
+    SPARK_URI=$(cat manifest.json | jq .spark_uri)
+    SPARK_URI="${SPARK_URI%\"}"
+    SPARK_URI="${SPARK_URI#\"}"
+    SPARK_DIST_URI=${SPARK_URI}
+fi
+
+DIST_TGZ=$(basename "${SPARK_DIST_URI}")
+DIST="${DIST_TGZ%.*}"
+
+# fetch spark
+mkdir -p build/dist
+[ -f "build/dist/${DIST_TGZ}" ] || curl -o "build/dist/${DIST_TGZ}" "${SPARK_DIST_URI}"
+tar xvf build/dist/spark*.tgz -C build/dist
+
+# create docker context
 rm -rf build/docker
 mkdir -p build/docker/dist
-cp -r "$1/." build/docker/dist
+cp -r "build/dist/${DIST}/." build/docker/dist
 cp -r conf/* build/docker/dist/conf
 cp -r docker/* build/docker
 
-pushd build/docker
-docker build -t $2 .
-popd
+# build docker
+(cd build/docker && docker build -t "${DOCKER_IMAGE}" .)
+
+# push docker
+docker push "${DOCKER_IMAGE}"

bin/make-manifest.sh

@@ -6,10 +6,8 @@
 import sys
 
 def main():
-    try:
-        os.mkdir('build/package')
-    except OSError:
-        pass
+    if not os.path.isdir('build/package'):
+        os.makedirs('build/package')
 
     with open("manifest.json") as f:
         manifest = json.load(f)

bin/make-package.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env bash
 
-# creates build/spark-universe, build/spark-universe.zip
+# creates
+# - build/spark-universe
+# - build/spark-universe.zip
 
 set -x -e
 
 rm -rf build/spark-universe*
 
 # make spark package
-./bin/make-package.py
+# TODO(mgummelt): remove this after some time
+# ./bin/make-package.py
 
 # download universe
 wget -O build/spark-universe.zip https://github.com/mesosphere/universe/archive/version-2.x.zip

bin/make-package.sh

@@ -4,7 +4,9 @@
 # Copy it as spark-env.sh and edit that to configure Spark for your site.
 
 # A custom HDFS config can be fetched via spark.mesos.uris.  This
-# moves those config files into the standard directory.
+# moves those config files into the standard directory.  In DCOS, the
+# CLI reads the "SPARK_HDFS_CONFIG_URL" marathon label in order to set
+# spark.mesos.uris
 mkdir -p "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/hdfs-site.xml" ] && cp "${MESOS_SANDBOX}/hdfs-site.xml" "${HADOOP_CONF_DIR}"
 [ -f "${MESOS_SANDBOX}/core-site.xml" ] && cp "${MESOS_SANDBOX}/core-site.xml" "${HADOOP_CONF_DIR}"

bin/make-universe.sh

@@ -15,17 +15,18 @@ if [ "${SPARK_SSL_ENABLED}" == true ]; then
 	OTHER_SCHEME=http
 fi
 
-export WEBUI_URL="${SCHEME}://${FRAMEWORK_NAME}${DNS_SUFFIX}:${SPARK_PROXY_PORT}"
-export HISTORY_SERVER_WEB_PROXY_BASE="/service/${FRAMEWORK_NAME}/history"
-export DISPATCHER_UI_WEB_PROXY_BASE="/service/${FRAMEWORK_NAME}"
+# TODO(mgummelt): I'm pretty sure this isn't used.  Remove after some time.
+# export WEBUI_URL="${SCHEME}://${FRAMEWORK_NAME}${DNS_SUFFIX}:${SPARK_PROXY_PORT}"
+
+export HISTORY_SERVER_WEB_PROXY_BASE="/service/${DCOS_SERVICE_NAME}/history"
+export DISPATCHER_UI_WEB_PROXY_BASE="/service/${DCOS_SERVICE_NAME}"
 
 # configure history server
 if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then
     ln -s /var/lib/runit/service/history-server /etc/service/history-server
 fi
 
-# remove whole lines with the wrong scheme, remove #<SCHEME># string only for
-# the scheme we want to configure.
+# Update nginx spark.conf to use http or https
 grep -v "#${OTHER_SCHEME}#" /etc/nginx/conf.d/spark.conf.template |
 	sed "s,#${SCHEME}#,," >/etc/nginx/conf.d/spark.conf
 
@@ -39,7 +40,7 @@ sed -i "s,<PROTOCOL>,${SPARK_SSL_PROTOCOL}," /etc/nginx/conf.d/spark.conf
 # bytes cipher strings of Java.
 # sed -i "s,<ENABLED_ALGORITHMS>,${SPARK_SSL_ENABLEDALGORITHMS//,/:}," /etc/nginx/conf.d/spark.conf
 
-# extract cert and key from keystore
+# extract cert and key from keystore, write to /etc/nginx/spark.{crt,key}
 if [ "${SPARK_SSL_ENABLED}" == true ]; then
 	KEYDIR=`mktemp -d`
 	trap "rm -rf $KEYDIR" EXIT

conf/spark-env.sh

@@ -8,14 +8,25 @@ export APPLICATION_WEB_PROXY_BASE="${DISPATCHER_UI_WEB_PROXY_BASE}"
 
 cd /opt/spark/dist
 
-if [ "$FRAMEWORK_NAME" != "spark" ]; then
-    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_$FRAMEWORK_NAME"
+export SPARK_DAEMON_JAVA_OPTS=""
+if [ "${DCOS_SERVICE_NAME}" != "spark" ]; then
+    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.zookeeper.dir=/spark_mesos_dispatcher_${DCOS_SERVICE_NAME}"
 fi
 
 if [ "$SPARK_DISPATCHER_MESOS_ROLE" != "" ]; then
     export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.role=$SPARK_DISPATCHER_MESOS_ROLE"
 fi
 
+if [ "$SPARK_DISPATCHER_MESOS_PRINCIPAL" != "" ]; then
+    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.principal=$SPARK_DISPATCHER_MESOS_PRINCIPAL"
+fi
+
+if [ "$SPARK_DISPATCHER_MESOS_SECRET" != "" ]; then
+    export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.mesos.secret=$SPARK_DISPATCHER_MESOS_SECRET"
+fi
+
+
+
 HISTORY_SERVER_CONF=""
 if [ "${ENABLE_HISTORY_SERVER:=false}" = "true" ]; then
     HISTORY_SERVER_CONF="spark.mesos.historyServer.url=${HISTORY_SERVER_WEB_PROXY_BASE}"
@@ -39,12 +50,13 @@ add_if_non_empty spark.ssl.trustStorePassword "${SPARK_SSL_TRUSTSTOREPASSWORD}"
 add_if_non_empty spark.ssl.protocol "${SPARK_SSL_PROTOCOL}"
 add_if_non_empty spark.ssl.enabledAlgorithms "${SPARK_SSL_ENABLEDALGORITHMS}"
 
+export ZK="master.mesos:2181"
 exec /opt/spark/dist/bin/spark-class \
     org.apache.spark.deploy.mesos.MesosClusterDispatcher \
     --port "${DISPATCHER_PORT}" \
     --webui-port "${DISPATCHER_UI_PORT}" \
-    --master "${MESOS_MASTER}" \
+    --master "mesos://zk://${ZK}/mesos" \
     --zk "${ZK}" \
     --host "${HOST}" \
-    --name "${FRAMEWORK_NAME}" \
+    --name "${DCOS_SERVICE_NAME}" \
     --properties-file "conf/mesos-cluster-dispatcher.properties"