Skip to content

Commit 400a300

Browse files
mgummeltmgummelt
authored
Merge pull request apache#64 from mesosphere/revert-go-cli
[SPARK-344] Revert "Implement a Go Spark CLI"
2 parents df9bbad + 80fb49f commit 400a300

37 files changed

+1279
-810
lines changed

‎bin/jenkins-package-test.sh‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -14,33 +14,6 @@ function run() {
1414
}
1515

1616
pushd "${SPARK_BUILD_DIR}"
17+
1718
run
1819
popd
19-
# #!/bin/bash
20-
21-
# export VERSION=${ghprbActualCommit}
22-
# if [ -z "$VERSION" ]; then
23-
# export VERSION=${GIT_COMMIT}
24-
# fi
25-
26-
# export DOCKER_IMAGE=mesosphere/spark-dev:${VERSION}
27-
28-
# export S3_BUCKET=infinity-artifacts
29-
# export S3_PREFIX=autodelete7d/spark/${VERSION}
30-
# # fill in any missing DEV_* AWS envvars required by test.sh:
31-
# if [ -z "$DEV_S3_BUCKET" ]; then
32-
# export DEV_S3_BUCKET=$S3_BUCKET
33-
# fi
34-
# if [ -z "$DEV_S3_PREFIX" ]; then
35-
# export DEV_S3_PREFIX=$S3_PREFIX
36-
# fi
37-
# if [ -z "$DEV_AWS_ACCESS_KEY_ID" ]; then
38-
# export DEV_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
39-
# fi
40-
# if [ -z "$DEV_AWS_SECRET_ACCESS_KEY" ]; then
41-
# export DEV_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
42-
# fi
43-
44-
# source spark-build/bin/jenkins.sh
45-
46-
# spark_test

‎bin/universe.sh‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Builds a universe for this spark package
44
#
55
# Manifest config:
6+
# cli_version - version label to use for CLI package
67
# spark_uri - where fetch spark distribution from (or SPARK_DIST_URI if provided)
78
#
89
# ENV vars:
@@ -16,6 +17,7 @@ set -e -x -o pipefail
1617
BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
1718
BASEDIR="${BIN_DIR}/.."
1819

20+
# set CLI_VERSION, SPARK_DIST_URI, and DOCKER_IMAGE:
1921
configure_env() {
2022
if [ -z "${SPARK_DIST_URI}" ]; then
2123
SPARK_DIST_URI=$(cat $BASEDIR/manifest.json | jq .spark_uri)
@@ -24,6 +26,11 @@ configure_env() {
2426
echo "Using Spark dist URI: $SPARK_DIST_URI"
2527
fi
2628

29+
CLI_VERSION=$(cat $BASEDIR/manifest.json | jq .cli_version)
30+
CLI_VERSION="${CLI_VERSION%\"}"
31+
CLI_VERSION="${CLI_VERSION#\"}"
32+
echo "Using CLI Version: $CLI_VERSION"
33+
2734
if [ -z "$DOCKER_IMAGE" ]; then
2835
# determine image label based on git commit:
2936
if [ -n "$ghprbActualCommit" ]; then
@@ -55,7 +62,7 @@ fetch_commons_tools() {
5562
build_cli() {
5663
pwd
5764
ls -all $BASEDIR/cli
58-
make --directory=$BASEDIR/cli all
65+
CLI_VERSION=${CLI_VERSION} make --directory=$BASEDIR/cli all
5966
}
6067

6168
build_push_docker() {
@@ -72,19 +79,15 @@ upload_cli_and_stub_universe() {
7279
# Build/upload package using custom template parameters: TEMPLATE_X_Y_Z => {{x-y-z}}
7380
# ARTIFACT_DIR="https://${S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}" \
7481
# S3_DIR_PATH=${S3_PREFIX:-} \
82+
TEMPLATE_CLI_VERSION=${CLI_VERSION} \
7583
TEMPLATE_SPARK_DIST_URI=${SPARK_DIST_URI} \
7684
TEMPLATE_DOCKER_IMAGE=${DOCKER_IMAGE} \
77-
TEMPLATE_PACKAGE_VERSION=${VERSION} \
7885
${COMMONS_TOOLS_DIR}/ci_upload.py \
7986
spark \
8087
${BASEDIR}/package/ \
81-
${BASEDIR}/cli/dcos-spark/dcos-spark-darwin \
82-
${BASEDIR}/cli/dcos-spark/dcos-spark-linux \
83-
${BASEDIR}/cli/dcos-spark/dcos-spark.exe \
84-
${BASEDIR}/cli/python/dist/*.whl
88+
${BASEDIR}/cli/dist/*.whl
8589
}
8690

87-
# set SPARK_URI and DOCKER_IMAGE:
8891
configure_env
8992
fetch_commons_tools
9093
build_cli

‎cli/Makefile‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
1-
all: go python-wrapper
1+
all: env test packages
22

3-
go:
4-
bin/build-go.sh
5-
python-wrapper: go
6-
bin/build-python-wrapper.sh
73
clean:
84
bin/clean.sh
5+
6+
env: clean
7+
bin/env.sh
8+
9+
test:
10+
bin/test.sh
11+
12+
packages:
13+
bin/packages.sh
14+
15+
binary: env
16+
pyinstaller binary/binary.spec
17+
18+
.PHONY: binary

‎cli/README.rst‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
DCOS Spark Subcommand
2+
==========================
3+
Basic DCOS subcommand
4+
5+
Setup
6+
-----
7+
#. Make sure you meet requirements for installing packages_
8+
#. Clone git repo for the dcos spark cli::
9+
10+
git clone git@github.com:mesosphere/dcos-spark.git
11+
12+
#. Change directory to the repo directory::
13+
14+
cd dcos-spark
15+
16+
#. Make sure that you have virtualenv installed. If not type::
17+
18+
sudo pip install virtualenv
19+
20+
#. Create a virtualenv for the project::
21+
22+
make env
23+
24+
Configure Environment and Run
25+
-----------------------------
26+
27+
#. TODO: Talk about how to configure the root dcos cli
28+
29+
#. :code:`source` the setup file to add the :code:`dcos-spark` command line interface to your
30+
:code:`PATH`::
31+
32+
source env/bin/activate
33+
34+
#. Get started by calling the DCOS Spark CLI's help::
35+
36+
dcos-spark help
37+
38+
When invoking dcos subcommands from the dev env it includes the :code:`dcos-` as a prefix, however when this is included into DCOS it will be a subcommand of dcos and will be invoked by :code:`dcos spark`. :code:`dcos-spark help` translates to :code:`dcos spark help` when installed into the dcos-cli.
39+
40+
#. Leaving the env
41+
42+
deactivate
43+
44+
Binary:
45+
-----------
46+
47+
Create Binary:
48+
##############
49+
50+
#. Install pyinstaller::
51+
52+
pip install pyinstaller
53+
54+
#. Create spark cli binary::
55+
56+
make binary
57+
58+
59+
Running Tests:
60+
--------------
61+
62+
Setup
63+
#####
64+
65+
Tox, our test runner, tests against both Python 2.7 and Python 3.4 environments.
66+
67+
If you're using OS X, be sure to use the officially distributed Python 3.4 installer_ since the
68+
Homebrew version is missing a necessary library.
69+
70+
Running
71+
#######
72+
73+
Tox will run unit and integration tests in both Python environments using a temporarily created
74+
virtualenv.
75+
76+
You should ensure :code:`DCOS_CONFIG` is set and that the config file points to the Marathon
77+
instance you want to use for integration tests.
78+
79+
There are two ways to run tests, you can either use the virtualenv created by :code:`make env`
80+
above::
81+
82+
make test
83+
84+
Or, assuming you have tox installed (via :code:`sudo pip install tox`)::
85+
86+
tox
87+
88+
Other Useful Commands
89+
#####################
90+
91+
#. List all of the supported test environments::
92+
93+
tox --listenvs
94+
95+
#. Run a specific set of tests::
96+
97+
tox -e <testenv>
98+
99+
.. _packages: https://packaging.python.org/en/latest/installing.html#installing-requirements

‎cli/bin/binary.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash -e
2+
#
3+
# Copyright (C) 2015 Mesosphere, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
echo "Building binary..."
18+
pyinstaller binary/binary.spec
19+
20+
docker-check() {
21+
time=2
22+
command="/bin/sh -c \"docker ps\""
23+
24+
if hash expect 2>/dev/null; then
25+
expect -c "set echo \"-noecho\"; set timeout $time; spawn -noecho $command; expect timeout { exit 1 } eof { exit 0 }"
26+
27+
if [ $? = 1 ] ; then
28+
echo "Docker execution timed out. Make sure docker-machine start docker-vm is started."
29+
exit 0;
30+
fi
31+
fi
32+
}
33+
34+
if [ "$(uname)" == "Darwin" ]; then
35+
# Do something under Mac OS X platform
36+
mkdir -p dist/darwin
37+
mv dist/dcos-spark dist/darwin
38+
shasum -a 256 dist/darwin/dcos-spark | awk '{print $1}' > dist/darwin/dcos-spark.sha
39+
echo "Darin Build Complete!"
40+
41+
# linux build on a darwin plaform if docker runs
42+
docker-check
43+
docker rmi -f spark-binary || true
44+
docker rm spark-binary || true
45+
docker build -f binary/Dockerfile.linux-binary -t spark-binary .
46+
docker run --name spark-binary spark-binary
47+
mkdir -p dist/linux
48+
docker cp spark-binary:/dcos-spark/dist/linux dist/
49+
50+
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
51+
# Do something under GNU/Linux platform #statements
52+
mkdir -p dist/linux
53+
mv dist/dcos-spark dist/linux
54+
sha256sum dist/linux/dcos-spark | awk '{print $1}' > dist/linux/dcos-spark.sha
55+
echo "Linux Build Complete"
56+
fi
57+
58+
echo "Build finished!"

‎cli/bin/build-go.sh‎

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎cli/bin/build-python-wrapper.sh‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎cli/bin/clean.sh‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
1-
#!/bin/bash
1+
#!/bin/bash -e
22

3-
set +x
3+
set -eux
44

5-
BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
6-
cd $BIN_DIR/..
5+
BASEDIR=`dirname $0`/..
76

8-
rm -f dcos-spark/dcos-spark*
9-
rm -rf python/build/ python/dist/ python/bin_wrapper.egg-info/
7+
rm -rf $BASEDIR/.tox $BASEDIR/env $BASEDIR/build $BASEDIR/dist
8+
echo "Deleted virtualenv and test artifacts."

‎cli/bin/env.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
BASEDIR=`dirname $0`/..
6+
7+
if [ ! -d "$BASEDIR/env" ]; then
8+
virtualenv -q $BASEDIR/env --prompt='(dcos-spark) '
9+
echo "Virtualenv created."
10+
fi
11+
12+
if [ -f "$BASEDIR/env/bin/activate" ]; then
13+
source $BASEDIR/env/bin/activate
14+
else
15+
$BASEDIR/env/Scripts/activate
16+
fi
17+
18+
echo "Virtualenv activated."
19+
20+
if [ ! -f "$BASEDIR/env/updated" -o $BASEDIR/setup.py -nt $BASEDIR/env/updated ]; then
21+
pip install -e $BASEDIR
22+
touch $BASEDIR/env/updated
23+
echo "Requirements installed."
24+
fi
25+
26+
pip install -r $BASEDIR/requirements.txt

0 commit comments

Comments
 (0)