Skip to content

Commit 2045cb2

Browse files
author
Arthur Rand
authored
[Spark-339][WIP] golang cli (apache#168)
* Add support for Kerberos args to Go CLI module Tested by launching a job with phony Kerberos args against the Python CLI and then the Go CLI with the same args, then checking the resulting job for differences. From that testing, assuming Python works, Go should work too.. * wip, decode base64 secrets * improved logging * change makefile back * makefile... * use libmesos bundle instead of private image * update docs and remove tgt? * remove dead code * update cli module * fix typo * fixed hdfs.md with tgt instructions * remove /dispatcher add universe.sh * added DCOS_SPACE support * small fixes, need to change tgt thing again to use envvar * Remove docker login. * change jenkins script to reflect no more dispatcher dir * make error * remove dead code * added debug logging * move cli version thing around * envvar madness * more logging * remove envvar to build_cli * revert docker_login, back to jenkins * remove dead go code * use envvar for tgt instead of spark flag * intermediate commit, added logging, experimenting with R commands * reverted some logging * R support * added py-files flag * quick check, worked locally * handle spark application flags * try again for tests, looks ok? * oops, fix * fix foldered dispatcher? * put name in * explicit args to run_tests * cleaned up a litle * test testing protocol, prev released candidate * tryout custom distribution * enable secrets and cni test * updated cni and cni-labels tests * added some version of tools * don't fetch tools * chasing failure around, set commons tools to new dir * fix path in run_tests * fix envvar mistake * one more try.. * disable secrets, added supervise * fix supervise test? * fix streaming test * retry secrets test * better wait for spark jobs to be killed * quick change to delete_secret * disable flaky secrets test, for now, try new dist * enable secrets test again * remove older cli tests, hopefully fix delete_secrets * move creating secrets to setup * revert back to post-style secret generation * change principal to service_account * update manifest
1 parent 921332c commit 2045cb2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+5492
-1611
lines changed

‎.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
.idea/
22
.cache/
33
build/
4-
dcos-commons-tools/
54
tests/env
65
tests/venv
76
tests/.cache

‎Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ dist:
44
docker: dist
55
bin/docker.sh
66

7+
universe:
8+
bin/universe.sh
9+
710
test:
811
bin/test.sh
912

10-
.PHONY: dist docker test
13+
.PHONY: dist docker test universe

‎bin/dcos-commons-tools/airgap_linter.py

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Validates that the supplied framework (identified by the directory passed in) will function, to the best
4+
# of our knowledge, in an airgapped cluster.
5+
#
6+
# The checks are:
7+
# - No external URIs defined in anything but resource.json
8+
# - No images defined in anything but resource.json (and templated into the svc.yml)
9+
#
10+
11+
import re
12+
import sys
13+
import os
14+
15+
16+
def extract_uris(file_name):
17+
with open(file_name, "r") as file:
18+
lines = file.readlines()
19+
20+
matcher = re.compile(".*https?:\/\/([^\/\?]*)", re.IGNORECASE)
21+
matches = []
22+
for line in lines:
23+
# Do not grab comments
24+
if line.startswith("*") or line.startswith("#") or line.startswith("//"):
25+
continue
26+
# Do not grab "id" lines
27+
if "\"id\":" in line:
28+
continue
29+
30+
match = matcher.match(line)
31+
if match:
32+
matches.append(match.group(1))
33+
34+
return matches
35+
36+
37+
def validate_uris_in(file_name):
38+
uris = extract_uris(file_name)
39+
40+
bad_uri = False
41+
for uri in uris:
42+
# A FQDN is a valid internal FQDN if it contains .dcos or ends with .mesos.
43+
if not (".dcos" in uri or uri.endswith(".mesos")):
44+
print("Found a bad URI:", uri, "in:", file_name)
45+
bad_uri = True
46+
47+
return not bad_uri
48+
49+
50+
def get_files_to_check_for_uris(framework_directory):
51+
# There's a set of files that will always be present.
52+
files = [os.path.join(framework_directory, "universe", "config.json"),
53+
os.path.join(framework_directory, "universe", "marathon.json.mustache")]
54+
55+
# Always check every file in the `dist` directory of the scheduler.
56+
dist_dir = os.path.join(framework_directory, "src", "main", "dist")
57+
58+
for dp, dn, filenames in os.walk(dist_dir):
59+
for file in filenames:
60+
files.append(os.path.join(dp, file))
61+
62+
return files
63+
64+
65+
def validate_all_uris(framework_directory):
66+
bad_file = False
67+
files = get_files_to_check_for_uris(framework_directory)
68+
for file in files:
69+
if not validate_uris_in(file):
70+
bad_file = True
71+
72+
return not bad_file
73+
74+
75+
def validate_images(framework_directory):
76+
svc_yml = os.path.join(framework_directory, "src", "main", "dist", "svc.yml")
77+
78+
with open(svc_yml, "r") as file:
79+
lines = file.readlines()
80+
81+
bad_image = False
82+
for line in lines:
83+
line = line.strip()
84+
if "image:" in line:
85+
image_matcher = re.compile("image:\s?(.*)$", re.IGNORECASE)
86+
match = image_matcher.match(line)
87+
image_path = match.group(1)
88+
env_var_matcher = re.compile("\{\{[A-Z0-9_]*\}\}")
89+
if not env_var_matcher.match(image_path):
90+
print("Bad image found in svc.yml. It is a direct reference instead of a templated reference:", image_path)
91+
bad_image = True
92+
93+
return not bad_image
94+
95+
96+
def print_help():
97+
print("""Scans a framework for any airgap issues. Checks all files for external URIs,
98+
and docker images for direct references
99+
100+
usage: python airgap_linter.py <framework-directory>""")
101+
102+
103+
def main(argv):
104+
if len(argv) < 2:
105+
print_help()
106+
sys.exit(0)
107+
108+
framework_directory = argv[1]
109+
110+
if not os.path.isdir(framework_directory):
111+
print("Supplied framework directory", framework_directory, "does not exist or is not a directory.")
112+
113+
uris_valid = validate_all_uris(framework_directory)
114+
images_valid = validate_images(framework_directory)
115+
116+
invalid = False
117+
if not uris_valid:
118+
invalid = True
119+
120+
if not images_valid:
121+
invalid = True
122+
123+
if invalid:
124+
print("Airgap check FAILED. This framework will NOT work in an airgap. Fix the detected issues.")
125+
sys.exit(1)
126+
127+
print("Airgap check complete. This framework will _probably_ work in an airgap, but for the love of everything test that.")
128+
sys.exit(0)
129+
130+
131+
if __name__ == '__main__':
132+
sys.exit(main(sys.argv))

‎bin/dcos-commons-tools/branch_changes.py

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/python3
2+
3+
import subprocess
4+
import sys
5+
6+
def fetch_origin_master():
7+
subprocess.check_call([b'git', b'fetch', b'origin', b'master', b'--quiet'])
8+
9+
def get_changed_files():
10+
fetch_origin_master()
11+
12+
output = subprocess.check_output([b'git', b'diff', b'origin/master',
13+
b'--name-only'])
14+
files = output.splitlines()
15+
return files
16+
17+
def categorize_file(filename):
18+
if filename.endswith(b".md"):
19+
return "DOC"
20+
if filename.startswith(b"docs/"):
21+
return "DOC"
22+
if filename.startswith(b"frameworks/"):
23+
literal, specific_framework, rest = filename.split(b'/', 2)
24+
return specific_framework.decode('utf-8')
25+
return "SDK"
26+
27+
def categorize_branch_changes():
28+
file_types = set()
29+
for filename in get_changed_files():
30+
file_type = categorize_file(filename)
31+
file_types.add(file_type)
32+
33+
# any SDK changes means build/test everything
34+
if "SDK" in file_types:
35+
return {"SDK"}
36+
# only DOC changes means build/test nothing
37+
if file_types == {"DOC"}:
38+
return {"DOC"}
39+
# Otherwise return the frameworks changed
40+
file_types.discard("DOC")
41+
return file_types
42+
43+
saved_branch_changes = None
44+
def memoized_branch_changes():
45+
global saved_branch_changes
46+
if not saved_branch_changes:
47+
saved_branch_changes = categorize_branch_changes()
48+
return saved_branch_changes
49+
50+
get_branch_changetypes=memoized_branch_changes
51+
52+
__all__ = ('get_branch_changetypes',)
53+
54+
if __name__ == "__main__":
55+
for item in get_branch_changetypes():
56+
sys.stdout.write(item + "\n")

‎bin/dcos-commons-tools/build_cli.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
# exit immediately on failure
4+
set -e
5+
6+
if [ $# -lt 2 ]; then
7+
echo "Syntax: $0 <cli-exe-name> </path/to/framework/cli> <repo-relative/path/to/framework/cli>"
8+
exit 1
9+
fi
10+
11+
CLI_EXE_NAME=$1
12+
CLI_DIR=$2
13+
REPO_CLI_RELATIVE_PATH=$3 # eg 'frameworks/helloworld/cli/'
14+
15+
source $TOOLS_DIR/init_paths.sh
16+
17+
# ---
18+
19+
# go
20+
$TOOLS_DIR/build_go_exe.sh $REPO_CLI_RELATIVE_PATH/$CLI_EXE_NAME windows
21+
$TOOLS_DIR/build_go_exe.sh $REPO_CLI_RELATIVE_PATH/$CLI_EXE_NAME darwin
22+
$TOOLS_DIR/build_go_exe.sh $REPO_CLI_RELATIVE_PATH/$CLI_EXE_NAME linux

‎bin/dcos-commons-tools/build_framework.sh

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#!/bin/bash
2+
3+
# Prevent jenkins from immediately killing the script when a step fails, allowing us to notify github:
4+
set +e
5+
6+
usage() {
7+
echo "Syntax: $0 [--cli-only] <framework-name> </path/to/framework> [local|aws]"
8+
}
9+
10+
cli_only=
11+
12+
while :; do
13+
case $1 in
14+
--help|-h|-\?)
15+
usage
16+
exit
17+
;;
18+
--cli-only)
19+
cli_only="true"
20+
;;
21+
--)
22+
shift
23+
break
24+
;;
25+
-*)
26+
echo "unknown option $1" >&2
27+
exit 1
28+
;;
29+
*)
30+
break
31+
;;
32+
esac
33+
34+
shift
35+
done
36+
37+
38+
if [ $# -lt 2 ]; then
39+
exit 1
40+
fi
41+
42+
PUBLISH_STEP=$1
43+
shift
44+
FRAMEWORK_NAME=$1
45+
shift
46+
FRAMEWORK_DIR=$1
47+
shift
48+
ARTIFACT_FILES=$@
49+
50+
51+
echo PUBLISH_STEP=$PUBLISH_STEP
52+
echo FRAMEWORK_NAME=$FRAMEWORK_NAME
53+
echo FRAMEWORK_DIR=$FRAMEWORK_DIR
54+
echo ARTIFACT_FILES=$ARTIFACT_FILES
55+
56+
# default paths/names within a framework directory:
57+
CLI_DIR=${CLI_DIR:=${FRAMEWORK_DIR}/cli}
58+
UNIVERSE_DIR=${UNIVERSE_DIR:=${FRAMEWORK_DIR}/universe}
59+
CLI_EXE_NAME=${CLI_EXE_NAME:=dcos-${FRAMEWORK_NAME}}
60+
BUILD_BOOTSTRAP=${BUILD_BOOTSTRAP:=yes}
61+
62+
source $TOOLS_DIR/init_paths.sh
63+
64+
# GitHub notifier config
65+
_notify_github() {
66+
GIT_REPOSITORY_ROOT=$REPO_ROOT_DIR ${TOOLS_DIR}/github_update.py $1 build:${FRAMEWORK_NAME} $2
67+
}
68+
69+
# Used below in-order, but here for cli-only
70+
build_cli() {
71+
# CLI (Go):
72+
# /home/user/dcos-commons/frameworks/helloworld/cli => frameworks/helloworld/cli
73+
REPO_CLI_RELATIVE_PATH="$(echo $CLI_DIR | cut -c $((2 + ${#REPO_ROOT_DIR}))-)"
74+
${TOOLS_DIR}/build_cli.sh ${CLI_EXE_NAME} ${CLI_DIR} ${REPO_CLI_RELATIVE_PATH}
75+
if [ $? -ne 0 ]; then
76+
_notify_github failure "CLI build failed"
77+
exit 1
78+
fi
79+
}
80+
81+
if [ x"$cli_only" = xtrue ]; then
82+
build_cli
83+
exit
84+
fi
85+
86+
_notify_github pending "Build running"
87+
88+
# Verify airgap (except for hello world)
89+
if [ $FRAMEWORK_NAME != "hello-world" ];
90+
then
91+
${TOOLS_DIR}/airgap_linter.py ${FRAMEWORK_DIR}
92+
fi
93+
94+
# Ensure executor build up to date
95+
${REPO_ROOT_DIR}/gradlew distZip -p ${REPO_ROOT_DIR}/sdk/executor
96+
97+
# Service (Java):
98+
${REPO_ROOT_DIR}/gradlew -p ${FRAMEWORK_DIR} check distZip
99+
if [ $? -ne 0 ]; then
100+
_notify_github failure "Gradle build failed"
101+
exit 1
102+
fi
103+
104+
INCLUDE_BOOTSTRAP=""
105+
if [ "$BUILD_BOOTSTRAP" == "yes" ]; then
106+
# Executor Bootstrap (Go):
107+
BOOTSTRAP_DIR=${TOOLS_DIR}/../sdk/bootstrap
108+
${BOOTSTRAP_DIR}/build.sh
109+
if [ $? -ne 0 ]; then
110+
_notify_github failure "Bootstrap build failed"
111+
exit 1
112+
fi
113+
INCLUDE_BOOTSTRAP="${BOOTSTRAP_DIR}/bootstrap.zip"
114+
fi
115+
116+
build_cli
117+
118+
_notify_github success "Build succeeded"
119+
120+
case "$PUBLISH_STEP" in
121+
local)
122+
echo "Launching HTTP artifact server"
123+
PUBLISH_SCRIPT=${TOOLS_DIR}/publish_http.py
124+
;;
125+
aws)
126+
echo "Uploading to S3"
127+
PUBLISH_SCRIPT=${TOOLS_DIR}/publish_aws.py
128+
;;
129+
*)
130+
echo "---"
131+
echo "Build complete, skipping publish step."
132+
echo "Use one of the following additional arguments to get something that runs on a cluster:"
133+
echo "- 'local': Host the build in a local HTTP server for use by a DC/OS Vagrant cluster."
134+
echo "- 'aws': Upload the build to S3."
135+
;;
136+
esac
137+
138+
if [ -n "$PUBLISH_SCRIPT" ]; then
139+
$PUBLISH_SCRIPT \
140+
${FRAMEWORK_NAME} \
141+
${UNIVERSE_DIR} \
142+
${INCLUDE_BOOTSTRAP} \
143+
${CLI_DIR}/dcos-*/dcos-*-linux \
144+
${CLI_DIR}/dcos-*/dcos-*-darwin \
145+
${CLI_DIR}/dcos-*/dcos-*.exe \
146+
${ARTIFACT_FILES}
147+
fi

0 commit comments

Comments
 (0)