forked from ml-lab/DeepVideoAnalytics
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdvactl
executable file
·169 lines (156 loc) · 6.47 KB
/
dvactl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
#!/usr/bin/env python
import argparse
import os
import json
import random
from deploy import compose, kube
try:
import requests
except ImportError:
print "Warning! requests module is required to use exec functionality."
def get_mode():
if not os.path.isfile("config.json"):
return configure()
return json.load(file("config.json"))
def configure_kube():
config_template = {
"dbusername": "pguser",
"dbpassword": "pgpass",
"rabbithost": "rbbit",
"rabbitusername": "rabbituser",
"rabbitpassword": "rabbitpass",
"awskey": "none",
"awssecret": "none",
"mediabucket": "dvamedia_{random_string}",
"secretkey": "{random_string}",
"superuser": "admin",
"superpass": "please_change_this",
"superemail": "admin@test.com",
"cloudfsprefix": "gs",
"cors_origin": "*",
"redishost": "redis-master",
"redispassword": "1234567890",
"zone": "us-west1-b",
"cluster_name": "dvacluster",
"machine_type": "custom-22-84480",
"nodes": 1,
"branch": "stable"
}
config = {"mode": "kube"}
print "Creating configuration for kubernetes from kubeconfig.example.json"
if os.path.isfile('config.json'):
existing_config = json.load(file('config.json'))
else:
existing_config = {}
for k, v in sorted(config_template.items()):
if k in existing_config:
v = existing_config[k]
if (type(v) is str or type(v) is unicode) and "{random_string}" in v:
v = v.format(random_string=random.randint(0, 100000000))
new_value = raw_input(
"Enter value for {} (Current value is '{}' press enter to keep current value) >>".format(k, v))
if new_value.strip():
if type(v) is int:
config[k] = int(new_value)
else:
config[k] = new_value
else:
config[k] = v
return config
def configure_compose(mode):
gpu_count = 0
init_process = '/root/DVA/configs/custom_defaults/init_process.json'
init_models = '/root/DVA/configs/custom_defaults/trained_models.json'
process = raw_input("Please specify init process or press enter to keep default"
" ( /root/DVA/configs/custom_defaults/init_process.json ) >>").strip()
if process.strip():
init_process = process
models = raw_input("Please specify default models or press enter to keep default"
" ( /root/DVA/configs/custom_defaults/trained_models.json ) >>").strip()
if process.strip():
init_models = models
envs = {}
if mode == 'test':
envs['ENABLE_CLOUDFS'] = 1
try:
envs.update(compose.test.load_envs(os.path.expanduser('~/media.env')))
print '~/media.env found. writing bucket information to config.json'
except:
print '~/media.env not found. required for testing rfs mode.'
try:
envs.update(compose.test.load_envs(os.path.expanduser('~/aws.env')))
print '~/aws.env found. writing credentials to config.json'
except:
print '~/aws.env not found. required for testing rfs mode.'
else:
if os.path.isfile(os.path.expanduser('~/aws.env')):
envs.update(compose.test.load_envs(os.path.expanduser('~/aws.env')))
print '~/aws.env found. writing credentials to config.json'
else:
print '{} not found. not passing AWS creds.'.format(os.path.expanduser('~/aws.env'))
if os.path.isfile(os.path.expanduser('~/do.env')):
envs.update(compose.test.load_envs(os.path.expanduser('~/do.env')))
print '~/do.env found. writing credentials to config.json'
else:
print '{} not found. not passing Digital Ocean creds.'.format(os.path.expanduser('~/do.env'))
if mode == 'gpu':
gpu_count = int(raw_input("Please select number of GPUs >>").strip())
return {"mode": mode, 'gpus': gpu_count, 'init_process': init_process, 'init_models': init_models,
'credentials': envs}
def configure(mode=None):
if mode is None or not mode:
mode = raw_input("Please select mode { dev, test, cpu, gpu, kube } >>").strip()
if mode not in {'dev', 'test', 'cpu', 'gpu', 'kube'}:
raise ValueError("{} is not a valid mode".format(mode))
if mode == 'kube':
mode_dict = configure_kube()
else:
mode_dict = configure_compose(mode)
with open("config.json", 'w') as f:
json.dump(mode_dict, f, indent=4)
print "Saved config.json"
return mode_dict
def exec_script(script_path):
creds = json.load(file('creds.json'))
server, token = creds['server'], creds['token']
headers = {'Authorization': 'Token {}'.format(token)}
r = requests.post("{server}queries/".format(server=server), data={'script': file(script_path).read()},
headers=headers)
r.raise_for_status()
if r.ok:
print r.json()
if __name__ == '__main__':
help_text = """
Available options
./dvactl configure
./dvactl create
./dvactl create_premptible # create premptible node pool for Kubernetes
./dvactl start
./dvactl auth # recreates creds.json
./dvactl exec -f script.json # run process using creds.json and REST API
./dvactl stop
./dvactl clean
"""
parser = argparse.ArgumentParser()
parser.add_argument("action",
help="Select action out of { configure | create | start | auth | exec | stop | clean "
"| clean_restart ")
parser.add_argument("-f", help="path to script to exec, e.g. process_livestream.json",
default="")
args = parser.parse_args()
if args.action == 'configure':
configure()
elif args.action == 'exec':
if args.f.strip():
exec_script(args.f)
else:
raise ValueError("Please specify script path e.g. ./dvactl exec -f test.json")
else:
mode_dict = get_mode()
if mode_dict['mode'] == 'kube':
kube.handle_kube_operations(args)
else:
if args.action == 'create':
raise ValueError("create is not required for compose, its used in kube mdoe to create GKE cluster")
compose.handle_compose_operations(args, mode_dict['mode'], mode_dict['gpus'], mode_dict['init_process'],
mode_dict['init_models'], mode_dict['credentials'])