Skip to content
This repository was archived by the owner on Oct 5, 2023. It is now read-only.

Commit eaabf70

Browse files
buggtbbuggtb
authored
Merge pull request #1 from unity-sds/sps
initial commit
2 parents 8588d2c + f2f7a59 commit eaabf70

21 files changed

+1616
-0
lines changed

‎charts/sps/.helmignore

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎charts/sps/Chart.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v2
2+
name: sps
3+
description: SPS Helm Chart
4+
5+
type: application
6+
version: 0.1.0
7+
appVersion: "0.1.0"
8+

‎charts/sps/celeryconfig.py

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
broker_url = "amqp://guest:guest@rabbitmq:5672//"
2+
result_backend = "redis://:redis"
3+
4+
task_serializer = "msgpack"
5+
result_serializer = "msgpack"
6+
accept_content = ["msgpack"]
7+
8+
task_acks_late = True
9+
result_expires = 86400
10+
worker_prefetch_multiplier = 1
11+
12+
event_serializer = "msgpack"
13+
worker_send_task_events = True
14+
task_send_sent_event = True
15+
task_track_started = True
16+
17+
task_queue_max_priority = 10
18+
19+
task_reject_on_worker_lost = True
20+
21+
broker_heartbeat = 120
22+
broker_heartbeat_checkrate = 2
23+
24+
broker_pool_limit = None
25+
broker_transport_options = { "confirm_publish": True }
26+
27+
imports = [
28+
"hysds.task_worker",
29+
"hysds.job_worker",
30+
"hysds.orchestrator",
31+
]
32+
33+
CELERY_SEND_TASK_ERROR_EMAILS = False
34+
ADMINS = (
35+
('{{ ADMIN_NAME }}', '{{ ADMIN_EMAIL }}'),
36+
)
37+
SERVER_EMAIL = '{{ HOST_STRING }}'
38+
39+
HYSDS_HANDLE_SIGNALS = False
40+
HYSDS_JOB_STATUS_EXPIRES = 86400
41+
42+
BACKOFF_MAX_VALUE = 64
43+
BACKOFF_MAX_TRIES = 10
44+
45+
HARD_TIME_LIMIT_GAP = 300
46+
47+
PYMONITOREDRUNNER_CFG = {
48+
"rabbitmq": {
49+
"hostname": "{{ MOZART_RABBIT_PVT_IP }}",
50+
"port": 5672,
51+
"queue": "stdouterr"
52+
},
53+
54+
"StreamObserverFileWriter": {
55+
"stdout_filepath": "_stdout.txt",
56+
"stderr_filepath": "_stderr.txt"
57+
},
58+
59+
"StreamObserverMessenger": {
60+
"send_interval": 1
61+
}
62+
}
63+
64+
MOZART_URL = "https://{{ MOZART_PVT_IP }}/mozart/"
65+
MOZART_REST_URL = "http://{{ MOZART_PVT_IP }}:8888/api/v0.1"
66+
JOBS_ES_URL = "http://mozart-es:9200"
67+
JOBS_PROCESSED_QUEUE = "jobs_processed"
68+
USER_RULES_JOB_QUEUE = "user_rules_job"
69+
ON_DEMAND_JOB_QUEUE = "on_demand_job"
70+
USER_RULES_JOB_INDEX = "user_rules-mozart"
71+
STATUS_ALIAS = "job_status"
72+
73+
TOSCA_URL = "https://{{ GRQ_PVT_IP }}/search/"
74+
GRQ_URL = "http://{{ GRQ_PVT_IP }}:{{ GRQ_PORT }}"
75+
GRQ_REST_URL = "http://{{ GRQ_PVT_IP }}:{{ GRQ_PORT }}/api/v0.1"
76+
GRQ_UPDATE_URL = "http://{{ GRQ_PVT_IP }}:{{ GRQ_PORT }}/api/v0.1/grq/dataset/index"
77+
78+
79+
GRQ_AWS_ES = False
80+
GRQ_ES_HOST = "127.0.0.1"
81+
GRQ_ES_PORT = 9200
82+
GRQ_ES_PROTOCOL = "http"
83+
GRQ_ES_URL = '%s://%s:%d' % (GRQ_ES_PROTOCOL, GRQ_ES_HOST, GRQ_ES_PORT)
84+
85+
86+
DATASET_PROCESSED_QUEUE = "dataset_processed"
87+
USER_RULES_DATASET_QUEUE = "user_rules_dataset"
88+
ON_DEMAND_DATASET_QUEUE = "on_demand_dataset"
89+
USER_RULES_DATASET_INDEX = "user_rules-grq"
90+
DATASET_ALIAS = "grq"
91+
92+
HYSDS_IOS_MOZART = "hysds_ios-mozart"
93+
HYSDS_IOS_GRQ = "hysds_ios-grq"
94+
95+
USER_RULES_TRIGGER_QUEUE = "user_rules_trigger"
96+
97+
PROCESS_EVENTS_TASKS_QUEUE = "process_events_tasks"
98+
99+
METRICS_ES_URL = "http://{{ METRICS_ES_PVT_IP }}:9200"
100+
101+
# REDIS_JOB_STATUS_URL = "redis://:{{ MOZART_REDIS_PASSWORD }}@{{ MOZART_REDIS_PVT_IP }}"
102+
REDIS_JOB_STATUS_URL = "redis://:redis"
103+
REDIS_JOB_STATUS_KEY = "logstash"
104+
REDIS_JOB_INFO_URL = "redis://:{{ METRICS_REDIS_PASSWORD }}@{{ METRICS_REDIS_PVT_IP }}"
105+
REDIS_JOB_INFO_KEY = "logstash"
106+
REDIS_INSTANCE_METRICS_URL = "redis://:{{ METRICS_REDIS_PASSWORD }}@{{ METRICS_REDIS_PVT_IP }}"
107+
REDIS_INSTANCE_METRICS_KEY = "logstash"
108+
# REDIS_UNIX_DOMAIN_SOCKET = "unix://:{{ MOZART_REDIS_PASSWORD }}@/tmp/redis.sock"
109+
REDIS_UNIX_DOMAIN_SOCKET = "unix://:/tmp/redis.sock"
110+
111+
WORKER_CONTIGUOUS_FAILURE_THRESHOLD = 10
112+
WORKER_CONTIGUOUS_FAILURE_TIME = 5.
113+
114+
ROOT_WORK_DIR = "/data/work"
115+
WEBDAV_URL = None
116+
WEBDAV_PORT = 8085
117+
118+
WORKER_MOUNT_BLACKLIST = [
119+
"/dev",
120+
"/etc",
121+
"/lib",
122+
"/proc",
123+
"/usr",
124+
"/var",
125+
]
126+
127+
CONTAINER_REGISTRY = "{{ CONTAINER_REGISTRY }}"
128+
129+
AWS_REGION = "{{ AWS_REGION }}"

‎charts/sps/event_status.template.json

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"mappings": {
3+
"properties": {
4+
"resource": {
5+
"type": "keyword",
6+
"ignore_above": 256,
7+
"copy_to": ["text_fields"]
8+
},
9+
"type": {
10+
"type": "keyword",
11+
"ignore_above": 256,
12+
"copy_to": ["text_fields"]
13+
},
14+
"status": {
15+
"type": "keyword",
16+
"ignore_above": 256,
17+
"copy_to": ["text_fields"]
18+
},
19+
"timestamp": {
20+
"type": "date"
21+
},
22+
"hostname": {
23+
"type": "keyword",
24+
"ignore_above": 256,
25+
"copy_to": ["text_fields"]
26+
},
27+
"uuid": {
28+
"type": "keyword",
29+
"ignore_above": 256,
30+
"copy_to": ["text_fields"]
31+
},
32+
"tags": {
33+
"type": "text",
34+
"fields": {
35+
"keyword": {
36+
"type": "keyword",
37+
"ignore_above": 256
38+
}
39+
},
40+
"copy_to": ["text_fields"]
41+
},
42+
"event": {
43+
"type": "object",
44+
"enabled": false
45+
},
46+
"text_fields": {
47+
"type": "text"
48+
}
49+
}
50+
},
51+
"index_patterns": ["event_status*"],
52+
"settings": {
53+
"index.refresh_interval": "5s",
54+
"analysis": {
55+
"analyzer": {
56+
"default": {
57+
"filter": ["lowercase", "word_delimiter"],
58+
"tokenizer": "keyword"
59+
}
60+
}
61+
}
62+
},
63+
"aliases": {
64+
"job_status": {}
65+
}
66+
}

0 commit comments

Comments
 (0)