Add DAG manager

Makefile

@@ -21,10 +21,7 @@ SHELL := /bin/bash
 # Cortex
 
 devstart:
-	@kill $(shell pgrep -f rerun); ./dev/operator_local.sh || true
-
-killdev:
-	@kill $(shell pgrep -f rerun)
+	@./dev/operator_local.sh || true
 
 kubectl:
 	@eksctl utils write-kubeconfig --name="cortex"
@@ -138,8 +135,6 @@ ci-build-images:
 	@./build/build-image.sh images/fluentd fluentd
 	@./build/build-image.sh images/nginx-controller nginx-controller
 	@./build/build-image.sh images/nginx-backend nginx-backend
-	@./build/build-image.sh images/argo-controller argo-controller
-	@./build/build-image.sh images/argo-executor argo-executor
 	@./build/build-image.sh images/python-packager python-packager
 	@./build/build-image.sh images/cluster-autoscaler cluster-autoscaler
 	@./build/build-image.sh images/nvidia nvidia
@@ -160,8 +155,6 @@ ci-push-images:
 	@./build/push-image.sh fluentd
 	@./build/push-image.sh nginx-controller
 	@./build/push-image.sh nginx-backend
-	@./build/push-image.sh argo-controller
-	@./build/push-image.sh argo-executor
 	@./build/push-image.sh python-packager
 	@./build/push-image.sh cluster-autoscaler
 	@./build/push-image.sh nvidia

cortex.sh

@@ -117,8 +117,6 @@ export CORTEX_NODES_MAX="${CORTEX_NODES_MAX:-5}"
 export CORTEX_NAMESPACE="${CORTEX_NAMESPACE:-cortex}"
 
 export CORTEX_IMAGE_MANAGER="${CORTEX_IMAGE_MANAGER:-cortexlabs/manager:$CORTEX_VERSION_STABLE}"
-export CORTEX_IMAGE_ARGO_CONTROLLER="${CORTEX_IMAGE_ARGO_CONTROLLER:-cortexlabs/argo-controller:$CORTEX_VERSION_STABLE}"
-export CORTEX_IMAGE_ARGO_EXECUTOR="${CORTEX_IMAGE_ARGO_EXECUTOR:-cortexlabs/argo-executor:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_FLUENTD="${CORTEX_IMAGE_FLUENTD:-cortexlabs/fluentd:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_NGINX_BACKEND="${CORTEX_IMAGE_NGINX_BACKEND:-cortexlabs/nginx-backend:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_NGINX_CONTROLLER="${CORTEX_IMAGE_NGINX_CONTROLLER:-cortexlabs/nginx-controller:$CORTEX_VERSION_STABLE}"
@@ -177,8 +175,6 @@ function install_cortex() {
     -e CORTEX_NODE_TYPE=$CORTEX_NODE_TYPE \
     -e CORTEX_LOG_GROUP=$CORTEX_LOG_GROUP \
     -e CORTEX_BUCKET=$CORTEX_BUCKET \
-    -e CORTEX_IMAGE_ARGO_CONTROLLER=$CORTEX_IMAGE_ARGO_CONTROLLER \
-    -e CORTEX_IMAGE_ARGO_EXECUTOR=$CORTEX_IMAGE_ARGO_EXECUTOR \
     -e CORTEX_IMAGE_FLUENTD=$CORTEX_IMAGE_FLUENTD \
     -e CORTEX_IMAGE_NGINX_BACKEND=$CORTEX_IMAGE_NGINX_BACKEND \
     -e CORTEX_IMAGE_NGINX_CONTROLLER=$CORTEX_IMAGE_NGINX_CONTROLLER \

dev/operator_local.sh

@@ -26,6 +26,9 @@ export CONST_OPERATOR_TRANSFORMERS_DIR=$ROOT/pkg/transformers
 export CONST_OPERATOR_ESTIMATORS_DIR=$ROOT/pkg/estimators
 export CONST_OPERATOR_IN_CLUSTER=false
 
+kill $(pgrep -f rerun) >/dev/null 2>&1 || true
+
 rerun -watch $ROOT/pkg $ROOT/cli -ignore $ROOT/vendor $ROOT/bin -run sh -c \
 "go build -o $ROOT/bin/operator $ROOT/pkg/operator && go build -installsuffix cgo -o $ROOT/bin/cortex $ROOT/cli && $ROOT/bin/operator"
-# go run -race $ROOT/pkg/operator/operator.go
+
+# go run -race $ROOT/pkg/operator/operator.go  # Check for race conditions. Doesn't seem to catch them all?

dev/registry.sh

@@ -36,8 +36,6 @@ function ecr_login() {
 
 function create_registry() {
   aws ecr create-repository --repository-name=cortexlabs/manager --region=$REGISTRY_REGION || true
-  aws ecr create-repository --repository-name=cortexlabs/argo-controller --region=$REGISTRY_REGION || true
-  aws ecr create-repository --repository-name=cortexlabs/argo-executor --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/fluentd --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/nginx-backend --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/nginx-controller --region=$REGISTRY_REGION || true
@@ -138,8 +136,6 @@ elif [ "$cmd" = "update" ]; then
     build_and_push $ROOT/images/nginx-controller nginx-controller latest
     build_and_push $ROOT/images/nginx-backend nginx-backend latest
     build_and_push $ROOT/images/fluentd fluentd latest
-    build_and_push $ROOT/images/argo-controller argo-controller latest
-    build_and_push $ROOT/images/argo-executor argo-executor latest
     build_and_push $ROOT/images/tf-serve tf-serve latest
     build_and_push $ROOT/images/tf-serve-gpu tf-serve-gpu latest
     build_and_push $ROOT/images/onnx-serve-gpu onnx-serve-gpu latest

docs/cluster/config.md

@@ -40,8 +40,6 @@ export CORTEX_NAMESPACE="cortex"
 
 # Image paths
 export CORTEX_IMAGE_MANAGER="cortexlabs/manager:master"
-export CORTEX_IMAGE_ARGO_CONTROLLER="cortexlabs/argo-controller:master"
-export CORTEX_IMAGE_ARGO_EXECUTOR="cortexlabs/argo-executor:master"
 export CORTEX_IMAGE_FLUENTD="cortexlabs/fluentd:master"
 export CORTEX_IMAGE_NGINX_BACKEND="cortexlabs/nginx-backend:master"
 export CORTEX_IMAGE_NGINX_CONTROLLER="cortexlabs/nginx-controller:master"

docs/cluster/development.md

@@ -56,8 +56,6 @@ export CORTEX_NODES_MAX="5"
 export CORTEX_NAMESPACE="cortex"
 
 export CORTEX_IMAGE_MANAGER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/manager:latest"
-export CORTEX_IMAGE_ARGO_CONTROLLER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/argo-controller:latest"
-export CORTEX_IMAGE_ARGO_EXECUTOR="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/argo-executor:latest"
 export CORTEX_IMAGE_FLUENTD="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/fluentd:latest"
 export CORTEX_IMAGE_NGINX_BACKEND="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/nginx-backend:latest"
 export CORTEX_IMAGE_NGINX_CONTROLLER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/nginx-controller:latest"

go.mod

@@ -4,11 +4,10 @@
 //   go mod tidy
 //   replace these lines in go.mod:
 //     github.com/GoogleCloudPlatform/spark-on-k8s-operator v1alpha1-0.5-2.4.0
-//     github.com/argoproj/argo v2.3.0
 //     github.com/cortexlabs/yaml v2.2.4
-//     k8s.io/client-go v10.0.0
-//     k8s.io/api 89a74a8d264df0e993299876a8cde88379b940ee
-//     k8s.io/apimachinery 2b1284ed4c93a43499e781493253e2ac5959c4fd
+//     k8s.io/client-go v12.0.0
+//     k8s.io/api 7525909cc6da
+//     k8s.io/apimachinery 1799e75a0719
 //       (note: go to the commit for the client-go release and browse to Godeps/Godeps.json to find the SHAs for k8s.io/api and k8s.io/apimachinery)
 //   go mod tidy
 //   check the diff in this file
@@ -19,39 +18,23 @@ go 1.12
 
 require (
 	github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20181208011959-62db1d66dafa
-	github.com/argoproj/argo v2.3.0+incompatible
-	github.com/aws/aws-sdk-go v1.20.12
+	github.com/aws/aws-sdk-go v1.20.20
 	github.com/cortexlabs/yaml v0.0.0-20190626164117-202ab3a3d475
 	github.com/davecgh/go-spew v1.1.1
-	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
-	github.com/ghodss/yaml v1.0.0
-	github.com/go-openapi/spec v0.19.2 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/google/btree v1.0.0 // indirect
-	github.com/google/gofuzz v1.0.0 // indirect
-	github.com/googleapis/gnostic v0.3.0 // indirect
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.0
-	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/json-iterator/go v1.1.6 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/spf13/cobra v0.0.5
 	github.com/stretchr/testify v1.3.0
 	github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8
-	github.com/ugorji/go/codec v1.1.5-pre
+	github.com/ugorji/go/codec v1.1.7
 	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
-	k8s.io/api v0.0.0-20181204000039-89a74a8d264d
-	k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93
-	k8s.io/client-go v10.0.0+incompatible
-	k8s.io/klog v0.3.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 // indirect
-	sigs.k8s.io/yaml v1.1.0 // indirect
+	k8s.io/api v0.0.0-20190620084959-7cf5895f2711
+	k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719
+	k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
+	k8s.io/utils v0.0.0-20190712204705-3dccf664f023 // indirect
 )