build: Enhance Codespaces (argoproj#10599)

Signed-off-by: Alex Collins <alex_collins@intuit.com> Signed-off-by: Alex Collins <alexec@users.noreply.github.com>
agilgur5 · Mar 14, 2023 · 05e1318 · 05e1318
1 parent 941f0b6
commit 05e1318
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 90 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,7 +1,18 @@
 {
-    "build": { "dockerfile": "Dockerfile" },
-    "hostRequirements": {"cpus": 4},
-    "onCreateCommand": ".devcontainer/pre-build.sh",
-    "workspaceMount": "source=${localWorkspaceFolder},target=${localEnv:HOME}/go/src/github.com/argoproj/argo-workflows,type=bind",
-    "workspaceFolder": "${localEnv:HOME}/go/src/github.com/argoproj/argo-workflows"
+  "image": "mcr.microsoft.com/vscode/devcontainers/base:ubuntu",
+  "features": {
+    "ghcr.io/devcontainers/features/go:1": {
+      "version": "1.19"
+    },
+    "ghcr.io/devcontainers/features/node:1": {
+      "version": "16"
+    },
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+  },
+  "hostRequirements": {
+    "cpus": 4
+  },
+  "onCreateCommand": ".devcontainer/pre-build.sh",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/vscode/go/src/github.com/argoproj/argo-workflows,type=bind",
+  "workspaceFolder": "/home/vscode/go/src/github.com/argoproj/argo-workflows"
 }
diff --git a/.devcontainer/pre-build.sh b/.devcontainer/pre-build.sh
@@ -6,8 +6,14 @@ wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
 k3d cluster get k3s-default || k3d cluster create --wait
 k3d kubeconfig merge --kubeconfig-merge-default
 
+# install kubectl
+curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl
+chmod +x ./kubectl
+sudo mv ./kubectl /usr/local/bin/kubectl
+kubectl cluster-info
+
 # install kit
 curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | sh
 
-# do time consuming tasks, e.g. download deps and initial build
-kit build
+# download dependencies and do first-pass compile
+CI=1 kit pre-up
diff --git a/.dockerignore b/.dockerignore
@@ -3,6 +3,7 @@
 *.iml
 *.md
 *.yaml
+.devcontainer
 # The .git folder gets modified every time git runs preventing caching of builds. Once removed, we must pass GIT_COMMIT as Docker build-args.
 .git
 .github
@@ -19,6 +20,7 @@ manifests
 plugins
 sdks
 site
+tasks.yaml
 test/e2e
 ui/dist
 ui/node_modules

diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -140,25 +140,18 @@ jobs:
           echo '127.0.0.1 azurite'  | sudo tee -a /etc/hosts
       - run: make install PROFILE=${{matrix.profile}} STATIC_FILES=false
         name: Install manifests
-      - run: make controller $(go env GOPATH)/bin/goreman STATIC_FILES=false
+      - run: make controller kit STATIC_FILES=false
         name: Build controller
       - run: make cli STATIC_FILES=false
         if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
         name: Build CLI
-      - run:  ./hack/port-forward.sh
-        name: Start port forward
-      - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false LOGS=false > /tmp/argo.log 2>&1 &
+      - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false > /tmp/argo.log 2>&1 &
         name: Start controller/API
-      - run: make wait
+      - run: make wait API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
         timeout-minutes: 4
         name: Wait for controller to be up
       - name: Run tests ${{matrix.test}}
-        # https://github.com/marketplace/actions/retry-step
-        uses: nick-fields/retry@v2.8.3
-        with:
-          timeout_minutes: 20
-          max_attempts: 2
-          command: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false
+        run: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false
       - if: ${{ failure() }}
         name: MinIO/MySQL deployment
         run: |

diff --git a/Makefile b/Makefile
@@ -17,8 +17,6 @@ RELEASE_TAG           := $(shell if [[ "$(GIT_TAG)" =~ ^v[0-9]+\.[0-9]+\.[0-9]+.
 DEV_BRANCH            := $(shell [ "$(GIT_BRANCH)" = master ] || [ `echo $(GIT_BRANCH) | cut -c -8` = release- ] || [ `echo $(GIT_BRANCH) | cut -c -4` = dev- ] || [ $(RELEASE_TAG) = true ] && echo false || echo true)
 SRC                   := $(GOPATH)/src/github.com/argoproj/argo-workflows
 
-GREP_LOGS             := ""
-
 
 # docker image publishing options
 IMAGE_NAMESPACE       ?= quay.io/argoproj
@@ -53,14 +51,17 @@ else
 STATIC_FILES          ?= $(shell [ $(DEV_BRANCH) = true ] && echo false || echo true)
 endif
 
-# start the Controller
-CTRL                  ?= true
-# tail logs
-LOGS                  ?= $(CTRL)
 # start the UI
-UI                    ?= $(shell [ $(CTRL) = true ] && echo false || echo true)
+UI                    ?= false
 # start the Argo Server
 API                   ?= $(UI)
+TASKS                 := controller
+ifeq ($(API),true)
+TASKS                 := controller server
+endif
+ifeq ($(UI),true)
+TASKS                 := controller server ui
+endif
 GOTEST                ?= go test -v -p 20
 PROFILE               ?= minimal
 PLUGINS               ?= $(shell [ $PROFILE = plugins ] && echo false || echo true)
@@ -453,26 +454,29 @@ dist/argosay:
 	mkdir -p dist
 	cp test/e2e/images/argosay/v2/argosay dist/
 
-$(GOPATH)/bin/goreman:
-	go install github.com/mattn/goreman@v0.3.11
+.PHONY: kit
+kit:
+ifeq ($(shell command -v kit),)
+ifeq ($(shell uname),Darwin)
+	brew tap kitproj/kit --custom-remote https://github.com/kitproj/kit
+	brew install kit
+else
+	curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | sh
+endif
+endif
+
 
 .PHONY: start
 ifeq ($(RUN_MODE),local)
 ifeq ($(API),true)
-start: install controller cli $(GOPATH)/bin/goreman
+start: install controller kit cli
 else
-start: install controller $(GOPATH)/bin/goreman
+start: install controller kit
 endif
 else
-start: install
+start: install kit
 endif
 	@echo "starting STATIC_FILES=$(STATIC_FILES) (DEV_BRANCH=$(DEV_BRANCH), GIT_BRANCH=$(GIT_BRANCH)), AUTH_MODE=$(AUTH_MODE), RUN_MODE=$(RUN_MODE), MANAGED_NAMESPACE=$(MANAGED_NAMESPACE)"
-ifneq ($(CTRL),true)
-	@echo "⚠️️  not starting controller. If you want to test the controller, use 'make start CTRL=true' to start it"
-endif
-ifneq ($(LOGS),true)
-	@echo "⚠️️  not starting logs. If you want to tail logs, use 'make start LOGS=true' to start it"
-endif
 ifneq ($(API),true)
 	@echo "⚠️️  not starting API. If you want to test the API, use 'make start API=true' to start it"
 endif
@@ -490,18 +494,10 @@ endif
 	grep '127.0.0.1.*minio' /etc/hosts
 	grep '127.0.0.1.*postgres' /etc/hosts
 	grep '127.0.0.1.*mysql' /etc/hosts
-	./hack/port-forward.sh
 ifeq ($(RUN_MODE),local)
-	env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) CTRL=$(CTRL) LOGS=$(LOGS) UI=$(UI) API=$(API) PLUGINS=$(PLUGINS) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start $(shell if [ -z $GREP_LOGS ]; then echo; else echo "| grep \"$(GREP_LOGS)\""; fi)
+	env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) ARGO_SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) ARGO_LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) ARGO_AUTH_MODE=$(AUTH_MODE) ARGO_NAMESPACED=$(NAMESPACED) ARGO_NAMESPACE=$(KUBE_NAMESPACE) ARGO_MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) ARGO_EXECUTOR_PLUGINS=$(PLUGINS) PROFILE=$(PROFILE) kit $(TASKS)
 endif
 
-$(GOPATH)/bin/stern:
-	go install github.com/stern/stern@latest
-
-.PHONY: logs
-logs: $(GOPATH)/bin/stern
-	$(GOPATH)/bin/stern -l workflows.argoproj.io/workflow 2>&1
-
 .PHONY: wait
 wait:
 	# Wait for workflow controller

diff --git a/Procfile b/Procfile
diff --git a/hack/port-forward.sh b/hack/port-forward.sh
@@ -18,9 +18,6 @@ wait-for() {
   kubectl -n argo wait --timeout 2m --for=condition=Available deploy/$1
 }
 
-wait-for minio
-pf minio 9000
-pf minio 9001
 
 dex=$(kubectl -n argo get pod -l app=dex -o name)
 if [[ "$dex" != "" ]]; then
@@ -63,3 +60,8 @@ if [[ "$azurite" != "" ]]; then
   wait-for azurite
   pf azurite 10000
 fi
+
+# forward MinIO last, so we can just wait for port 9000, and know that all ports are ready
+wait-for minio
+pf minio 9000
+pf minio 9001
diff --git a/tasks.yaml b/tasks.yaml
@@ -18,26 +18,19 @@ spec:
   tasks:
     - name: go-deps
       command: go mod download
-      mutex: deps
-    - name: static-files
-      command: make server/static/files.go STATIC_FILES=false
-      dependencies: go-deps
-    - command: go build -v ./...
-      name: go-build
-      dependencies: static-files go-deps
-    - name: cluster-info
-      command: kubectl cluster-info
-    - name: docker-ps
-      command: docker ps
+      mutex: downloads
     - name: install
       command: sh -c "make install PROFILE=$PROFILE"
       env:
         - PROFILE=minimal
-      dependencies: cluster-info go-deps
+      dependencies: go-deps
       watch: manifests
-    - command: make ./dist/workflow-controller
-      dependencies: go-build
-      name: build-controller
+      mutex: docker
+    - name: build-controller
+      command: make ./dist/workflow-controller
+      watch: cmd/workflow-controller config errors persist pkg util workflow
+      dependencies: go-deps
+      mutex: build
     - name: port-forward
       command: ./hack/port-forward.sh
       ports: 9000
@@ -61,16 +54,16 @@ spec:
         - UPPERIO_DB_DEBUG=1
         - ARCHIVED_WORKFLOW_GC_PERIOD=30s
       ports: "9090"
-      watch: cmd/workflow-controller config errors persist pkg util workflow
     - name: build-argo
       command: make ./dist/argo
-      dependencies: go-build
+      dependencies: go-deps
       env:
         - STATIC_FILES=false
       watch: cmd/argo config errors persist pkg util server workflow
+      mutex: build
     - name: server
       command: ./dist/argo server
-      dependencies: build-argo controller port-forward
+      dependencies: build-argo port-forward
       env:
         - ARGO_X_FRAME_OPTIONS=SAMEORIGIN
         - ARGO_SECURE=false
@@ -82,23 +75,25 @@ spec:
         - UPPERIO_DB_DEBUG=1
       ports: "2746"
     - name: ui-deps
-      command: yarn --cwd ui install
-      mutex: deps
-    - name: ui-build
-      command: yarn --cwd ui build
-      dependencies: ui-deps
+      command: yarn install
+      workingDir: ui
+      mutex: downloads
     - name: ui
-      command: yarn --cwd ui start
+      command: yarn start
+      workingDir: ui
       dependencies: ui-deps server
       ports: "8080"
     - name: executor
       command: make argoexec-image
-      dependencies: docker-ps
       watch: cmd/argoexec config errors pkg util workflow
+      mutex: docker
     - name: example
       command: kubectl create -f examples/hello-world.yaml
-      dependencies: controller
+      dependencies: install
+      mutex: docker
     - name: build
-      dependencies: build-controller build-argo ui-build
+      dependencies: build-controller build-argo
+    - name: pre-up
+      dependencies: build install executor example
     - name: up
-      dependencies: controller server executor ui example
+      dependencies: pre-up controller server ui