From 05e13182bb6be97ba6f91a24ea0876cc349d9ab6 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Tue, 14 Mar 2023 11:54:10 -0700 Subject: [PATCH] build: Enhance Codespaces (#10599) Signed-off-by: Alex Collins Signed-off-by: Alex Collins --- .devcontainer/Dockerfile | 10 ------- .devcontainer/devcontainer.json | 21 ++++++++++---- .devcontainer/pre-build.sh | 10 +++++-- .dockerignore | 2 ++ .github/workflows/ci-build.yaml | 15 +++------- Makefile | 50 +++++++++++++++----------------- Procfile | 4 --- hack/port-forward.sh | 8 ++++-- tasks.yaml | 51 +++++++++++++++------------------ 9 files changed, 81 insertions(+), 90 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 Procfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index e68fa2fabc91..000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM mcr.microsoft.com/vscode/devcontainers/universal - -ARG PROTOC_ZIP="protoc-3.14.0-linux-x86_64.zip" - -RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/${PROTOC_ZIP} \ - && unzip -o ${PROTOC_ZIP} -d /usr/local bin/protoc \ - && unzip -o ${PROTOC_ZIP} -d /usr/local 'include/*' \ - && rm -f ${PROTOC_ZIP} \ - && chmod 755 /usr/local/bin/protoc \ - && chmod -R 755 /usr/local/include/ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c185f29e4613..1d37c1f2292e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,18 @@ { - "build": { "dockerfile": "Dockerfile" }, - "hostRequirements": {"cpus": 4}, - "onCreateCommand": ".devcontainer/pre-build.sh", - "workspaceMount": "source=${localWorkspaceFolder},target=${localEnv:HOME}/go/src/github.com/argoproj/argo-workflows,type=bind", - "workspaceFolder": "${localEnv:HOME}/go/src/github.com/argoproj/argo-workflows" + "image": "mcr.microsoft.com/vscode/devcontainers/base:ubuntu", + "features": { + "ghcr.io/devcontainers/features/go:1": { + "version": "1.19" + }, + "ghcr.io/devcontainers/features/node:1": { + "version": "16" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": {} + }, + "hostRequirements": { + "cpus": 4 + }, + "onCreateCommand": ".devcontainer/pre-build.sh", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/vscode/go/src/github.com/argoproj/argo-workflows,type=bind", + "workspaceFolder": "/home/vscode/go/src/github.com/argoproj/argo-workflows" } \ No newline at end of file diff --git a/.devcontainer/pre-build.sh b/.devcontainer/pre-build.sh index e90ad567d7c1..e7385a8fc7d8 100755 --- a/.devcontainer/pre-build.sh +++ b/.devcontainer/pre-build.sh @@ -6,8 +6,14 @@ wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash k3d cluster get k3s-default || k3d cluster create --wait k3d kubeconfig merge --kubeconfig-merge-default +# install kubectl +curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +kubectl cluster-info + # install kit curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | sh -# do time consuming tasks, e.g. download deps and initial build -kit build \ No newline at end of file +# download dependencies and do first-pass compile +CI=1 kit pre-up diff --git a/.dockerignore b/.dockerignore index d46cf7aa6ed3..f027dd582079 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ *.iml *.md *.yaml +.devcontainer # The .git folder gets modified every time git runs preventing caching of builds. Once removed, we must pass GIT_COMMIT as Docker build-args. .git .github @@ -19,6 +20,7 @@ manifests plugins sdks site +tasks.yaml test/e2e ui/dist ui/node_modules diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 743976ce1d7f..b0c0de2e83aa 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -140,25 +140,18 @@ jobs: echo '127.0.0.1 azurite' | sudo tee -a /etc/hosts - run: make install PROFILE=${{matrix.profile}} STATIC_FILES=false name: Install manifests - - run: make controller $(go env GOPATH)/bin/goreman STATIC_FILES=false + - run: make controller kit STATIC_FILES=false name: Build controller - run: make cli STATIC_FILES=false if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} name: Build CLI - - run: ./hack/port-forward.sh - name: Start port forward - - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false LOGS=false > /tmp/argo.log 2>&1 & + - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false > /tmp/argo.log 2>&1 & name: Start controller/API - - run: make wait + - run: make wait API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} timeout-minutes: 4 name: Wait for controller to be up - name: Run tests ${{matrix.test}} - # https://github.com/marketplace/actions/retry-step - uses: nick-fields/retry@v2.8.3 - with: - timeout_minutes: 20 - max_attempts: 2 - command: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false + run: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false - if: ${{ failure() }} name: MinIO/MySQL deployment run: | diff --git a/Makefile b/Makefile index 8b6732291fd6..646bfdb40818 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,6 @@ RELEASE_TAG := $(shell if [[ "$(GIT_TAG)" =~ ^v[0-9]+\.[0-9]+\.[0-9]+. DEV_BRANCH := $(shell [ "$(GIT_BRANCH)" = master ] || [ `echo $(GIT_BRANCH) | cut -c -8` = release- ] || [ `echo $(GIT_BRANCH) | cut -c -4` = dev- ] || [ $(RELEASE_TAG) = true ] && echo false || echo true) SRC := $(GOPATH)/src/github.com/argoproj/argo-workflows -GREP_LOGS := "" - # docker image publishing options IMAGE_NAMESPACE ?= quay.io/argoproj @@ -53,14 +51,17 @@ else STATIC_FILES ?= $(shell [ $(DEV_BRANCH) = true ] && echo false || echo true) endif -# start the Controller -CTRL ?= true -# tail logs -LOGS ?= $(CTRL) # start the UI -UI ?= $(shell [ $(CTRL) = true ] && echo false || echo true) +UI ?= false # start the Argo Server API ?= $(UI) +TASKS := controller +ifeq ($(API),true) +TASKS := controller server +endif +ifeq ($(UI),true) +TASKS := controller server ui +endif GOTEST ?= go test -v -p 20 PROFILE ?= minimal PLUGINS ?= $(shell [ $PROFILE = plugins ] && echo false || echo true) @@ -453,26 +454,29 @@ dist/argosay: mkdir -p dist cp test/e2e/images/argosay/v2/argosay dist/ -$(GOPATH)/bin/goreman: - go install github.com/mattn/goreman@v0.3.11 +.PHONY: kit +kit: +ifeq ($(shell command -v kit),) +ifeq ($(shell uname),Darwin) + brew tap kitproj/kit --custom-remote https://github.com/kitproj/kit + brew install kit +else + curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | sh +endif +endif + .PHONY: start ifeq ($(RUN_MODE),local) ifeq ($(API),true) -start: install controller cli $(GOPATH)/bin/goreman +start: install controller kit cli else -start: install controller $(GOPATH)/bin/goreman +start: install controller kit endif else -start: install +start: install kit endif @echo "starting STATIC_FILES=$(STATIC_FILES) (DEV_BRANCH=$(DEV_BRANCH), GIT_BRANCH=$(GIT_BRANCH)), AUTH_MODE=$(AUTH_MODE), RUN_MODE=$(RUN_MODE), MANAGED_NAMESPACE=$(MANAGED_NAMESPACE)" -ifneq ($(CTRL),true) - @echo "⚠️️ not starting controller. If you want to test the controller, use 'make start CTRL=true' to start it" -endif -ifneq ($(LOGS),true) - @echo "⚠️️ not starting logs. If you want to tail logs, use 'make start LOGS=true' to start it" -endif ifneq ($(API),true) @echo "⚠️️ not starting API. If you want to test the API, use 'make start API=true' to start it" endif @@ -490,18 +494,10 @@ endif grep '127.0.0.1.*minio' /etc/hosts grep '127.0.0.1.*postgres' /etc/hosts grep '127.0.0.1.*mysql' /etc/hosts - ./hack/port-forward.sh ifeq ($(RUN_MODE),local) - env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) CTRL=$(CTRL) LOGS=$(LOGS) UI=$(UI) API=$(API) PLUGINS=$(PLUGINS) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start $(shell if [ -z $GREP_LOGS ]; then echo; else echo "| grep \"$(GREP_LOGS)\""; fi) + env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) ARGO_SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) ARGO_LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) ARGO_AUTH_MODE=$(AUTH_MODE) ARGO_NAMESPACED=$(NAMESPACED) ARGO_NAMESPACE=$(KUBE_NAMESPACE) ARGO_MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) ARGO_EXECUTOR_PLUGINS=$(PLUGINS) PROFILE=$(PROFILE) kit $(TASKS) endif -$(GOPATH)/bin/stern: - go install github.com/stern/stern@latest - -.PHONY: logs -logs: $(GOPATH)/bin/stern - $(GOPATH)/bin/stern -l workflows.argoproj.io/workflow 2>&1 - .PHONY: wait wait: # Wait for workflow controller diff --git a/Procfile b/Procfile deleted file mode 100644 index bdf3714af7bd..000000000000 --- a/Procfile +++ /dev/null @@ -1,4 +0,0 @@ -controller: [ "$CTRL" = "true" ] && ./hack/free-port.sh 9090 && ARGO_EXECUTOR_PLUGINS=${PLUGINS} ARGO_REMOVE_PVC_PROTECTION_FINALIZER=true ARGO_PROGRESS_PATCH_TICK_DURATION=7s DEFAULT_REQUEUE_TIME=${DEFAULT_REQUEUE_TIME} LEADER_ELECTION_IDENTITY=local ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image ${IMAGE_NAMESPACE}/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --managed-namespace=${MANAGED_NAMESPACE} --loglevel ${LOG_LEVEL} -argo-server: [ "$API" = "true" ] &&./hack/free-port.sh 2746 && UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE --x-frame-options=SAMEORIGIN -ui: [ "$UI" = "true" ] && ./hack/free-port.sh 8080 && yarn --cwd ui install && yarn --cwd ui start -logs: [ "$LOGS" = "true" ] && make logs \ No newline at end of file diff --git a/hack/port-forward.sh b/hack/port-forward.sh index 6a8bb086840b..e2e7cce7e1e7 100755 --- a/hack/port-forward.sh +++ b/hack/port-forward.sh @@ -18,9 +18,6 @@ wait-for() { kubectl -n argo wait --timeout 2m --for=condition=Available deploy/$1 } -wait-for minio -pf minio 9000 -pf minio 9001 dex=$(kubectl -n argo get pod -l app=dex -o name) if [[ "$dex" != "" ]]; then @@ -63,3 +60,8 @@ if [[ "$azurite" != "" ]]; then wait-for azurite pf azurite 10000 fi + +# forward MinIO last, so we can just wait for port 9000, and know that all ports are ready +wait-for minio +pf minio 9000 +pf minio 9001 \ No newline at end of file diff --git a/tasks.yaml b/tasks.yaml index c2ae3f74011c..9413aaec296e 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -18,26 +18,19 @@ spec: tasks: - name: go-deps command: go mod download - mutex: deps - - name: static-files - command: make server/static/files.go STATIC_FILES=false - dependencies: go-deps - - command: go build -v ./... - name: go-build - dependencies: static-files go-deps - - name: cluster-info - command: kubectl cluster-info - - name: docker-ps - command: docker ps + mutex: downloads - name: install command: sh -c "make install PROFILE=$PROFILE" env: - PROFILE=minimal - dependencies: cluster-info go-deps + dependencies: go-deps watch: manifests - - command: make ./dist/workflow-controller - dependencies: go-build - name: build-controller + mutex: docker + - name: build-controller + command: make ./dist/workflow-controller + watch: cmd/workflow-controller config errors persist pkg util workflow + dependencies: go-deps + mutex: build - name: port-forward command: ./hack/port-forward.sh ports: 9000 @@ -61,16 +54,16 @@ spec: - UPPERIO_DB_DEBUG=1 - ARCHIVED_WORKFLOW_GC_PERIOD=30s ports: "9090" - watch: cmd/workflow-controller config errors persist pkg util workflow - name: build-argo command: make ./dist/argo - dependencies: go-build + dependencies: go-deps env: - STATIC_FILES=false watch: cmd/argo config errors persist pkg util server workflow + mutex: build - name: server command: ./dist/argo server - dependencies: build-argo controller port-forward + dependencies: build-argo port-forward env: - ARGO_X_FRAME_OPTIONS=SAMEORIGIN - ARGO_SECURE=false @@ -82,23 +75,25 @@ spec: - UPPERIO_DB_DEBUG=1 ports: "2746" - name: ui-deps - command: yarn --cwd ui install - mutex: deps - - name: ui-build - command: yarn --cwd ui build - dependencies: ui-deps + command: yarn install + workingDir: ui + mutex: downloads - name: ui - command: yarn --cwd ui start + command: yarn start + workingDir: ui dependencies: ui-deps server ports: "8080" - name: executor command: make argoexec-image - dependencies: docker-ps watch: cmd/argoexec config errors pkg util workflow + mutex: docker - name: example command: kubectl create -f examples/hello-world.yaml - dependencies: controller + dependencies: install + mutex: docker - name: build - dependencies: build-controller build-argo ui-build + dependencies: build-controller build-argo + - name: pre-up + dependencies: build install executor example - name: up - dependencies: controller server executor ui example + dependencies: pre-up controller server ui