Merge branch 'master' into deadline

.github/workflows/ci-build.yaml

@@ -112,13 +112,8 @@ jobs:
       - run: make install controller cli $(go env GOPATH)/bin/goreman PROFILE=${{matrix.profile}} E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info
       - run: make start PROFILE=${{matrix.profile}} E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info UI=false > /tmp/log/argo-e2e/argo.log 2>&1 &
         timeout-minutes: 4
-      - name: make/pull argoexec-image
-        run: |
-          if [ ${{matrix.test}} == test-executor ]; then
-            make argoexec-image STATIC_FILES=false
-          else
-            docker pull argoproj/argoexec:latest
-          fi
+      - name: make argoexec-image
+        run: make argoexec-image STATIC_FILES=false
       - run: make wait
         timeout-minutes: 4
       - run: make ${{matrix.test}} E2E_TIMEOUT=1m STATIC_FILES=false

Makefile

@@ -222,7 +222,7 @@ argoexec-image:
 
 %-image:
 	[ ! -e dist/$* ] || mv dist/$* .
-	docker buildx build -t $(IMAGE_NAMESPACE)/$*:$(VERSION) --target $* .
+	docker buildx build -t $(IMAGE_NAMESPACE)/$*:$(VERSION) --target $* --output=type=docker .
 	[ ! -e $* ] || mv $* dist/
 	docker run --rm -t $(IMAGE_NAMESPACE)/$*:$(VERSION) version
 	if [ $(K3D) = true ]; then k3d image import $(IMAGE_NAMESPACE)/$*:$(VERSION); fi
@@ -409,9 +409,6 @@ install: dist/kustomize
 	dist/kustomize build --load_restrictor=none test/e2e/manifests/$(PROFILE) | sed 's/argoproj\//$(IMAGE_NAMESPACE)\//' | sed 's/containerRuntimeExecutor: docker/containerRuntimeExecutor: $(E2E_EXECUTOR)/' | kubectl -n $(KUBE_NAMESPACE) apply --prune -l app.kubernetes.io/part-of=argo -f -
 ifeq ($(PROFILE),stress)
 	kubectl -n $(KUBE_NAMESPACE) apply -f test/stress/massive-workflow.yaml
-	kubectl -n $(KUBE_NAMESPACE) rollout restart deploy workflow-controller
-	kubectl -n $(KUBE_NAMESPACE) rollout restart deploy argo-server
-	kubectl -n $(KUBE_NAMESPACE) rollout restart deploy minio
 endif
 ifeq ($(RUN_MODE),kubernetes)
 	# scale to 2 replicas so we touch upon leader election
@@ -461,7 +458,7 @@ $(GOPATH)/bin/goreman:
 ifeq ($(RUN_MODE),local)
 start: install controller cli $(GOPATH)/bin/goreman
 else
-start: install argoexec-image workflow-controller-image argocli-image
+start: install
 endif
 	@echo "starting STATIC_FILES=$(STATIC_FILES) (DEV_BRANCH=$(DEV_BRANCH), GIT_BRANCH=$(GIT_BRANCH)), AUTH_MODE=$(AUTH_MODE), RUN_MODE=$(RUN_MODE)"
 	# Check dex, minio, postgres and mysql are in hosts file

USERS.md

@@ -59,7 +59,7 @@ Currently, the following organizations are **officially** using Argo Workflows:
 1. [Gladly](https://gladly.com/)
 1. [Glovo](https://www.glovoapp.com)
 1. [Google](https://www.google.com/intl/en/about/our-company/)
-1. [Graviti](https://www.graviti.cn)
+1. [Graviti](https://www.graviti.com)
 1. [Greenhouse](https://greenhouse.io)
 1. [Habx](https://www.habx.com/)
 1. [Helio](https://helio.exchange)
@@ -74,6 +74,7 @@ Currently, the following organizations are **officially** using Argo Workflows:
 1. [İşbank](https://www.isbank.com.tr/en)
 1. [Jungle](https://www.jungle.ai/)
 1. [Karius](https://www.kariusdx.com/)
+1. [KarrotPay](https://www.daangnpay.com/)
 1. [Kasa](https://www.kasa.co.kr/)
 1. [KintoHub](https://www.kintohub.com/)
 1. [Localytics](https://www.localytics.com/)

cmd/argo/lint/lint.go

@@ -130,11 +130,15 @@ func Lint(ctx context.Context, opts *LintOptions) (*LintResults, error) {
 			case err != nil:
 				return err
 			case strings.HasPrefix(path, "/dev/") || lintExt[filepath.Ext(path)]:
-				f, err := os.Open(path)
+				f, err := os.Open(filepath.Clean(path))
 				if err != nil {
 					return err
 				}
-				defer f.Close()
+				defer func() {
+					if err := f.Close(); err != nil {
+						log.Fatalf("Error closing file[%s]: %v", path, err)
+					}
+				}()
 				r = f
 			case info.IsDir():
 				return nil // skip

cmd/argoexec/commands/emissary.go

@@ -78,7 +78,7 @@ func NewEmissaryCommand() *cobra.Command {
 					for _, y := range x.Dependencies {
 						logger.Infof("waiting for dependency %q", y)
 						for {
-							data, err := ioutil.ReadFile(varRunArgo + "/ctr/" + y + "/exitcode")
+							data, err := ioutil.ReadFile(filepath.Clean(varRunArgo + "/ctr/" + y + "/exitcode"))
 							if os.IsNotExist(err) {
 								time.Sleep(time.Second)
 								continue
@@ -109,7 +109,7 @@ func NewEmissaryCommand() *cobra.Command {
 			command.Stderr = os.Stderr
 
 			// this may not be that important an optimisation, except for very long logs we don't want to capture
-			if includeScriptOutput {
+			if includeScriptOutput || template.SaveLogsAsArtifact() {
 				logger.Info("capturing logs")
 				stdout, err := os.Create(varRunArgo + "/ctr/" + containerName + "/stdout")
 				if err != nil {
@@ -132,7 +132,7 @@ func NewEmissaryCommand() *cobra.Command {
 
 			go func() {
 				for {
-					data, _ := ioutil.ReadFile(varRunArgo + "/ctr/" + containerName + "/signal")
+					data, _ := ioutil.ReadFile(filepath.Clean(varRunArgo + "/ctr/" + containerName + "/signal"))
 					_ = os.Remove(varRunArgo + "/ctr/" + containerName + "/signal")
 					s, _ := strconv.Atoi(string(data))
 					if s > 0 {
@@ -212,7 +212,7 @@ func saveParameter(srcPath string) error {
 		logger.Infof("no need to save parameter - on overlapping volume: %s", srcPath)
 		return nil
 	}
-	src, err := os.Open(srcPath)
+	src, err := os.Open(filepath.Clean(srcPath))
 	if os.IsNotExist(err) { // might be optional, so we ignore
 		logger.WithError(err).Errorf("cannot save parameter %s", srcPath)
 		return nil

cmd/argoexec/commands/root.go

@@ -4,6 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"time"
+
+	wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 
 	"github.com/argoproj/pkg/cli"
 	kubecli "github.com/argoproj/pkg/kube/cli"
@@ -32,11 +35,10 @@ const (
 )
 
 var (
-	clientConfig       clientcmd.ClientConfig
-	logLevel           string // --loglevel
-	glogLevel          int    // --gloglevel
-	logFormat          string // --log-format
-	podAnnotationsPath string // --pod-annotations
+	clientConfig clientcmd.ClientConfig
+	logLevel     string // --loglevel
+	glogLevel    int    // --gloglevel
+	logFormat    string // --log-format
 )
 
 func init() {
@@ -66,7 +68,6 @@ func NewRootCommand() *cobra.Command {
 	command.AddCommand(cmd.NewVersionCmd(CLIName))
 
 	clientConfig = kubecli.AddKubectlFlagsToCmd(&command)
-	command.PersistentFlags().StringVar(&podAnnotationsPath, "pod-annotations", common.PodMetadataAnnotationsPath, "Pod annotations file from k8s downward API")
 	command.PersistentFlags().StringVar(&logLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error")
 	command.PersistentFlags().IntVar(&glogLevel, "gloglevel", 0, "Set the glog logging level")
 	command.PersistentFlags().StringVar(&logFormat, "log-format", "text", "The formatter to use for logs. One of: text|json")
@@ -97,10 +98,12 @@ func initExecutor() *executor.WorkflowExecutor {
 		log.Fatalf("Unable to determine pod name from environment variable %s", common.EnvVarPodName)
 	}
 
-	tmpl, err := executor.LoadTemplate(podAnnotationsPath)
-	checkErr(err)
+	tmpl := &wfv1.Template{}
+	checkErr(json.Unmarshal([]byte(os.Getenv(common.EnvVarTemplate)), tmpl))
 
 	includeScriptOutput := os.Getenv(common.EnvVarIncludeScriptOutput) == "true"
+	deadline, err := time.Parse(time.RFC3339, os.Getenv(common.EnvVarDeadline))
+	checkErr(err)
 
 	var cre executor.ContainerRuntimeExecutor
 	switch executorType {
@@ -117,14 +120,14 @@ func initExecutor() *executor.WorkflowExecutor {
 	}
 	checkErr(err)
 
-	wfExecutor := executor.NewExecutor(clientset, restClient, podName, namespace, podAnnotationsPath, cre, *tmpl, includeScriptOutput)
-	yamlBytes, _ := json.Marshal(&wfExecutor.Template)
+	wfExecutor := executor.NewExecutor(clientset, restClient, podName, namespace, cre, *tmpl, includeScriptOutput, deadline)
 	log.
 		WithField("version", version.String()).
 		WithField("namespace", namespace).
 		WithField("podName", podName).
-		WithField("template", string(yamlBytes)).
+		WithField("template", wfv1.MustMarshallJSON(&wfExecutor.Template)).
 		WithField("includeScriptOutput", includeScriptOutput).
+		WithField("deadline", deadline).
 		Info("Executor initialized")
 	return &wfExecutor
 }

docs/fields.md

@@ -691,6 +691,8 @@ WorkflowSpec is the specification of a Workflow.
 
 - [`work-avoidance.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml)
 
+- [`workflow-count-resourcequota.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-count-resourcequota.yaml)
+
 - [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
 
 - [`event-consumer-workflowtemplate.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workflowtemplate.yaml)
@@ -1093,6 +1095,8 @@ CronWorkflowSpec is the specification of a CronWorkflow
 
 - [`work-avoidance.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml)
 
+- [`workflow-count-resourcequota.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-count-resourcequota.yaml)
+
 - [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
 
 - [`event-consumer-workflowtemplate.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workflowtemplate.yaml)
@@ -1452,6 +1456,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate.
 
 - [`work-avoidance.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml)
 
+- [`workflow-count-resourcequota.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-count-resourcequota.yaml)
+
 - [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
 
 - [`event-consumer-workflowtemplate.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workflowtemplate.yaml)
@@ -4635,6 +4641,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al
 
 - [`work-avoidance.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml)
 
+- [`workflow-count-resourcequota.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-count-resourcequota.yaml)
+
 - [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
 
 - [`event-consumer-workflowtemplate.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workflowtemplate.yaml)
@@ -5886,6 +5894,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and
 
 - [`work-avoidance.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml)
 
+- [`workflow-count-resourcequota.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-count-resourcequota.yaml)
+
 - [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
 
 - [`event-consumer-workflowtemplate.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workflowtemplate.yaml)

docs/roadmap.md

@@ -33,10 +33,6 @@ Improve the handling of artifacts loaded from AWS, GCS, Artifactory et al.  For
 
 Make it easier to understand how long and how much resource workflows use by supporting [automatic duration prediction](https://github.com/argoproj/argo-workflows/issues/2717) for newly started workflows and [historical workflow reports](https://github.com/argoproj/argo-workflows/issues/3557).
 
-## RBAC for SSO
-
-The #1 community voted feature is [RBAC for SSO](https://github.com/argoproj/argo-workflows/issues/3525). This will allow users to provide different access to different users within the GUI.
-
 ## Scaling, Reliability, Performance
 
 Continue to improve support for large numbers (10,000+) of massive (2,000+ node) workflows.

docs/stress-testing.md

@@ -1,32 +1,44 @@
 # Stress Testing
 
-Create a cluster in [`jesse-sb` project](https://console.cloud.google.com/access/iam?cloudshell=false&project=jesse-sb).
+Create a cluster in [`jesse-sb` project](https://console.cloud.google.com/access/iam?cloudshell=false&project=jesse-sb)
+with at least 21 nodes.
 
 Install `gcloud` binary.
 
 Login to GCP: `gloud auth login`
 
-Connect to your new cluster.
+Get your KUBECONFIG, something like:
 
-Make sure you've logged in to Docker Hub: `docker login`
+```
+gcloud container clusters get-credentials cluster-1 --zone us-central1-c --project jesse-sb
+```
 
-Run `make start PROFILE=stress IMAGE_NAMESPACE=alexcollinsintuit DOCKER_PUSH=true`.
+Run `make start PROFILE=stress`.
+
+Make sure pods are running:
+
+```
+kubectl get deployments
+```
 
 If this fails, just try running it again.
 
-Open https://localhost:2746 and check you can run a workflow.
+* Open https://localhost:2746/workflows/argo and check it loads and that you can run a workflow.
+* Open http://localhost:9090/metrics and check you can see the Prometheus metrics.
+* Open http://localhost:9091/graph and check you can see a Prometheus graph.
+* Open http://localhost:6060 and check you can access pprof.
 
 Open `test/stress/main.go` and run it with a small number (e.g. 10) workflows and make sure they complete.
 
-Do you get `ImagePullBackOff`? Make sure image is `argoproj/argosay:v2` in  `kubectl -n argo edit workflowtemplate massive-workflow`.
+Do you get `ImagePullBackOff`? Make sure image is `argoproj/argosay:v2`
+in  `kubectl -n argo edit workflowtemplate massive-workflow`.
 
-Open http://localhost:9091/graph.
-
-You can use [this Tab Auto Refresh Chrome extension](https://chrome.google.com/webstore/detail/tab-auto-refresh/oomoeacogjkolheacgdkkkhbjipaomkn) to auto-refresh the page.
+You can
+use [this Tab Auto Refresh Chrome extension](https://chrome.google.com/webstore/detail/tab-auto-refresh/oomoeacogjkolheacgdkkkhbjipaomkn)
+to auto-refresh the page.
 
 Open `test/stress/main.go` and run it with a large number (e.g. 10000).
 
-Use Prometheus to analyse this.
-
-Finally, you can capture PProf using `./hack/capture-pprof.sh`.
+* Use Prometheus to analyse this.
+* Use PProf using `./hack/capture-pprof.sh` and look at allocs and profile for costly areas.
 

docs/workflow-controller-configmap.yaml

@@ -293,9 +293,9 @@ data:
     clientSecret:
       name: client-secret-secret
       key: client-secret-key
-    # This is the redirect URL supplied to the provider (required). It must
+    # This is the redirect URL supplied to the provider (optional). It must
     # be in the form <argo-server-root-url>/oauth2/callback. It must be
-    # browser-accessible.
+    # browser-accessible. If omitted, will be automatically generated.
     redirectUrl: https://argo-server/oauth2/callback
     # Additional scopes to request. Typically needed for SSO RBAC. >= v2.12
     scopes:

docs/workflow-executors.md

@@ -38,6 +38,8 @@ The executor to be used in your workflows can be changed in [the configmap](./wo
     * Operations performed against the local Kubelet
 * Artifacts:
     * Output artifacts must be saved on volumes (e.g. [emptyDir](empty-dir.md)) and not the base image layer (e.g. `/tmp`)
+* Step/Task result:
+    * Warnings that normally goes to stderr will get captured in a step or a dag task's `outputs.result`. May require changes if your pipeline is conditioned on `steps/tasks.name.outputs.result`
 * Configuration:
     * Additional Kubelet configuration maybe needed
 
@@ -55,6 +57,8 @@ The executor to be used in your workflows can be changed in [the configmap](./wo
     * Log retrieval and container operations performed against the remote Kubernetes API
 * Artifacts:
     * Output artifacts must be saved on volumes (e.g. [emptyDir](empty-dir.md)) and not the base image layer (e.g. `/tmp`)
+* Step/Task result:
+    * Warnings that normally goes to stderr will get captured in a step or a dag task's `outputs.result`. May require changes if your pipeline is conditioned on `steps/tasks.name.outputs.result`
 * Configuration:
     * No additional configuration needed.
 

examples/README.md

@@ -1436,6 +1436,8 @@ Currently only a single resource can be managed by a resource template so either
 
 Resources created in this way are independent of the workflow. If you want the resource to be deleted when the workflow is deleted then you can use [Kubernetes garbage collection](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/) with the workflow resource as an owner reference ([example](./k8s-owner-reference.yaml)).
 
+You can also collect data about the resource in output parameters (see more at [k8s-jobs.yaml](./k8s-jobs.yaml))
+
 **Note:**
 When patching, the resource will accept another attribute, `mergeStrategy`, which can either be `strategic`, `merge`, or `json`. If this attribute is not supplied, it will default to `strategic`. Keep in mind that Custom Resources cannot be patched with `strategic`, so a different strategy must be chosen. For example, suppose you have the [CronTab CustomResourceDefinition](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#create-a-customresourcedefinition) defined, and the following instance of a CronTab:
 

examples/buildkit-template.yaml

@@ -68,8 +68,7 @@ spec:
               parameters:
                 - name: path
                   value: "{{workflow.parameters.path}}"
-            dependencies:
-              - clone
+            depends: "clone"
           - name: image
             template: image
             arguments:
@@ -78,8 +77,7 @@ spec:
                   value: "{{workflow.parameters.path}}"
                 - name: image
                   value: "{{workflow.parameters.image}}"
-            dependencies:
-              - build
+            depends: "build"
     - name: clone
       inputs:
         parameters:

examples/cluster-workflow-template/cluster-wftmpl-dag.yaml

@@ -25,7 +25,7 @@ spec:
           - name: message
             value: A
       - name: B
-        dependencies: [A]
+        depends: "A"
         templateRef:
           name: cluster-workflow-template-whalesay-template
           template: whalesay-template
@@ -35,13 +35,13 @@ spec:
           - name: message
             value: B
       - name: C
-        dependencies: [A]
+        depends: "A"
         templateRef:
           name: cluster-workflow-template-inner-dag
           template: inner-diamond
           clusterScope: true
       - name: D
-        dependencies: [B, C]
+        depends: "B && C"
         templateRef:
           name: cluster-workflow-template-whalesay-template
           template: whalesay-template