fix(resource): catch fatal kubectl errors (argoproj#13321)

Signed-off-by: Anton Gilgur <agilgur5@gmail.com>

Author: agilgur5

test/e2e/resource_template_test.go

@@ -165,6 +165,18 @@ func (s *ResourceTemplateSuite) TestResourceTemplateWithOutputs() {
 		})
 }
 
+func (s *ResourceTemplateSuite) TestResourceTemplateFailed() {
+	s.Given().
+		Workflow("@testdata/resource-templates/failed.yaml").
+		When().
+		SubmitWorkflow().
+		WaitForWorkflow().
+		Then().
+		ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+			assert.Equal(t, wfv1.WorkflowFailed, status.Phase)
+		})
+}
+
 func TestResourceTemplateSuite(t *testing.T) {
 	suite.Run(t, new(ResourceTemplateSuite))
 }

test/e2e/testdata/resource-templates/failed.yaml

@@ -0,0 +1,17 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: failed-
+spec:
+  entrypoint: main
+  templates:
+  - name: main
+    resource:
+      action: patch
+      manifest: |
+        apiVersion: v1
+        # kind: Pod -- missing this causes an error
+        metadata:
+          name: {{pod.name}}
+          annotations:
+            foo: bar

workflow/executor/resource.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/gengo/namer"
 	gengotypes "k8s.io/gengo/types"
 	kubectlcmd "k8s.io/kubectl/pkg/cmd"
+	kubectlutil "k8s.io/kubectl/pkg/cmd/util"
 
 	"github.com/argoproj/argo-workflows/v3/errors"
 	wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
@@ -370,6 +371,13 @@ func runKubectl(args ...string) ([]byte, error) {
 	defer func() {
 		os.Args = osArgs
 	}()
+
+	var fatalErr error
+	// catch `os.Exit(1)` from kubectl
+	kubectlutil.BehaviorOnFatal(func(msg string, code int) {
+		fatalErr = errors.New(fmt.Sprint(code), msg)
+	})
+
 	var buf bytes.Buffer
 	if err := kubectlcmd.NewKubectlCommand(kubectlcmd.KubectlOptions{
 		Arguments: args,
@@ -383,5 +391,8 @@ func runKubectl(args ...string) ([]byte, error) {
 	}).Execute(); err != nil {
 		return nil, err
 	}
+	if fatalErr != nil {
+		return nil, fatalErr
+	}
 	return buf.Bytes(), nil
 }