feat: allow specifying args to be passed to kustomize build (#7414)

* feat: allow specifying args to be passed to kustomize build

* chore: fix references

* fix: make upgrade work properly

* feat: allow specifying args to be passed to kustomize build

* chore: fix references

* fix: make upgrade work properly

* fix: integration test examples

Co-authored-by: tejal29 <tejal29@gmail.com>

integration/testdata/debug/skaffold.yaml

@@ -29,8 +29,11 @@ manifests:
 profiles:
 - name: kustomize
   patches:
+  - op: remove
+    path: /manifests/rawYaml
   manifests:
     kustomize:
+      paths:
       - "."
 # use GCP Buildpacks to build the individual projects
 - name: buildpacks
@@ -62,6 +65,8 @@ profiles:
     #    builder: "gcr.io/buildpacks/builder:v1"
 - name: docker
   patches:
+  - op: remove
+    path: /manifests/rawYaml
   build:
     artifacts:
     - image: skaffold-debug-nodejs

pkg/skaffold/render/generate/generate.go

@@ -91,34 +91,18 @@ func (g Generator) Generate(ctx context.Context, out io.Writer) (manifest.Manife
 	var manifests manifest.ManifestList
 
 	// Generate kustomize Manifests
-	_, endTrace := instrumentation.StartTrace(ctx, "Render_expandGlobKustomizeManifests")
-	kustomizePaths, err := resolveRemoteAndLocal(g.config.Kustomize, g.workingDir)
-	if err != nil {
-		event.DeployInfoEvent(fmt.Errorf("could not expand the glob kustomize manifests: %w", err))
-		return nil, err
-	}
-	endTrace()
-	kustomizePathMap := make(map[string]bool)
-	for _, path := range kustomizePaths {
-		if dir, ok := isKustomizeDir(path); ok {
-			kustomizePathMap[dir] = true
-		}
-	}
-	for kPath := range kustomizePathMap {
-		// TODO: kustomize kpt-fn not available yet. See https://github.com/GoogleContainerTools/kpt/issues/1447
-		cmd := exec.CommandContext(ctx, "kustomize", "build", kPath)
-		out, err := util.RunCmdOut(ctx, cmd)
+	if g.config.Kustomize != nil && len(g.config.Kustomize.Paths) != 0 {
+		kustomizeManifests, err := g.generateKustomizeManifests(ctx)
 		if err != nil {
 			return nil, err
 		}
-		if len(out) == 0 {
-			continue
+		for _, m := range kustomizeManifests {
+			manifests.Append(m)
 		}
-		manifests.Append(out)
 	}
 
 	// Generate in-place hydrated kpt Manifests
-	_, endTrace = instrumentation.StartTrace(ctx, "Render_expandGlobKptManifests")
+	_, endTrace := instrumentation.StartTrace(ctx, "Render_expandGlobKptManifests")
 	kptPaths, err := resolveRemoteAndLocal(g.config.Kpt, g.workingDir)
 	if err != nil {
 		event.DeployInfoEvent(fmt.Errorf("could not expand the glob kpt manifests: %w", err))
@@ -174,6 +158,38 @@ func (g Generator) Generate(ctx context.Context, out io.Writer) (manifest.Manife
 	return manifests, nil
 }
 
+func (g Generator) generateKustomizeManifests(ctx context.Context) ([][]byte, error) {
+	var manifests [][]byte
+
+	_, endTrace := instrumentation.StartTrace(ctx, "Render_expandGlobKustomizeManifests")
+	kustomizePaths, err := resolveRemoteAndLocal(g.config.Kustomize.Paths, g.workingDir)
+	if err != nil {
+		event.DeployInfoEvent(fmt.Errorf("could not expand the glob kustomize manifests: %w", err))
+		return nil, err
+	}
+	endTrace()
+	kustomizePathMap := make(map[string]bool)
+	for _, path := range kustomizePaths {
+		if dir, ok := isKustomizeDir(path); ok {
+			kustomizePathMap[dir] = true
+		}
+	}
+	for kPath := range kustomizePathMap {
+		// TODO: kustomize kpt-fn not available yet. See https://github.com/GoogleContainerTools/kpt/issues/1447
+		cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, kustomizeBuildArgs(g.config.Kustomize.BuildArgs, kPath)...)...)
+		out, err := util.RunCmdOut(ctx, cmd)
+		if err != nil {
+			return nil, err
+		}
+		if len(out) == 0 {
+			continue
+		}
+		manifests = append(manifests, out)
+	}
+
+	return manifests, nil
+}
+
 // isKustomizeDir checks if the path is managed by kustomize. A more reliable approach is parsing the kustomize content
 // resources, bases, overlays. However, this switches the manifests parsing from kustomize/kpt to skaffold. To avoid
 // skaffold render.generate mis-use, we expect the users do not place non-kustomize manifests under the kustomization.yaml directory, so as the kpt manifests.
@@ -200,6 +216,24 @@ func isKustomizeDir(path string) (string, bool) {
 	return "", false
 }
 
+// kustomizeBuildArgs returns a list of build args to be passed to kustomize build.
+func kustomizeBuildArgs(buildArgs []string, kustomizePath string) []string {
+	var args []string
+
+	if len(buildArgs) > 0 {
+		for _, v := range buildArgs {
+			parts := strings.Split(v, " ")
+			args = append(args, parts...)
+		}
+	}
+
+	if len(kustomizePath) > 0 {
+		args = append(args, kustomizePath)
+	}
+
+	return args
+}
+
 func isKptDir(path string) (string, bool) {
 	fileInfo, err := os.Stat(path)
 	if err != nil {
@@ -220,16 +254,19 @@ func isKptDir(path string) (string, bool) {
 
 // walkManifests finds out all the manifests from the `.manifests.generate`, so they can be registered in the file watcher.
 // Note: the logic about manifest dependencies shall separate from the "Generate" function, which requires "context" and
-// only be called when a renderig action is needed (normally happens after the file watcher registration).
+// only be called when a rendering action is needed (normally happens after the file watcher registration).
 func (g Generator) walkManifests() ([]string, error) {
 	var dependencyPaths []string
+
 	// Generate kustomize Manifests
-	kustomizePaths, err := resolveRemoteAndLocal(g.config.Kustomize, g.workingDir)
-	if err != nil {
-		event.DeployInfoEvent(fmt.Errorf("could not expand the glob kustomize manifests: %w", err))
-		return nil, err
+	if g.config.Kustomize != nil {
+		kustomizePaths, err := resolveRemoteAndLocal(g.config.Kustomize.Paths, g.workingDir)
+		if err != nil {
+			event.DeployInfoEvent(fmt.Errorf("could not expand the glob kustomize manifests: %w", err))
+			return nil, err
+		}
+		dependencyPaths = append(dependencyPaths, kustomizePaths...)
 	}
-	dependencyPaths = append(dependencyPaths, kustomizePaths...)
 
 	// Generate in-place hydrated kpt Manifests
 	kptPaths, err := resolveRemoteAndLocal(g.config.Kpt, g.workingDir)

pkg/skaffold/render/generate/generate_test.go

@@ -201,7 +201,9 @@ func TestManifestDeps(t *testing.T) {
 		{
 			description: "kustomize dir",
 			generateConfig: latest.Generate{
-				Kustomize: []string{"kustomize-sample"},
+				Kustomize: &latest.Kustomize{
+					Paths: []string{"kustomize-sample"},
+				},
 			},
 			expected: []string{"kustomize-sample/kustomization.yaml", "kustomize-sample/patch.yaml"},
 		},
@@ -215,9 +217,11 @@ func TestManifestDeps(t *testing.T) {
 		{
 			description: "multi manifest, mixed dir and file",
 			generateConfig: latest.Generate{
-				RawK8s:    []string{"rawYaml-sample"},
-				Kustomize: []string{"kustomize-sample"},
-				Kpt:       []string{"kpt-sample"},
+				RawK8s: []string{"rawYaml-sample"},
+				Kustomize: &latest.Kustomize{
+					Paths: []string{"kustomize-sample"},
+				},
+				Kpt: []string{"kpt-sample"},
 			},
 			expected: []string{
 				"kpt-sample/Kptfile",

pkg/skaffold/schema/defaults/defaults.go

@@ -121,7 +121,7 @@ func SetDefaultRenderer(c *latest.SkaffoldConfig) {
 	if len(c.Render.Generate.RawK8s) > 0 {
 		return
 	}
-	if len(c.Render.Generate.Kustomize) > 0 {
+	if c.Render.Generate.Kustomize != nil {
 		return
 	}
 	if c.Render.Generate.Helm != nil {

pkg/skaffold/schema/defaults/defaults_test.go

@@ -450,12 +450,12 @@ func TestSetDefaultRenderer(t *testing.T) {
 			description: "kustomize manifests",
 			input: latest.RenderConfig{
 				Generate: latest.Generate{
-					Kustomize: []string{"/kmanifests"},
+					Kustomize: &latest.Kustomize{Paths: []string{"/kmanifests"}},
 				},
 			},
 			expected: latest.RenderConfig{
 				Generate: latest.Generate{
-					Kustomize: []string{"/kmanifests"},
+					Kustomize: &latest.Kustomize{Paths: []string{"/kmanifests"}},
 				},
 			},
 		},
@@ -490,14 +490,14 @@ func TestSetDefaultRenderer(t *testing.T) {
 			input: latest.RenderConfig{
 				Generate: latest.Generate{
 					Kpt:       []string{"/kmanifests1"},
-					Kustomize: []string{"/kmanifests2"},
+					Kustomize: &latest.Kustomize{Paths: []string{"/kmanifests2"}},
 					Helm:      &latest.Helm{Releases: []latest.HelmRelease{{Name: "test"}}},
 				},
 			},
 			expected: latest.RenderConfig{
 				Generate: latest.Generate{
 					Kpt:       []string{"/kmanifests1"},
-					Kustomize: []string{"/kmanifests2"},
+					Kustomize: &latest.Kustomize{Paths: []string{"/kmanifests2"}},
 					Helm:      &latest.Helm{Releases: []latest.HelmRelease{{Name: "test"}}},
 				},
 			},

pkg/skaffold/schema/latest/config.go

@@ -569,8 +569,9 @@ type Generate struct {
 	// RawK8s TODO: add description.
 	RawK8s []string `yaml:"rawYaml,omitempty" skaffold:"filepath"`
 
-	// Kustomize TODO: add description.
-	Kustomize []string `yaml:"kustomize,omitempty" skaffold:"filepath"`
+	// Kustomize defines the paths to be modified with kustomize, along with extra
+	// flags to be passed to kustomize
+	Kustomize *Kustomize `yaml:"kustomize,omitempty"`
 
 	// Helm TODO: add description.
 	Helm *Helm `yaml:"helm,omitempty"`
@@ -579,6 +580,17 @@ type Generate struct {
 	Kpt []string `yaml:"kpt,omitempty" skaffold:"filepath"`
 }
 
+// Kustomize defines the paths to be modified with kustomize, along with
+// extra flags to be passed to kustomize
+type Kustomize struct {
+	// Paths is the path to Kustomization files.
+	// Defaults to `["."]`.
+	Paths []string `yaml:"paths,omitempty" skaffold:"filepath"`
+
+	// BuildArgs are additional args passed to `kustomize build`.
+	BuildArgs []string `yaml:"buildArgs,omitempty"`
+}
+
 // Helm defines the manifests from helm releases.
 type Helm struct {
 	// Flags are additional option flags that are passed on the command

pkg/skaffold/schema/v2beta28/upgrade.go

@@ -48,10 +48,15 @@ func upgradeOnePipeline(oldPipeline, newPipeline interface{}) error {
 		newPL.Deploy.KubectlDeploy.Manifests = nil
 	}
 
-	// TODO(marlongamez): port over buildArgs config and copy that
 	// Copy kustomize deploy config to render config
 	if oldPL.Deploy.KustomizeDeploy != nil {
-		newPL.Render.Kustomize = oldPL.Deploy.KustomizeDeploy.KustomizePaths
+		newPL.Render.Kustomize = &next.Kustomize{
+			Paths:     oldPL.Deploy.KustomizeDeploy.KustomizePaths,
+			BuildArgs: oldPL.Deploy.KustomizeDeploy.BuildArgs,
+		}
+		if len(newPL.Render.Kustomize.Paths) == 0 {
+			newPL.Render.Kustomize.Paths = append(newPL.Render.Kustomize.Paths, ".")
+		}
 		// nil out kustomize deployer as it shouldn't be a thing anymore
 		newPL.Deploy.KustomizeDeploy = nil
 

pkg/skaffold/schema/v2beta28/upgrade_test.go

@@ -149,6 +149,7 @@ manifests:
   rawYaml:
     - k8s-*
   kustomize:
+    paths:
     - kustomization-main
   helm:
     releases:
@@ -181,7 +182,8 @@ profiles:
       rawYaml:
       - k8s-*
       kustomize:
-      - kustomization-test
+        paths:
+        - kustomization-test
     deploy:
       kubectl: {}
   - name: test local
@@ -195,6 +197,9 @@ profiles:
     manifests:
       rawYaml:
       - k8s-*
+      kustomize:
+        paths:
+        - "."
     deploy:
       kubectl: {}
 `