diff --git a/.gitignore b/.gitignore index cbced62d4c..68269f3568 100644 --- a/.gitignore +++ b/.gitignore @@ -1,33 +1,33 @@ -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -.idea -*.iml - -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# We use sed -i.bak when doing in-line replace, because it works better cross-platform -.bak - -# macOS -*.DS_store - -.bin - -# Hugo site -publishedSite/ -site/public/ -site/resources/ -site/.hugo_build.lock -**/node_modules/ - -# goreleaser artifacts -**/dist/ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +.idea +*.iml + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# We use sed -i.bak when doing in-line replace, because it works better cross-platform +.bak + +# macOS +*.DS_store + +.bin + +# Hugo site +publishedSite/ +site/public/ +site/resources/ +site/.hugo_build.lock +**/node_modules/ + +# goreleaser artifacts +**/dist/ diff --git a/api/internal/loader/fileloader.go b/api/internal/loader/fileloader.go index ce1100d104..3cc51bbc9d 100644 --- a/api/internal/loader/fileloader.go +++ b/api/internal/loader/fileloader.go @@ -1,345 +1,345 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package loader - -import ( - "fmt" - "io" - "log" - "net/http" - "net/url" - "path/filepath" - "strings" - - "sigs.k8s.io/kustomize/api/ifc" - "sigs.k8s.io/kustomize/api/internal/git" - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/filesys" -) - -// IsRemoteFile returns whether path has a url scheme that kustomize allows for -// remote files. See https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md -func IsRemoteFile(path string) bool { - u, err := url.Parse(path) - return err == nil && (u.Scheme == "http" || u.Scheme == "https") -} - -// FileLoader is a kustomization's interface to files. -// -// The directory in which a kustomization file sits -// is referred to below as the kustomization's _root_. -// -// An instance of fileLoader has an immutable root, -// and offers a `New` method returning a new loader -// with a new root. -// -// A kustomization file refers to two kinds of files: -// -// * supplemental data paths -// -// `Load` is used to visit these paths. -// -// These paths refer to resources, patches, -// data for ConfigMaps and Secrets, etc. -// -// The loadRestrictor may disallow certain paths -// or classes of paths. -// -// * bases (other kustomizations) -// -// `New` is used to load bases. -// -// A base can be either a remote git repo URL, or -// a directory specified relative to the current -// root. In the former case, the repo is locally -// cloned, and the new loader is rooted on a path -// in that clone. -// -// As loaders create new loaders, a root history -// is established, and used to disallow: -// -// - A base that is a repository that, in turn, -// specifies a base repository seen previously -// in the loading stack (a cycle). -// -// - An overlay depending on a base positioned at -// or above it. I.e. '../foo' is OK, but '.', -// '..', '../..', etc. are disallowed. Allowing -// such a base has no advantages and encourages -// cycles, particularly if some future change -// were to introduce globbing to file -// specifications in the kustomization file. -// -// These restrictions assure that kustomizations -// are self-contained and relocatable, and impose -// some safety when relying on remote kustomizations, -// e.g. a remotely loaded ConfigMap generator specified -// to read from /etc/passwd will fail. -type FileLoader struct { - // Loader that spawned this loader. - // Used to avoid cycles. - referrer *FileLoader - - // An absolute, cleaned path to a directory. - // The Load function will read non-absolute - // paths relative to this directory. - root filesys.ConfirmedDir - - // Restricts behavior of Load function. - loadRestrictor LoadRestrictorFunc - - // If this is non-nil, the files were - // obtained from the given repository. - repoSpec *git.RepoSpec - - // File system utilities. - fSys filesys.FileSystem - - // Used to load from HTTP - http *http.Client - - // Used to clone repositories. - cloner git.Cloner - - // Used to clean up, as needed. - cleaner func() error -} - -// This redirect code does not process automaticali by http client and we can process it manualy -const MULTIPLE_CHOICES_REDIRECT_CODE = 300 - -// Repo returns the absolute path to the repo that contains Root if this fileLoader was created from a url -// or the empty string otherwise. -func (fl *FileLoader) Repo() string { - if fl.repoSpec != nil { - return fl.repoSpec.Dir.String() - } - return "" -} - -// Root returns the absolute path that is prepended to any -// relative paths used in Load. -func (fl *FileLoader) Root() string { - return fl.root.String() -} - -func NewLoaderOrDie( - lr LoadRestrictorFunc, - fSys filesys.FileSystem, path string) *FileLoader { - root, err := filesys.ConfirmDir(fSys, path) - if err != nil { - log.Fatalf("unable to make loader at '%s'; %v", path, err) - } - return newLoaderAtConfirmedDir( - lr, root, fSys, nil, git.ClonerUsingGitExec) -} - -// newLoaderAtConfirmedDir returns a new FileLoader with given root. -func newLoaderAtConfirmedDir( - lr LoadRestrictorFunc, - root filesys.ConfirmedDir, fSys filesys.FileSystem, - referrer *FileLoader, cloner git.Cloner) *FileLoader { - return &FileLoader{ - loadRestrictor: lr, - root: root, - referrer: referrer, - fSys: fSys, - cloner: cloner, - cleaner: func() error { return nil }, - } -} - -// New returns a new Loader, rooted relative to current loader, -// or rooted in a temp directory holding a git repo clone. -func (fl *FileLoader) New(path string) (ifc.Loader, error) { - if path == "" { - return nil, errors.Errorf("new root cannot be empty") - } - - repoSpec, err := git.NewRepoSpecFromURL(path) - if err == nil { - // Treat this as git repo clone request. - if err = fl.errIfRepoCycle(repoSpec); err != nil { - return nil, err - } - return newLoaderAtGitClone( - repoSpec, fl.fSys, fl, fl.cloner) - } - - if filepath.IsAbs(path) { - return nil, fmt.Errorf("new root '%s' cannot be absolute", path) - } - root, err := filesys.ConfirmDir(fl.fSys, fl.root.Join(path)) - if err != nil { - return nil, errors.WrapPrefixf(err, ErrRtNotDir.Error()) - } - if err = fl.errIfGitContainmentViolation(root); err != nil { - return nil, err - } - if err = fl.errIfArgEqualOrHigher(root); err != nil { - return nil, err - } - return newLoaderAtConfirmedDir( - fl.loadRestrictor, root, fl.fSys, fl, fl.cloner), nil -} - -// newLoaderAtGitClone returns a new Loader pinned to a temporary -// directory holding a cloned git repo. -func newLoaderAtGitClone( - repoSpec *git.RepoSpec, fSys filesys.FileSystem, - referrer *FileLoader, cloner git.Cloner) (ifc.Loader, error) { - cleaner := repoSpec.Cleaner(fSys) - err := cloner(repoSpec) - if err != nil { - cleaner() - return nil, err - } - root, f, err := fSys.CleanedAbs(repoSpec.AbsPath()) - if err != nil { - cleaner() - return nil, err - } - // We don't know that the path requested in repoSpec - // is a directory until we actually clone it and look - // inside. That just happened, hence the error check - // is here. - if f != "" { - cleaner() - return nil, fmt.Errorf( - "'%s' refers to file '%s'; expecting directory", - repoSpec.AbsPath(), f) - } - // Path in repo can contain symlinks that exit repo. We can only - // check for this after cloning repo. - if !root.HasPrefix(repoSpec.CloneDir()) { - _ = cleaner() - return nil, fmt.Errorf("%q refers to directory outside of repo %q", repoSpec.AbsPath(), - repoSpec.CloneDir()) - } - return &FileLoader{ - // Clones never allowed to escape root. - loadRestrictor: RestrictionRootOnly, - root: root, - referrer: referrer, - repoSpec: repoSpec, - fSys: fSys, - cloner: cloner, - cleaner: cleaner, - }, nil -} - -func (fl *FileLoader) errIfGitContainmentViolation( - base filesys.ConfirmedDir) error { - containingRepo := fl.containingRepo() - if containingRepo == nil { - return nil - } - if !base.HasPrefix(containingRepo.CloneDir()) { - return fmt.Errorf( - "security; bases in kustomizations found in "+ - "cloned git repos must be within the repo, "+ - "but base '%s' is outside '%s'", - base, containingRepo.CloneDir()) - } - return nil -} - -// Looks back through referrers for a git repo, returning nil -// if none found. -func (fl *FileLoader) containingRepo() *git.RepoSpec { - if fl.repoSpec != nil { - return fl.repoSpec - } - if fl.referrer == nil { - return nil - } - return fl.referrer.containingRepo() -} - -// errIfArgEqualOrHigher tests whether the argument, -// is equal to or above the root of any ancestor. -func (fl *FileLoader) errIfArgEqualOrHigher( - candidateRoot filesys.ConfirmedDir) error { - if fl.root.HasPrefix(candidateRoot) { - return fmt.Errorf( - "cycle detected: candidate root '%s' contains visited root '%s'", - candidateRoot, fl.root) - } - if fl.referrer == nil { - return nil - } - return fl.referrer.errIfArgEqualOrHigher(candidateRoot) -} - -// TODO(monopole): Distinguish branches? -// I.e. Allow a distinction between git URI with -// path foo and tag bar and a git URI with the same -// path but a different tag? -func (fl *FileLoader) errIfRepoCycle(newRepoSpec *git.RepoSpec) error { - // TODO(monopole): Use parsed data instead of Raw(). - if fl.repoSpec != nil && - strings.HasPrefix(fl.repoSpec.Raw(), newRepoSpec.Raw()) { - return fmt.Errorf( - "cycle detected: URI '%s' referenced by previous URI '%s'", - newRepoSpec.Raw(), fl.repoSpec.Raw()) - } - if fl.referrer == nil { - return nil - } - return fl.referrer.errIfRepoCycle(newRepoSpec) -} - -// Load returns the content of file at the given path, -// else an error. Relative paths are taken relative -// to the root. -func (fl *FileLoader) Load(path string) ([]byte, error) { - if IsRemoteFile(path) { - return fl.httpClientGetContent(path) - } - if !filepath.IsAbs(path) { - path = fl.root.Join(path) - } - path, err := fl.loadRestrictor(fl.fSys, fl.root, path) - if err != nil { - return nil, err - } - return fl.fSys.ReadFile(path) -} - -func (fl *FileLoader) httpClientGetContent(path string) ([]byte, error) { - var hc *http.Client - if fl.http != nil { - hc = fl.http - } else { - hc = &http.Client{} - } - resp, err := hc.Get(path) - if err != nil { - return nil, errors.Wrap(err) - } - defer resp.Body.Close() - // response unsuccessful - if resp.StatusCode < 200 || resp.StatusCode > 299 { - if resp.StatusCode == MULTIPLE_CHOICES_REDIRECT_CODE { - var newPath string = resp.Header.Get("Location") - return nil, &RedirectionError{ - Msg: "Response is redirect", - NewPath: newPath, - } - } else { - _, err = git.NewRepoSpecFromURL(path) - if err == nil { - return nil, errors.Errorf("URL is a git repository") - } - return nil, fmt.Errorf("%w: status code %d (%s)", ErrHTTP, resp.StatusCode, http.StatusText(resp.StatusCode)) - } - } - content, err := io.ReadAll(resp.Body) - return content, errors.Wrap(err) -} - -// Cleanup runs the cleaner. -func (fl *FileLoader) Cleanup() error { - return fl.cleaner() -} +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package loader + +import ( + "fmt" + "io" + "log" + "net/http" + "net/url" + "path/filepath" + "strings" + + "sigs.k8s.io/kustomize/api/ifc" + "sigs.k8s.io/kustomize/api/internal/git" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +// IsRemoteFile returns whether path has a url scheme that kustomize allows for +// remote files. See https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md +func IsRemoteFile(path string) bool { + u, err := url.Parse(path) + return err == nil && (u.Scheme == "http" || u.Scheme == "https") +} + +// FileLoader is a kustomization's interface to files. +// +// The directory in which a kustomization file sits +// is referred to below as the kustomization's _root_. +// +// An instance of fileLoader has an immutable root, +// and offers a `New` method returning a new loader +// with a new root. +// +// A kustomization file refers to two kinds of files: +// +// * supplemental data paths +// +// `Load` is used to visit these paths. +// +// These paths refer to resources, patches, +// data for ConfigMaps and Secrets, etc. +// +// The loadRestrictor may disallow certain paths +// or classes of paths. +// +// * bases (other kustomizations) +// +// `New` is used to load bases. +// +// A base can be either a remote git repo URL, or +// a directory specified relative to the current +// root. In the former case, the repo is locally +// cloned, and the new loader is rooted on a path +// in that clone. +// +// As loaders create new loaders, a root history +// is established, and used to disallow: +// +// - A base that is a repository that, in turn, +// specifies a base repository seen previously +// in the loading stack (a cycle). +// +// - An overlay depending on a base positioned at +// or above it. I.e. '../foo' is OK, but '.', +// '..', '../..', etc. are disallowed. Allowing +// such a base has no advantages and encourages +// cycles, particularly if some future change +// were to introduce globbing to file +// specifications in the kustomization file. +// +// These restrictions assure that kustomizations +// are self-contained and relocatable, and impose +// some safety when relying on remote kustomizations, +// e.g. a remotely loaded ConfigMap generator specified +// to read from /etc/passwd will fail. +type FileLoader struct { + // Loader that spawned this loader. + // Used to avoid cycles. + referrer *FileLoader + + // An absolute, cleaned path to a directory. + // The Load function will read non-absolute + // paths relative to this directory. + root filesys.ConfirmedDir + + // Restricts behavior of Load function. + loadRestrictor LoadRestrictorFunc + + // If this is non-nil, the files were + // obtained from the given repository. + repoSpec *git.RepoSpec + + // File system utilities. + fSys filesys.FileSystem + + // Used to load from HTTP + http *http.Client + + // Used to clone repositories. + cloner git.Cloner + + // Used to clean up, as needed. + cleaner func() error +} + +// This redirect code does not process automaticali by http client and we can process it manualy +const MULTIPLE_CHOICES_REDIRECT_CODE = 300 + +// Repo returns the absolute path to the repo that contains Root if this fileLoader was created from a url +// or the empty string otherwise. +func (fl *FileLoader) Repo() string { + if fl.repoSpec != nil { + return fl.repoSpec.Dir.String() + } + return "" +} + +// Root returns the absolute path that is prepended to any +// relative paths used in Load. +func (fl *FileLoader) Root() string { + return fl.root.String() +} + +func NewLoaderOrDie( + lr LoadRestrictorFunc, + fSys filesys.FileSystem, path string) *FileLoader { + root, err := filesys.ConfirmDir(fSys, path) + if err != nil { + log.Fatalf("unable to make loader at '%s'; %v", path, err) + } + return newLoaderAtConfirmedDir( + lr, root, fSys, nil, git.ClonerUsingGitExec) +} + +// newLoaderAtConfirmedDir returns a new FileLoader with given root. +func newLoaderAtConfirmedDir( + lr LoadRestrictorFunc, + root filesys.ConfirmedDir, fSys filesys.FileSystem, + referrer *FileLoader, cloner git.Cloner) *FileLoader { + return &FileLoader{ + loadRestrictor: lr, + root: root, + referrer: referrer, + fSys: fSys, + cloner: cloner, + cleaner: func() error { return nil }, + } +} + +// New returns a new Loader, rooted relative to current loader, +// or rooted in a temp directory holding a git repo clone. +func (fl *FileLoader) New(path string) (ifc.Loader, error) { + if path == "" { + return nil, errors.Errorf("new root cannot be empty") + } + + repoSpec, err := git.NewRepoSpecFromURL(path) + if err == nil { + // Treat this as git repo clone request. + if err = fl.errIfRepoCycle(repoSpec); err != nil { + return nil, err + } + return newLoaderAtGitClone( + repoSpec, fl.fSys, fl, fl.cloner) + } + + if filepath.IsAbs(path) { + return nil, fmt.Errorf("new root '%s' cannot be absolute", path) + } + root, err := filesys.ConfirmDir(fl.fSys, fl.root.Join(path)) + if err != nil { + return nil, errors.WrapPrefixf(err, ErrRtNotDir.Error()) + } + if err = fl.errIfGitContainmentViolation(root); err != nil { + return nil, err + } + if err = fl.errIfArgEqualOrHigher(root); err != nil { + return nil, err + } + return newLoaderAtConfirmedDir( + fl.loadRestrictor, root, fl.fSys, fl, fl.cloner), nil +} + +// newLoaderAtGitClone returns a new Loader pinned to a temporary +// directory holding a cloned git repo. +func newLoaderAtGitClone( + repoSpec *git.RepoSpec, fSys filesys.FileSystem, + referrer *FileLoader, cloner git.Cloner) (ifc.Loader, error) { + cleaner := repoSpec.Cleaner(fSys) + err := cloner(repoSpec) + if err != nil { + cleaner() + return nil, err + } + root, f, err := fSys.CleanedAbs(repoSpec.AbsPath()) + if err != nil { + cleaner() + return nil, err + } + // We don't know that the path requested in repoSpec + // is a directory until we actually clone it and look + // inside. That just happened, hence the error check + // is here. + if f != "" { + cleaner() + return nil, fmt.Errorf( + "'%s' refers to file '%s'; expecting directory", + repoSpec.AbsPath(), f) + } + // Path in repo can contain symlinks that exit repo. We can only + // check for this after cloning repo. + if !root.HasPrefix(repoSpec.CloneDir()) { + _ = cleaner() + return nil, fmt.Errorf("%q refers to directory outside of repo %q", repoSpec.AbsPath(), + repoSpec.CloneDir()) + } + return &FileLoader{ + // Clones never allowed to escape root. + loadRestrictor: RestrictionRootOnly, + root: root, + referrer: referrer, + repoSpec: repoSpec, + fSys: fSys, + cloner: cloner, + cleaner: cleaner, + }, nil +} + +func (fl *FileLoader) errIfGitContainmentViolation( + base filesys.ConfirmedDir) error { + containingRepo := fl.containingRepo() + if containingRepo == nil { + return nil + } + if !base.HasPrefix(containingRepo.CloneDir()) { + return fmt.Errorf( + "security; bases in kustomizations found in "+ + "cloned git repos must be within the repo, "+ + "but base '%s' is outside '%s'", + base, containingRepo.CloneDir()) + } + return nil +} + +// Looks back through referrers for a git repo, returning nil +// if none found. +func (fl *FileLoader) containingRepo() *git.RepoSpec { + if fl.repoSpec != nil { + return fl.repoSpec + } + if fl.referrer == nil { + return nil + } + return fl.referrer.containingRepo() +} + +// errIfArgEqualOrHigher tests whether the argument, +// is equal to or above the root of any ancestor. +func (fl *FileLoader) errIfArgEqualOrHigher( + candidateRoot filesys.ConfirmedDir) error { + if fl.root.HasPrefix(candidateRoot) { + return fmt.Errorf( + "cycle detected: candidate root '%s' contains visited root '%s'", + candidateRoot, fl.root) + } + if fl.referrer == nil { + return nil + } + return fl.referrer.errIfArgEqualOrHigher(candidateRoot) +} + +// TODO(monopole): Distinguish branches? +// I.e. Allow a distinction between git URI with +// path foo and tag bar and a git URI with the same +// path but a different tag? +func (fl *FileLoader) errIfRepoCycle(newRepoSpec *git.RepoSpec) error { + // TODO(monopole): Use parsed data instead of Raw(). + if fl.repoSpec != nil && + strings.HasPrefix(fl.repoSpec.Raw(), newRepoSpec.Raw()) { + return fmt.Errorf( + "cycle detected: URI '%s' referenced by previous URI '%s'", + newRepoSpec.Raw(), fl.repoSpec.Raw()) + } + if fl.referrer == nil { + return nil + } + return fl.referrer.errIfRepoCycle(newRepoSpec) +} + +// Load returns the content of file at the given path, +// else an error. Relative paths are taken relative +// to the root. +func (fl *FileLoader) Load(path string) ([]byte, error) { + if IsRemoteFile(path) { + return fl.httpClientGetContent(path) + } + if !filepath.IsAbs(path) { + path = fl.root.Join(path) + } + path, err := fl.loadRestrictor(fl.fSys, fl.root, path) + if err != nil { + return nil, err + } + return fl.fSys.ReadFile(path) +} + +func (fl *FileLoader) httpClientGetContent(path string) ([]byte, error) { + var hc *http.Client + if fl.http != nil { + hc = fl.http + } else { + hc = &http.Client{} + } + resp, err := hc.Get(path) + if err != nil { + return nil, errors.Wrap(err) + } + defer resp.Body.Close() + // response unsuccessful + if resp.StatusCode < 200 || resp.StatusCode > 299 { + if resp.StatusCode == MULTIPLE_CHOICES_REDIRECT_CODE { + var newPath string = resp.Header.Get("Location") + return nil, &RedirectionError{ + Msg: "Response is redirect", + NewPath: newPath, + } + } else { + _, err = git.NewRepoSpecFromURL(path) + if err == nil { + return nil, errors.Errorf("URL is a git repository") + } + return nil, fmt.Errorf("%w: status code %d (%s)", ErrHTTP, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + } + content, err := io.ReadAll(resp.Body) + return content, errors.Wrap(err) +} + +// Cleanup runs the cleaner. +func (fl *FileLoader) Cleanup() error { + return fl.cleaner() +} diff --git a/api/internal/loader/fileloader_test.go b/api/internal/loader/fileloader_test.go index f789d4d683..8502613557 100644 --- a/api/internal/loader/fileloader_test.go +++ b/api/internal/loader/fileloader_test.go @@ -1,709 +1,709 @@ -/// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package loader - -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "path" - "path/filepath" - "reflect" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "sigs.k8s.io/kustomize/api/ifc" - "sigs.k8s.io/kustomize/api/internal/git" - "sigs.k8s.io/kustomize/api/internal/loader" - "sigs.k8s.io/kustomize/api/konfig" - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/filesys" -) - -func TestIsRemoteFile(t *testing.T) { - cases := map[string]struct { - url string - valid bool - }{ - "https file": { - "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/examples/helloWorld/configMap.yaml", - true, - }, - "malformed https": { - // TODO(annasong): Maybe we want to fix this. Needs more research. - "https:/raw.githubusercontent.com/kubernetes-sigs/kustomize/master/examples/helloWorld/configMap.yaml", - true, - }, - "https dir": { - "https://github.com/kubernetes-sigs/kustomize//examples/helloWorld/", - true, - }, - "no scheme": { - "github.com/kubernetes-sigs/kustomize//examples/helloWorld/", - false, - }, - "ssh": { - "ssh://git@github.com/kubernetes-sigs/kustomize.git", - false, - }, - "local": { - "pod.yaml", - false, - }, - } - for name, test := range cases { - test := test - t.Run(name, func(t *testing.T) { - require.Equal(t, test.valid, IsRemoteFile(test.url)) - }) - } -} - -type testData struct { - path string - expectedContent string -} - -var testCases = []testData{ - { - path: "foo/project/fileA.yaml", - expectedContent: "fileA content", - }, - { - path: "foo/project/subdir1/fileB.yaml", - expectedContent: "fileB content", - }, - { - path: "foo/project/subdir2/fileC.yaml", - expectedContent: "fileC content", - }, - { - path: "foo/project/fileD.yaml", - expectedContent: "fileD content", - }, -} - -func MakeFakeFs(td []testData) filesys.FileSystem { - fSys := filesys.MakeFsInMemory() - for _, x := range td { - fSys.WriteFile(x.path, []byte(x.expectedContent)) - } - return fSys -} - -func makeLoader() *FileLoader { - return NewLoaderOrDie( - RestrictionRootOnly, MakeFakeFs(testCases), filesys.Separator) -} - -func TestLoaderLoad(t *testing.T) { - require := require.New(t) - - l1 := makeLoader() - repo := l1.Repo() - require.Empty(repo) - require.Equal("/", l1.Root()) - - for _, x := range testCases { - b, err := l1.Load(x.path) - require.NoError(err) - - if !reflect.DeepEqual([]byte(x.expectedContent), b) { - t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) - } - } - l2, err := l1.New("foo/project") - require.NoError(err) - - repo = l2.Repo() - require.Empty(repo) - require.Equal("/foo/project", l2.Root()) - - for _, x := range testCases { - b, err := l2.Load(strings.TrimPrefix(x.path, "foo/project/")) - require.NoError(err) - - if !reflect.DeepEqual([]byte(x.expectedContent), b) { - t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) - } - } - l2, err = l1.New("foo/project/") // Assure trailing slash stripped - require.NoError(err) - require.Equal("/foo/project", l2.Root()) -} - -func TestLoaderNewSubDir(t *testing.T) { - require := require.New(t) - - l1, err := makeLoader().New("foo/project") - require.NoError(err) - - l2, err := l1.New("subdir1") - require.NoError(err) - require.Equal("/foo/project/subdir1", l2.Root()) - - x := testCases[1] - b, err := l2.Load("fileB.yaml") - require.NoError(err) - - if !reflect.DeepEqual([]byte(x.expectedContent), b) { - t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) - } -} - -func TestLoaderBadRelative(t *testing.T) { - require := require.New(t) - - l1, err := makeLoader().New("foo/project/subdir1") - require.NoError(err) - require.Equal("/foo/project/subdir1", l1.Root()) - - // Cannot cd into a file. - l2, err := l1.New("fileB.yaml") - require.Error(err) - - // It's not okay to stay at the same place. - l2, err = l1.New(filesys.SelfDir) - require.Error(err) - - // It's not okay to go up and back down into same place. - l2, err = l1.New("../subdir1") - require.Error(err) - - // It's not okay to go up via a relative path. - l2, err = l1.New("..") - require.Error(err) - - // It's not okay to go up via an absolute path. - l2, err = l1.New("/foo/project") - require.Error(err) - - // It's not okay to go to the root. - l2, err = l1.New("/") - require.Error(err) - - // It's okay to go up and down to a sibling. - l2, err = l1.New("../subdir2") - require.NoError(err) - require.Equal("/foo/project/subdir2", l2.Root()) - - x := testCases[2] - b, err := l2.Load("fileC.yaml") - require.NoError(err) - if !reflect.DeepEqual([]byte(x.expectedContent), b) { - t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) - } - - // It's not OK to go over to a previously visited directory. - // Must disallow going back and forth in a cycle. - l1, err = l2.New("../subdir1") - require.Error(err) -} - -func TestNewEmptyLoader(t *testing.T) { - _, err := makeLoader().New("") - require.Error(t, err) -} - -func TestNewRemoteLoaderDoesNotExist(t *testing.T) { - _, err := makeLoader().New("https://example.com/org/repo") - require.ErrorContains(t, err, "fetch") -} - -func TestLoaderLocalScheme(t *testing.T) { - // It is unlikely but possible for a reference with a url scheme to - // actually refer to a local file or directory. - t.Run("file", func(t *testing.T) { - fSys, dir := setupOnDisk(t) - parts := []string{ - "ssh:", - "resource.yaml", - } - require.NoError(t, fSys.Mkdir(dir.Join(parts[0]))) - const content = "resource config" - require.NoError(t, fSys.WriteFile( - dir.Join(filepath.Join(parts...)), - []byte(content), - )) - actualContent, err := NewLoaderOrDie(RestrictionRootOnly, - fSys, - dir.String(), - ).Load(strings.Join(parts, "//")) - require.NoError(t, err) - require.Equal(t, content, string(actualContent)) - }) - t.Run("directory", func(t *testing.T) { - fSys, dir := setupOnDisk(t) - parts := []string{ - "https:", - "root", - } - require.NoError(t, fSys.MkdirAll(dir.Join(filepath.Join(parts...)))) - ldr, err := NewLoaderOrDie(RestrictionRootOnly, - fSys, - dir.String(), - ).New(strings.Join(parts, "//")) - require.NoError(t, err) - require.Empty(t, ldr.Repo()) - }) -} - -const ( - contentOk = "hi there, i'm OK data" - contentExteriorData = "i am data from outside the root" -) - -// Create a structure like this -// -// /tmp/kustomize-test-random -// ├── base -// │ ├── okayData -// │ ├── symLinkToOkayData -> okayData -// │ └── symLinkToExteriorData -> ../exteriorData -// └── exteriorData -func commonSetupForLoaderRestrictionTest(t *testing.T) (string, filesys.FileSystem) { - t.Helper() - fSys, tmpDir := setupOnDisk(t) - dir := tmpDir.String() - - fSys.Mkdir(filepath.Join(dir, "base")) - - fSys.WriteFile( - filepath.Join(dir, "base", "okayData"), []byte(contentOk)) - - fSys.WriteFile( - filepath.Join(dir, "exteriorData"), []byte(contentExteriorData)) - - os.Symlink( - filepath.Join(dir, "base", "okayData"), - filepath.Join(dir, "base", "symLinkToOkayData")) - os.Symlink( - filepath.Join(dir, "exteriorData"), - filepath.Join(dir, "base", "symLinkToExteriorData")) - return dir, fSys -} - -// Make sure everything works when loading files -// in or below the loader root. -func doSanityChecksAndDropIntoBase( - t *testing.T, l ifc.Loader) ifc.Loader { - t.Helper() - require := require.New(t) - - data, err := l.Load(path.Join("base", "okayData")) - require.NoError(err) - require.Equal(contentOk, string(data)) - - data, err = l.Load("exteriorData") - require.NoError(err) - require.Equal(contentExteriorData, string(data)) - - // Drop in. - l, err = l.New("base") - require.NoError(err) - - // Reading okayData works. - data, err = l.Load("okayData") - require.NoError(err) - require.Equal(contentOk, string(data)) - - // Reading local symlink to okayData works. - data, err = l.Load("symLinkToOkayData") - require.NoError(err) - require.Equal(contentOk, string(data)) - - return l -} - -func TestRestrictionRootOnlyInRealLoader(t *testing.T) { - require := require.New(t) - dir, fSys := commonSetupForLoaderRestrictionTest(t) - - var l ifc.Loader - - l = NewLoaderOrDie(RestrictionRootOnly, fSys, dir) - - l = doSanityChecksAndDropIntoBase(t, l) - - // Reading symlink to exteriorData fails. - _, err := l.Load("symLinkToExteriorData") - require.Error(err) - require.Contains(err.Error(), "is not in or below") - - // Attempt to read "up" fails, though earlier we were - // able to read this file when root was "..". - _, err = l.Load("../exteriorData") - require.Error(err) - require.Contains(err.Error(), "is not in or below") -} - -func TestRestrictionNoneInRealLoader(t *testing.T) { - dir, fSys := commonSetupForLoaderRestrictionTest(t) - - var l ifc.Loader - - l = NewLoaderOrDie(RestrictionNone, fSys, dir) - - l = doSanityChecksAndDropIntoBase(t, l) - - // Reading symlink to exteriorData works. - _, err := l.Load("symLinkToExteriorData") - require.NoError(t, err) - - // Attempt to read "up" works. - _, err = l.Load("../exteriorData") - require.NoError(t, err) -} - -func splitOnNthSlash(v string, n int) (string, string) { - left := "" - for i := 0; i < n; i++ { - k := strings.Index(v, "/") - if k < 0 { - break - } - left += v[:k+1] - v = v[k+1:] - } - return left[:len(left)-1], v -} - -func TestSplit(t *testing.T) { - p := "a/b/c/d/e/f/g" - if left, right := splitOnNthSlash(p, 2); left != "a/b" || right != "c/d/e/f/g" { - t.Fatalf("got left='%s', right='%s'", left, right) - } - if left, right := splitOnNthSlash(p, 3); left != "a/b/c" || right != "d/e/f/g" { - t.Fatalf("got left='%s', right='%s'", left, right) - } - if left, right := splitOnNthSlash(p, 6); left != "a/b/c/d/e/f" || right != "g" { - t.Fatalf("got left='%s', right='%s'", left, right) - } -} - -func TestNewLoaderAtGitClone(t *testing.T) { - require := require.New(t) - - rootURL := "github.com/someOrg/someRepo" - pathInRepo := "foo/base" - url := rootURL + "/" + pathInRepo - coRoot := "/tmp" - fSys := filesys.MakeFsInMemory() - fSys.MkdirAll(coRoot) - fSys.MkdirAll(coRoot + "/" + pathInRepo) - fSys.WriteFile( - coRoot+"/"+pathInRepo+"/"+ - konfig.DefaultKustomizationFileName(), - []byte(` -whatever -`)) - - repoSpec, err := git.NewRepoSpecFromURL(url) - require.NoError(err) - - l, err := newLoaderAtGitClone( - repoSpec, fSys, nil, - git.DoNothingCloner(filesys.ConfirmedDir(coRoot))) - require.NoError(err) - repo := l.Repo() - require.Equal(coRoot, repo) - require.Equal(coRoot+"/"+pathInRepo, l.Root()) - - _, err = l.New(url) - require.Error(err) - - _, err = l.New(rootURL + "/" + "foo") - require.Error(err) - - pathInRepo = "foo/overlay" - fSys.MkdirAll(coRoot + "/" + pathInRepo) - url = rootURL + "/" + pathInRepo - l2, err := l.New(url) - require.NoError(err) - - repo = l2.Repo() - require.Equal(coRoot, repo) - require.Equal(coRoot+"/"+pathInRepo, l2.Root()) -} - -func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) { - require := require.New(t) - - // Define an overlay-base structure in the file system. - topDir := "/whatever" - cloneRoot := topDir + "/someClone" - fSys := filesys.MakeFsInMemory() - fSys.MkdirAll(topDir + "/highBase") - fSys.MkdirAll(cloneRoot + "/foo/base") - fSys.MkdirAll(cloneRoot + "/foo/overlay") - - var l1 ifc.Loader - - // Establish that a local overlay can navigate - // to the local bases. - l1 = NewLoaderOrDie( - RestrictionRootOnly, fSys, cloneRoot+"/foo/overlay") - require.Equal(cloneRoot+"/foo/overlay", l1.Root()) - - l2, err := l1.New("../base") - require.NoError(nil) - require.Equal(cloneRoot+"/foo/base", l2.Root()) - - l3, err := l2.New("../../../highBase") - require.NoError(err) - require.Equal(topDir+"/highBase", l3.Root()) - - // Establish that a Kustomization found in cloned - // repo can reach (non-remote) bases inside the clone - // but cannot reach a (non-remote) base outside the - // clone but legitimately on the local file system. - // This is to avoid a surprising interaction between - // a remote K and local files. The remote K would be - // non-functional on its own since by definition it - // would refer to a non-remote base file that didn't - // exist in its own repository, so presumably the - // remote K would be deliberately designed to phish - // for local K's. - repoSpec, err := git.NewRepoSpecFromURL( - "github.com/someOrg/someRepo/foo/overlay") - require.NoError(err) - - l1, err = newLoaderAtGitClone( - repoSpec, fSys, nil, - git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) - require.NoError(err) - require.Equal(cloneRoot+"/foo/overlay", l1.Root()) - - // This is okay. - l2, err = l1.New("../base") - require.NoError(err) - repo := l2.Repo() - require.Empty(repo) - require.Equal(cloneRoot+"/foo/base", l2.Root()) - - // This is not okay. - _, err = l2.New("../../../highBase") - require.Error(err) - require.Contains(err.Error(), - "base '/whatever/highBase' is outside '/whatever/someClone'") -} - -func TestLoaderDisallowsRemoteBaseExitRepo(t *testing.T) { - fSys, dir := setupOnDisk(t) - - repo := dir.Join("repo") - require.NoError(t, fSys.Mkdir(repo)) - - base := filepath.Join(repo, "base") - require.NoError(t, os.Symlink(dir.String(), base)) - - repoSpec, err := git.NewRepoSpecFromURL("https://github.com/org/repo/base") - require.NoError(t, err) - - _, err = newLoaderAtGitClone(repoSpec, fSys, nil, git.DoNothingCloner(filesys.ConfirmedDir(repo))) - require.Error(t, err) - require.Contains(t, err.Error(), fmt.Sprintf("%q refers to directory outside of repo %q", base, repo)) -} - -func TestLocalLoaderReferencingGitBase(t *testing.T) { - require := require.New(t) - - topDir := "/whatever" - cloneRoot := topDir + "/someClone" - fSys := filesys.MakeFsInMemory() - fSys.MkdirAll(topDir) - fSys.MkdirAll(cloneRoot + "/foo/base") - - l1 := newLoaderAtConfirmedDir( - RestrictionRootOnly, filesys.ConfirmedDir(topDir), fSys, nil, - git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) - require.Equal(topDir, l1.Root()) - - l2, err := l1.New("github.com/someOrg/someRepo/foo/base") - require.NoError(err) - repo := l2.Repo() - require.Equal(cloneRoot, repo) - require.Equal(cloneRoot+"/foo/base", l2.Root()) -} - -func TestRepoDirectCycleDetection(t *testing.T) { - require := require.New(t) - - topDir := "/cycles" - cloneRoot := topDir + "/someClone" - fSys := filesys.MakeFsInMemory() - fSys.MkdirAll(topDir) - fSys.MkdirAll(cloneRoot) - - l1 := newLoaderAtConfirmedDir( - RestrictionRootOnly, filesys.ConfirmedDir(topDir), fSys, nil, - git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) - p1 := "github.com/someOrg/someRepo/foo" - rs1, err := git.NewRepoSpecFromURL(p1) - require.NoError(err) - - l1.repoSpec = rs1 - _, err = l1.New(p1) - require.Error(err) - require.Contains(err.Error(), "cycle detected") -} - -func TestRepoIndirectCycleDetection(t *testing.T) { - require := require.New(t) - - topDir := "/cycles" - cloneRoot := topDir + "/someClone" - fSys := filesys.MakeFsInMemory() - fSys.MkdirAll(topDir) - fSys.MkdirAll(cloneRoot) - - l0 := newLoaderAtConfirmedDir( - RestrictionRootOnly, filesys.ConfirmedDir(topDir), fSys, nil, - git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) - - p1 := "github.com/someOrg/someRepo1" - p2 := "github.com/someOrg/someRepo2" - - l1, err := l0.New(p1) - require.NoError(err) - - l2, err := l1.New(p2) - require.NoError(err) - - _, err = l2.New(p1) - require.Error(err) - require.Contains(err.Error(), "cycle detected") -} - -// Inspired by https://hassansin.github.io/Unit-Testing-http-client-in-Go -type fakeRoundTripper func(req *http.Request) *http.Response - -func (f fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req), nil -} - -func makeFakeHTTPClient(fn fakeRoundTripper) *http.Client { - return &http.Client{ - Transport: fn, - } -} - -// TestLoaderHTTP test http file loader -func TestLoaderHTTP(t *testing.T) { - require := require.New(t) - - var testCasesFile = []testData{ - { - path: "http/file.yaml", - expectedContent: "file content", - }, - } - - l1 := NewLoaderOrDie( - RestrictionRootOnly, MakeFakeFs(testCasesFile), filesys.Separator) - require.Equal("/", l1.Root()) - - for _, x := range testCasesFile { - b, err := l1.Load(x.path) - require.NoError(err) - if !reflect.DeepEqual([]byte(x.expectedContent), b) { - t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) - } - } - - var testCasesHTTP = []testData{ - { - path: "http://example.com/resource.yaml", - expectedContent: "http content", - }, - { - path: "https://example.com/resource.yaml", - expectedContent: "https content", - }, - } - - for _, x := range testCasesHTTP { - hc := makeFakeHTTPClient(func(req *http.Request) *http.Response { - u := req.URL.String() - require.Equal(x.path, u) - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewBufferString(x.expectedContent)), - Header: make(http.Header), - } - }) - l2 := l1 - l2.http = hc - b, err := l2.Load(x.path) - require.NoError(err) - if !reflect.DeepEqual([]byte(x.expectedContent), b) { - t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) - } - } - - var testCaseUnsupported = []testData{ - { - path: "httpsnotreal://example.com/resource.yaml", - expectedContent: "invalid", - }, - } - for _, x := range testCaseUnsupported { - hc := makeFakeHTTPClient(func(req *http.Request) *http.Response { - t.Fatalf("unexpected request to URL %s", req.URL.String()) - return nil - }) - l2 := l1 - l2.http = hc - _, err := l2.Load(x.path) - require.Error(err) - } - - var testCaseRedirect = []testData{ - { - path: "https://example.com/resource.yaml", - expectedContent: "https content", - }, - } - for _, x := range testCaseRedirect { - expectedLocation := "https://redirect.com/resource.yaml" - hc := makeFakeHTTPClient(func(req *http.Request) *http.Response { - response := &http.Response{ - StatusCode: 300, - Body: io.NopCloser(bytes.NewBufferString("")), - Header: make(http.Header), - } - response.Header.Add("Location", expectedLocation) - return response - }) - l2 := l1 - l2.http = hc - _, err := l2.Load(x.path) - require.Error(err) - var redErr *loader.RedirectionError - var path string = "" - if errors.As(err, &redErr) { - path = redErr.NewPath - } - require.Equal(expectedLocation, path) - } -} - -// setupOnDisk sets up a file system on disk and directory that is cleaned after -// test completion. -// TODO(annasong): Move all loader tests that require real file system into -// api/krusty. -func setupOnDisk(t *testing.T) (filesys.FileSystem, filesys.ConfirmedDir) { - t.Helper() - - fSys := filesys.MakeFsOnDisk() - dir, err := filesys.NewTmpConfirmedDir() - require.NoError(t, err) - t.Cleanup(func() { - _ = fSys.RemoveAll(dir.String()) - }) - return fSys, dir -} +/// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package loader + +import ( + "bytes" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/kustomize/api/ifc" + "sigs.k8s.io/kustomize/api/internal/git" + "sigs.k8s.io/kustomize/api/internal/loader" + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +func TestIsRemoteFile(t *testing.T) { + cases := map[string]struct { + url string + valid bool + }{ + "https file": { + "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/examples/helloWorld/configMap.yaml", + true, + }, + "malformed https": { + // TODO(annasong): Maybe we want to fix this. Needs more research. + "https:/raw.githubusercontent.com/kubernetes-sigs/kustomize/master/examples/helloWorld/configMap.yaml", + true, + }, + "https dir": { + "https://github.com/kubernetes-sigs/kustomize//examples/helloWorld/", + true, + }, + "no scheme": { + "github.com/kubernetes-sigs/kustomize//examples/helloWorld/", + false, + }, + "ssh": { + "ssh://git@github.com/kubernetes-sigs/kustomize.git", + false, + }, + "local": { + "pod.yaml", + false, + }, + } + for name, test := range cases { + test := test + t.Run(name, func(t *testing.T) { + require.Equal(t, test.valid, IsRemoteFile(test.url)) + }) + } +} + +type testData struct { + path string + expectedContent string +} + +var testCases = []testData{ + { + path: "foo/project/fileA.yaml", + expectedContent: "fileA content", + }, + { + path: "foo/project/subdir1/fileB.yaml", + expectedContent: "fileB content", + }, + { + path: "foo/project/subdir2/fileC.yaml", + expectedContent: "fileC content", + }, + { + path: "foo/project/fileD.yaml", + expectedContent: "fileD content", + }, +} + +func MakeFakeFs(td []testData) filesys.FileSystem { + fSys := filesys.MakeFsInMemory() + for _, x := range td { + fSys.WriteFile(x.path, []byte(x.expectedContent)) + } + return fSys +} + +func makeLoader() *FileLoader { + return NewLoaderOrDie( + RestrictionRootOnly, MakeFakeFs(testCases), filesys.Separator) +} + +func TestLoaderLoad(t *testing.T) { + require := require.New(t) + + l1 := makeLoader() + repo := l1.Repo() + require.Empty(repo) + require.Equal("/", l1.Root()) + + for _, x := range testCases { + b, err := l1.Load(x.path) + require.NoError(err) + + if !reflect.DeepEqual([]byte(x.expectedContent), b) { + t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) + } + } + l2, err := l1.New("foo/project") + require.NoError(err) + + repo = l2.Repo() + require.Empty(repo) + require.Equal("/foo/project", l2.Root()) + + for _, x := range testCases { + b, err := l2.Load(strings.TrimPrefix(x.path, "foo/project/")) + require.NoError(err) + + if !reflect.DeepEqual([]byte(x.expectedContent), b) { + t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) + } + } + l2, err = l1.New("foo/project/") // Assure trailing slash stripped + require.NoError(err) + require.Equal("/foo/project", l2.Root()) +} + +func TestLoaderNewSubDir(t *testing.T) { + require := require.New(t) + + l1, err := makeLoader().New("foo/project") + require.NoError(err) + + l2, err := l1.New("subdir1") + require.NoError(err) + require.Equal("/foo/project/subdir1", l2.Root()) + + x := testCases[1] + b, err := l2.Load("fileB.yaml") + require.NoError(err) + + if !reflect.DeepEqual([]byte(x.expectedContent), b) { + t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) + } +} + +func TestLoaderBadRelative(t *testing.T) { + require := require.New(t) + + l1, err := makeLoader().New("foo/project/subdir1") + require.NoError(err) + require.Equal("/foo/project/subdir1", l1.Root()) + + // Cannot cd into a file. + l2, err := l1.New("fileB.yaml") + require.Error(err) + + // It's not okay to stay at the same place. + l2, err = l1.New(filesys.SelfDir) + require.Error(err) + + // It's not okay to go up and back down into same place. + l2, err = l1.New("../subdir1") + require.Error(err) + + // It's not okay to go up via a relative path. + l2, err = l1.New("..") + require.Error(err) + + // It's not okay to go up via an absolute path. + l2, err = l1.New("/foo/project") + require.Error(err) + + // It's not okay to go to the root. + l2, err = l1.New("/") + require.Error(err) + + // It's okay to go up and down to a sibling. + l2, err = l1.New("../subdir2") + require.NoError(err) + require.Equal("/foo/project/subdir2", l2.Root()) + + x := testCases[2] + b, err := l2.Load("fileC.yaml") + require.NoError(err) + if !reflect.DeepEqual([]byte(x.expectedContent), b) { + t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) + } + + // It's not OK to go over to a previously visited directory. + // Must disallow going back and forth in a cycle. + l1, err = l2.New("../subdir1") + require.Error(err) +} + +func TestNewEmptyLoader(t *testing.T) { + _, err := makeLoader().New("") + require.Error(t, err) +} + +func TestNewRemoteLoaderDoesNotExist(t *testing.T) { + _, err := makeLoader().New("https://example.com/org/repo") + require.ErrorContains(t, err, "fetch") +} + +func TestLoaderLocalScheme(t *testing.T) { + // It is unlikely but possible for a reference with a url scheme to + // actually refer to a local file or directory. + t.Run("file", func(t *testing.T) { + fSys, dir := setupOnDisk(t) + parts := []string{ + "ssh:", + "resource.yaml", + } + require.NoError(t, fSys.Mkdir(dir.Join(parts[0]))) + const content = "resource config" + require.NoError(t, fSys.WriteFile( + dir.Join(filepath.Join(parts...)), + []byte(content), + )) + actualContent, err := NewLoaderOrDie(RestrictionRootOnly, + fSys, + dir.String(), + ).Load(strings.Join(parts, "//")) + require.NoError(t, err) + require.Equal(t, content, string(actualContent)) + }) + t.Run("directory", func(t *testing.T) { + fSys, dir := setupOnDisk(t) + parts := []string{ + "https:", + "root", + } + require.NoError(t, fSys.MkdirAll(dir.Join(filepath.Join(parts...)))) + ldr, err := NewLoaderOrDie(RestrictionRootOnly, + fSys, + dir.String(), + ).New(strings.Join(parts, "//")) + require.NoError(t, err) + require.Empty(t, ldr.Repo()) + }) +} + +const ( + contentOk = "hi there, i'm OK data" + contentExteriorData = "i am data from outside the root" +) + +// Create a structure like this +// +// /tmp/kustomize-test-random +// ├── base +// │ ├── okayData +// │ ├── symLinkToOkayData -> okayData +// │ └── symLinkToExteriorData -> ../exteriorData +// └── exteriorData +func commonSetupForLoaderRestrictionTest(t *testing.T) (string, filesys.FileSystem) { + t.Helper() + fSys, tmpDir := setupOnDisk(t) + dir := tmpDir.String() + + fSys.Mkdir(filepath.Join(dir, "base")) + + fSys.WriteFile( + filepath.Join(dir, "base", "okayData"), []byte(contentOk)) + + fSys.WriteFile( + filepath.Join(dir, "exteriorData"), []byte(contentExteriorData)) + + os.Symlink( + filepath.Join(dir, "base", "okayData"), + filepath.Join(dir, "base", "symLinkToOkayData")) + os.Symlink( + filepath.Join(dir, "exteriorData"), + filepath.Join(dir, "base", "symLinkToExteriorData")) + return dir, fSys +} + +// Make sure everything works when loading files +// in or below the loader root. +func doSanityChecksAndDropIntoBase( + t *testing.T, l ifc.Loader) ifc.Loader { + t.Helper() + require := require.New(t) + + data, err := l.Load(path.Join("base", "okayData")) + require.NoError(err) + require.Equal(contentOk, string(data)) + + data, err = l.Load("exteriorData") + require.NoError(err) + require.Equal(contentExteriorData, string(data)) + + // Drop in. + l, err = l.New("base") + require.NoError(err) + + // Reading okayData works. + data, err = l.Load("okayData") + require.NoError(err) + require.Equal(contentOk, string(data)) + + // Reading local symlink to okayData works. + data, err = l.Load("symLinkToOkayData") + require.NoError(err) + require.Equal(contentOk, string(data)) + + return l +} + +func TestRestrictionRootOnlyInRealLoader(t *testing.T) { + require := require.New(t) + dir, fSys := commonSetupForLoaderRestrictionTest(t) + + var l ifc.Loader + + l = NewLoaderOrDie(RestrictionRootOnly, fSys, dir) + + l = doSanityChecksAndDropIntoBase(t, l) + + // Reading symlink to exteriorData fails. + _, err := l.Load("symLinkToExteriorData") + require.Error(err) + require.Contains(err.Error(), "is not in or below") + + // Attempt to read "up" fails, though earlier we were + // able to read this file when root was "..". + _, err = l.Load("../exteriorData") + require.Error(err) + require.Contains(err.Error(), "is not in or below") +} + +func TestRestrictionNoneInRealLoader(t *testing.T) { + dir, fSys := commonSetupForLoaderRestrictionTest(t) + + var l ifc.Loader + + l = NewLoaderOrDie(RestrictionNone, fSys, dir) + + l = doSanityChecksAndDropIntoBase(t, l) + + // Reading symlink to exteriorData works. + _, err := l.Load("symLinkToExteriorData") + require.NoError(t, err) + + // Attempt to read "up" works. + _, err = l.Load("../exteriorData") + require.NoError(t, err) +} + +func splitOnNthSlash(v string, n int) (string, string) { + left := "" + for i := 0; i < n; i++ { + k := strings.Index(v, "/") + if k < 0 { + break + } + left += v[:k+1] + v = v[k+1:] + } + return left[:len(left)-1], v +} + +func TestSplit(t *testing.T) { + p := "a/b/c/d/e/f/g" + if left, right := splitOnNthSlash(p, 2); left != "a/b" || right != "c/d/e/f/g" { + t.Fatalf("got left='%s', right='%s'", left, right) + } + if left, right := splitOnNthSlash(p, 3); left != "a/b/c" || right != "d/e/f/g" { + t.Fatalf("got left='%s', right='%s'", left, right) + } + if left, right := splitOnNthSlash(p, 6); left != "a/b/c/d/e/f" || right != "g" { + t.Fatalf("got left='%s', right='%s'", left, right) + } +} + +func TestNewLoaderAtGitClone(t *testing.T) { + require := require.New(t) + + rootURL := "github.com/someOrg/someRepo" + pathInRepo := "foo/base" + url := rootURL + "/" + pathInRepo + coRoot := "/tmp" + fSys := filesys.MakeFsInMemory() + fSys.MkdirAll(coRoot) + fSys.MkdirAll(coRoot + "/" + pathInRepo) + fSys.WriteFile( + coRoot+"/"+pathInRepo+"/"+ + konfig.DefaultKustomizationFileName(), + []byte(` +whatever +`)) + + repoSpec, err := git.NewRepoSpecFromURL(url) + require.NoError(err) + + l, err := newLoaderAtGitClone( + repoSpec, fSys, nil, + git.DoNothingCloner(filesys.ConfirmedDir(coRoot))) + require.NoError(err) + repo := l.Repo() + require.Equal(coRoot, repo) + require.Equal(coRoot+"/"+pathInRepo, l.Root()) + + _, err = l.New(url) + require.Error(err) + + _, err = l.New(rootURL + "/" + "foo") + require.Error(err) + + pathInRepo = "foo/overlay" + fSys.MkdirAll(coRoot + "/" + pathInRepo) + url = rootURL + "/" + pathInRepo + l2, err := l.New(url) + require.NoError(err) + + repo = l2.Repo() + require.Equal(coRoot, repo) + require.Equal(coRoot+"/"+pathInRepo, l2.Root()) +} + +func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) { + require := require.New(t) + + // Define an overlay-base structure in the file system. + topDir := "/whatever" + cloneRoot := topDir + "/someClone" + fSys := filesys.MakeFsInMemory() + fSys.MkdirAll(topDir + "/highBase") + fSys.MkdirAll(cloneRoot + "/foo/base") + fSys.MkdirAll(cloneRoot + "/foo/overlay") + + var l1 ifc.Loader + + // Establish that a local overlay can navigate + // to the local bases. + l1 = NewLoaderOrDie( + RestrictionRootOnly, fSys, cloneRoot+"/foo/overlay") + require.Equal(cloneRoot+"/foo/overlay", l1.Root()) + + l2, err := l1.New("../base") + require.NoError(nil) + require.Equal(cloneRoot+"/foo/base", l2.Root()) + + l3, err := l2.New("../../../highBase") + require.NoError(err) + require.Equal(topDir+"/highBase", l3.Root()) + + // Establish that a Kustomization found in cloned + // repo can reach (non-remote) bases inside the clone + // but cannot reach a (non-remote) base outside the + // clone but legitimately on the local file system. + // This is to avoid a surprising interaction between + // a remote K and local files. The remote K would be + // non-functional on its own since by definition it + // would refer to a non-remote base file that didn't + // exist in its own repository, so presumably the + // remote K would be deliberately designed to phish + // for local K's. + repoSpec, err := git.NewRepoSpecFromURL( + "github.com/someOrg/someRepo/foo/overlay") + require.NoError(err) + + l1, err = newLoaderAtGitClone( + repoSpec, fSys, nil, + git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) + require.NoError(err) + require.Equal(cloneRoot+"/foo/overlay", l1.Root()) + + // This is okay. + l2, err = l1.New("../base") + require.NoError(err) + repo := l2.Repo() + require.Empty(repo) + require.Equal(cloneRoot+"/foo/base", l2.Root()) + + // This is not okay. + _, err = l2.New("../../../highBase") + require.Error(err) + require.Contains(err.Error(), + "base '/whatever/highBase' is outside '/whatever/someClone'") +} + +func TestLoaderDisallowsRemoteBaseExitRepo(t *testing.T) { + fSys, dir := setupOnDisk(t) + + repo := dir.Join("repo") + require.NoError(t, fSys.Mkdir(repo)) + + base := filepath.Join(repo, "base") + require.NoError(t, os.Symlink(dir.String(), base)) + + repoSpec, err := git.NewRepoSpecFromURL("https://github.com/org/repo/base") + require.NoError(t, err) + + _, err = newLoaderAtGitClone(repoSpec, fSys, nil, git.DoNothingCloner(filesys.ConfirmedDir(repo))) + require.Error(t, err) + require.Contains(t, err.Error(), fmt.Sprintf("%q refers to directory outside of repo %q", base, repo)) +} + +func TestLocalLoaderReferencingGitBase(t *testing.T) { + require := require.New(t) + + topDir := "/whatever" + cloneRoot := topDir + "/someClone" + fSys := filesys.MakeFsInMemory() + fSys.MkdirAll(topDir) + fSys.MkdirAll(cloneRoot + "/foo/base") + + l1 := newLoaderAtConfirmedDir( + RestrictionRootOnly, filesys.ConfirmedDir(topDir), fSys, nil, + git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) + require.Equal(topDir, l1.Root()) + + l2, err := l1.New("github.com/someOrg/someRepo/foo/base") + require.NoError(err) + repo := l2.Repo() + require.Equal(cloneRoot, repo) + require.Equal(cloneRoot+"/foo/base", l2.Root()) +} + +func TestRepoDirectCycleDetection(t *testing.T) { + require := require.New(t) + + topDir := "/cycles" + cloneRoot := topDir + "/someClone" + fSys := filesys.MakeFsInMemory() + fSys.MkdirAll(topDir) + fSys.MkdirAll(cloneRoot) + + l1 := newLoaderAtConfirmedDir( + RestrictionRootOnly, filesys.ConfirmedDir(topDir), fSys, nil, + git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) + p1 := "github.com/someOrg/someRepo/foo" + rs1, err := git.NewRepoSpecFromURL(p1) + require.NoError(err) + + l1.repoSpec = rs1 + _, err = l1.New(p1) + require.Error(err) + require.Contains(err.Error(), "cycle detected") +} + +func TestRepoIndirectCycleDetection(t *testing.T) { + require := require.New(t) + + topDir := "/cycles" + cloneRoot := topDir + "/someClone" + fSys := filesys.MakeFsInMemory() + fSys.MkdirAll(topDir) + fSys.MkdirAll(cloneRoot) + + l0 := newLoaderAtConfirmedDir( + RestrictionRootOnly, filesys.ConfirmedDir(topDir), fSys, nil, + git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot))) + + p1 := "github.com/someOrg/someRepo1" + p2 := "github.com/someOrg/someRepo2" + + l1, err := l0.New(p1) + require.NoError(err) + + l2, err := l1.New(p2) + require.NoError(err) + + _, err = l2.New(p1) + require.Error(err) + require.Contains(err.Error(), "cycle detected") +} + +// Inspired by https://hassansin.github.io/Unit-Testing-http-client-in-Go +type fakeRoundTripper func(req *http.Request) *http.Response + +func (f fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func makeFakeHTTPClient(fn fakeRoundTripper) *http.Client { + return &http.Client{ + Transport: fn, + } +} + +// TestLoaderHTTP test http file loader +func TestLoaderHTTP(t *testing.T) { + require := require.New(t) + + var testCasesFile = []testData{ + { + path: "http/file.yaml", + expectedContent: "file content", + }, + } + + l1 := NewLoaderOrDie( + RestrictionRootOnly, MakeFakeFs(testCasesFile), filesys.Separator) + require.Equal("/", l1.Root()) + + for _, x := range testCasesFile { + b, err := l1.Load(x.path) + require.NoError(err) + if !reflect.DeepEqual([]byte(x.expectedContent), b) { + t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) + } + } + + var testCasesHTTP = []testData{ + { + path: "http://example.com/resource.yaml", + expectedContent: "http content", + }, + { + path: "https://example.com/resource.yaml", + expectedContent: "https content", + }, + } + + for _, x := range testCasesHTTP { + hc := makeFakeHTTPClient(func(req *http.Request) *http.Response { + u := req.URL.String() + require.Equal(x.path, u) + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(x.expectedContent)), + Header: make(http.Header), + } + }) + l2 := l1 + l2.http = hc + b, err := l2.Load(x.path) + require.NoError(err) + if !reflect.DeepEqual([]byte(x.expectedContent), b) { + t.Fatalf("in load expected %s, but got %s", x.expectedContent, b) + } + } + + var testCaseUnsupported = []testData{ + { + path: "httpsnotreal://example.com/resource.yaml", + expectedContent: "invalid", + }, + } + for _, x := range testCaseUnsupported { + hc := makeFakeHTTPClient(func(req *http.Request) *http.Response { + t.Fatalf("unexpected request to URL %s", req.URL.String()) + return nil + }) + l2 := l1 + l2.http = hc + _, err := l2.Load(x.path) + require.Error(err) + } + + var testCaseRedirect = []testData{ + { + path: "https://example.com/resource.yaml", + expectedContent: "https content", + }, + } + for _, x := range testCaseRedirect { + expectedLocation := "https://redirect.com/resource.yaml" + hc := makeFakeHTTPClient(func(req *http.Request) *http.Response { + response := &http.Response{ + StatusCode: 300, + Body: io.NopCloser(bytes.NewBufferString("")), + Header: make(http.Header), + } + response.Header.Add("Location", expectedLocation) + return response + }) + l2 := l1 + l2.http = hc + _, err := l2.Load(x.path) + require.Error(err) + var redErr *loader.RedirectionError + var path string = "" + if errors.As(err, &redErr) { + path = redErr.NewPath + } + require.Equal(expectedLocation, path) + } +} + +// setupOnDisk sets up a file system on disk and directory that is cleaned after +// test completion. +// TODO(annasong): Move all loader tests that require real file system into +// api/krusty. +func setupOnDisk(t *testing.T) (filesys.FileSystem, filesys.ConfirmedDir) { + t.Helper() + + fSys := filesys.MakeFsOnDisk() + dir, err := filesys.NewTmpConfirmedDir() + require.NoError(t, err) + t.Cleanup(func() { + _ = fSys.RemoveAll(dir.String()) + }) + return fSys, dir +} diff --git a/api/internal/localizer/localizer.go b/api/internal/localizer/localizer.go index b0e50fc181..a1c7f94c36 100644 --- a/api/internal/localizer/localizer.go +++ b/api/internal/localizer/localizer.go @@ -1,618 +1,618 @@ -// Copyright 2022 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package localizer - -import ( - "io/fs" - "log" - "os" - "path/filepath" - - "sigs.k8s.io/kustomize/api/ifc" - "sigs.k8s.io/kustomize/api/internal/generators" - "sigs.k8s.io/kustomize/api/internal/loader" - "sigs.k8s.io/kustomize/api/internal/target" - "sigs.k8s.io/kustomize/api/provider" - "sigs.k8s.io/kustomize/api/resmap" - "sigs.k8s.io/kustomize/api/types" - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/filesys" - "sigs.k8s.io/yaml" -) - -// localizer encapsulates all state needed to localize the root at ldr. -type localizer struct { - fSys filesys.FileSystem - - // underlying type is Loader - ldr ifc.Loader - - // root is at ldr.Root() - root filesys.ConfirmedDir - - rFactory *resmap.Factory - - // destination directory in newDir that mirrors root - dst string -} - -// Run attempts to localize the kustomization root at target with the given localize arguments -// and returns the path to the created newDir. -func Run(target, scope, newDir string, fSys filesys.FileSystem) (string, error) { - ldr, args, err := NewLoader(target, scope, newDir, fSys) - if err != nil { - return "", errors.Wrap(err) - } - defer func() { _ = ldr.Cleanup() }() - - toDst, err := filepath.Rel(args.Scope.String(), args.Target.String()) - if err != nil { - log.Panicf("cannot find path from %q to child directory %q: %s", args.Scope, args.Target, err) - } - dst := args.NewDir.Join(toDst) - if err = fSys.MkdirAll(dst); err != nil { - return "", errors.WrapPrefixf(err, "unable to create directory in localize destination") - } - - err = (&localizer{ - fSys: fSys, - ldr: ldr, - root: args.Target, - rFactory: resmap.NewFactory(provider.NewDepProvider().GetResourceFactory()), - dst: dst, - }).localize() - if err != nil { - errCleanup := fSys.RemoveAll(args.NewDir.String()) - if errCleanup != nil { - log.Printf("unable to clean localize destination: %s", errCleanup) - } - return "", errors.WrapPrefixf(err, "unable to localize target %q", target) - } - return args.NewDir.String(), nil -} - -// localize localizes the root that lc is at -func (lc *localizer) localize() error { - kustomization, kustFileName, err := lc.load() - if err != nil { - return err - } - err = lc.localizeNativeFields(kustomization) - if err != nil { - return err - } - err = lc.localizeBuiltinPlugins(kustomization) - if err != nil { - return err - } - - content, err := yaml.Marshal(kustomization) - if err != nil { - return errors.WrapPrefixf(err, "unable to serialize localized kustomization file") - } - if err = lc.fSys.WriteFile(filepath.Join(lc.dst, kustFileName), content); err != nil { - return errors.WrapPrefixf(err, "unable to write localized kustomization file") - } - return nil -} - -// load returns the kustomization at lc.root and the file name under which it was found -func (lc *localizer) load() (*types.Kustomization, string, error) { - content, kustFileName, err := target.LoadKustFile(lc.ldr) - if err != nil { - return nil, "", errors.Wrap(err) - } - - var kust types.Kustomization - err = (&kust).Unmarshal(content) - if err != nil { - return nil, "", errors.Wrap(err) - } - - // Localize intentionally does not replace legacy fields to return a localized kustomization - // with as much resemblance to the original as possible. - // Localize also intentionally does not enforce fields, as localize does not wish to unnecessarily - // repeat the responsibilities of kustomize build. - - return &kust, kustFileName, nil -} - -// localizeNativeFields localizes paths on kustomize-native fields, like configMapGenerator, that kustomize has a -// built-in understanding of. This excludes helm-related fields, such as `helmGlobals` and `helmCharts`. -func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error { - if path, exists := kust.OpenAPI["path"]; exists { - locPath, err := lc.localizeFile(path) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize openapi path") - } - kust.OpenAPI["path"] = locPath - } - - for fieldName, field := range map[string]struct { - paths []string - locFn func(string) (string, error) - }{ - "bases": { - // Allow use of deprecated field - //nolint:staticcheck - kust.Bases, - lc.localizeRoot, - }, - "components": { - kust.Components, - lc.localizeRoot, - }, - "configurations": { - kust.Configurations, - lc.localizeFile, - }, - "crds": { - kust.Crds, - lc.localizeFile, - }, - "resources": { - kust.Resources, - lc.localizeResource, - }, - } { - for i, path := range field.paths { - locPath, err := field.locFn(path) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize %s entry", fieldName) - } - field.paths[i] = locPath - } - } - - for i := range kust.ConfigMapGenerator { - if err := lc.localizeGenerator(&kust.ConfigMapGenerator[i].GeneratorArgs); err != nil { - return errors.WrapPrefixf(err, "unable to localize configMapGenerator") - } - } - for i := range kust.SecretGenerator { - if err := lc.localizeGenerator(&kust.SecretGenerator[i].GeneratorArgs); err != nil { - return errors.WrapPrefixf(err, "unable to localize secretGenerator") - } - } - if err := lc.localizeHelmInflationGenerator(kust); err != nil { - return err - } - if err := lc.localizeHelmCharts(kust); err != nil { - return err - } - if err := lc.localizePatches(kust.Patches); err != nil { - return errors.WrapPrefixf(err, "unable to localize patches") - } - //nolint:staticcheck - if err := lc.localizePatches(kust.PatchesJson6902); err != nil { - return errors.WrapPrefixf(err, "unable to localize patchesJson6902") - } - //nolint:staticcheck - for i, patch := range kust.PatchesStrategicMerge { - locPath, err := lc.localizeK8sResource(string(patch)) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize patchesStrategicMerge entry") - } - kust.PatchesStrategicMerge[i] = types.PatchStrategicMerge(locPath) - } - for i, replacement := range kust.Replacements { - locPath, err := lc.localizeFile(replacement.Path) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize replacements entry") - } - kust.Replacements[i].Path = locPath - } - return nil -} - -// localizeGenerator localizes the file paths on generator. -func (lc *localizer) localizeGenerator(generator *types.GeneratorArgs) error { - locEnvSrc, err := lc.localizeFile(generator.EnvSource) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize generator env file") - } - locEnvs := make([]string, len(generator.EnvSources)) - for i, env := range generator.EnvSources { - locEnvs[i], err = lc.localizeFile(env) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize generator envs file") - } - } - locFiles := make([]string, len(generator.FileSources)) - for i, file := range generator.FileSources { - locFiles[i], err = lc.localizeFileSource(file) - if err != nil { - return err - } - } - generator.EnvSource = locEnvSrc - generator.EnvSources = locEnvs - generator.FileSources = locFiles - return nil -} - -// localizeFileSource returns the localized file source found in configMap and -// secretGenerators. -func (lc *localizer) localizeFileSource(source string) (string, error) { - key, file, err := generators.ParseFileSource(source) - if err != nil { - return "", errors.Wrap(err) - } - locFile, err := lc.localizeFile(file) - if err != nil { - return "", errors.WrapPrefixf(err, "invalid file source %q", source) - } - var locSource string - if source == file { - locSource = locFile - } else { - locSource = key + "=" + locFile - } - return locSource, nil -} - -// localizeHelmInflationGenerator localizes helmChartInflationGenerator on kust. -// localizeHelmInflationGenerator localizes values files and copies local chart homes. -func (lc *localizer) localizeHelmInflationGenerator(kust *types.Kustomization) error { - for i, chart := range kust.HelmChartInflationGenerator { - locFile, err := lc.localizeFile(chart.Values) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize helmChartInflationGenerator entry %d values", i) - } - kust.HelmChartInflationGenerator[i].Values = locFile - - locDir, err := lc.copyChartHomeEntry(chart.ChartHome) - if err != nil { - return errors.WrapPrefixf(err, "unable to copy helmChartInflationGenerator entry %d", i) - } - kust.HelmChartInflationGenerator[i].ChartHome = locDir - } - return nil -} - -// localizeHelmCharts localizes helmCharts and helmGlobals on kust. -// localizeHelmCharts localizes values files and copies a local chart home. -func (lc *localizer) localizeHelmCharts(kust *types.Kustomization) error { - for i, chart := range kust.HelmCharts { - locFile, err := lc.localizeFile(chart.ValuesFile) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize helmCharts entry %d valuesFile", i) - } - kust.HelmCharts[i].ValuesFile = locFile - - for j, valuesFile := range chart.AdditionalValuesFiles { - locFile, err = lc.localizeFile(valuesFile) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize helmCharts entry %d additionalValuesFiles", i) - } - kust.HelmCharts[i].AdditionalValuesFiles[j] = locFile - } - } - if kust.HelmGlobals != nil { - locDir, err := lc.copyChartHomeEntry(kust.HelmGlobals.ChartHome) - if err != nil { - return errors.WrapPrefixf(err, "unable to copy helmGlobals") - } - kust.HelmGlobals.ChartHome = locDir - } else if len(kust.HelmCharts) > 0 { - _, err := lc.copyChartHomeEntry("") - if err != nil { - return errors.WrapPrefixf(err, "unable to copy default chart home") - } - } - return nil -} - -// localizePatches localizes the file paths on patches if they are non-empty -func (lc *localizer) localizePatches(patches []types.Patch) error { - for i := range patches { - locPath, err := lc.localizeFile(patches[i].Path) - if err != nil { - return err - } - patches[i].Path = locPath - } - return nil -} - -// localizeResource localizes resource path, a file or root, and returns the -// localized path -func (lc *localizer) localizeResource(path string) (string, error) { - var locPath string - - content, fileErr := lc.ldr.Load(path) - // The following check catches the case where path is a repo root. - // Load on a repo will successfully return its README in HTML. - // Because HTML does not follow resource formatting, we then correctly try - // to localize path as a root. - if fileErr == nil { - _, resErr := lc.rFactory.NewResMapFromBytes(content) - if resErr != nil { - fileErr = errors.WrapPrefixf(resErr, "invalid resource at file %q", path) - } else { - locPath, fileErr = lc.localizeFileWithContent(path, content) - } - } - if fileErr != nil { - var redErr *loader.RedirectionError - if errors.As(fileErr, &redErr) { - path = redErr.NewPath - } - - var rootErr error - locPath, rootErr = lc.localizeRoot(path) - if rootErr != nil { - err := PathLocalizeError{ - Path: path, - FileError: fileErr, - RootError: rootErr, - } - return "", err - } - } - return locPath, nil -} - -// localizeFile localizes file path if set and returns the localized path -func (lc *localizer) localizeFile(path string) (string, error) { - // Some localizable fields can be empty, for example, replacements.path. - // We rely on the build command to throw errors for the ones that cannot. - if path == "" { - return "", nil - } - content, err := lc.ldr.Load(path) - if err != nil { - return "", errors.Wrap(err) - } - return lc.localizeFileWithContent(path, content) -} - -// localizeFileWithContent writes content to the localized file path and returns the localized path. -func (lc *localizer) localizeFileWithContent(path string, content []byte) (string, error) { - var locPath string - if loader.IsRemoteFile(path) { - if lc.fSys.Exists(lc.root.Join(LocalizeDir)) { - return "", errors.Errorf("%s already contains %s needed to store file %q", lc.root, LocalizeDir, path) - } - locPath = locFilePath(path) - } else { - // ldr has checked that path must be relative; this is subject to change in beta. - - // We must clean path to: - // 1. avoid symlinks. A `kustomize build` run will fail if we write files to - // symlink paths outside the current root, given that we don't want to recreate - // the symlinks. Even worse, we could be writing files outside the localize destination. - // 2. avoid paths that temporarily traverse outside the current root, - // i.e. ../../../scope/target/current-root. The localized file will be surrounded by - // different directories than its source, and so an uncleaned path may no longer be valid. - locPath = cleanFilePath(lc.fSys, lc.root, path) - } - absPath := filepath.Join(lc.dst, locPath) - if err := lc.fSys.MkdirAll(filepath.Dir(absPath)); err != nil { - return "", errors.WrapPrefixf(err, "unable to create directories to localize file %q", path) - } - if err := lc.fSys.WriteFile(absPath, content); err != nil { - return "", errors.WrapPrefixf(err, "unable to localize file %q", path) - } - return locPath, nil -} - -// localizeRoot localizes root path if set and returns the localized path -func (lc *localizer) localizeRoot(path string) (string, error) { - if path == "" { - return "", nil - } - ldr, err := lc.ldr.New(path) - if err != nil { - return "", errors.Wrap(err) - } - defer func() { _ = ldr.Cleanup() }() - - root, err := filesys.ConfirmDir(lc.fSys, ldr.Root()) - if err != nil { - log.Panicf("unable to establish validated root reference %q: %s", path, err) - } - var locPath string - if repo := ldr.Repo(); repo != "" { - if lc.fSys.Exists(lc.root.Join(LocalizeDir)) { - return "", errors.Errorf("%s already contains %s needed to store root %q", lc.root, LocalizeDir, path) - } - locPath, err = locRootPath(path, repo, root, lc.fSys) - if err != nil { - return "", err - } - } else { - locPath, err = filepath.Rel(lc.root.String(), root.String()) - if err != nil { - log.Panicf("cannot find relative path between scoped localize roots %q and %q: %s", lc.root, root, err) - } - } - newDst := filepath.Join(lc.dst, locPath) - if err = lc.fSys.MkdirAll(newDst); err != nil { - return "", errors.WrapPrefixf(err, "unable to create root %q in localize destination", path) - } - err = (&localizer{ - fSys: lc.fSys, - ldr: ldr, - root: root, - rFactory: lc.rFactory, - dst: newDst, - }).localize() - if err != nil { - return "", errors.WrapPrefixf(err, "unable to localize root %q", path) - } - return locPath, nil -} - -// copyChartHomeEntry copies the helm chart home entry to lc dst -// at the same location relative to the root and returns said relative path. -// If entry is empty, copyChartHomeEntry returns the empty string. -// If entry does not exist, copyChartHome returns entry. -// -// copyChartHomeEntry copies the default home to the same location at dst, -// without following symlinks. An empty entry also indicates the default home. -func (lc *localizer) copyChartHomeEntry(entry string) (string, error) { - path := entry - if entry == "" { - path = types.HelmDefaultHome - } - if filepath.IsAbs(path) { - return "", errors.Errorf("absolute path %q not handled in alpha", path) - } - isDefault := lc.root.Join(path) == lc.root.Join(types.HelmDefaultHome) - locPath, err := lc.copyChartHome(path, !isDefault) - if err != nil { - return "", errors.WrapPrefixf(err, "unable to copy home %q", entry) - } - if entry == "" { - return "", nil - } - return locPath, nil -} - -// copyChartHome copies path relative to lc root to dst and returns the -// copied location relative to dst. If clean is true, copyChartHome uses path's -// delinked location as the copy destination. -// -// If path does not exist, copyChartHome returns path. -func (lc *localizer) copyChartHome(path string, clean bool) (string, error) { - path, err := filepath.Rel(lc.root.String(), lc.root.Join(path)) - if err != nil { - return "", errors.WrapPrefixf(err, "no path to chart home %q", path) - } - // Chart home may serve as untar destination. - // Note that we don't check if path is in scope. - if !lc.fSys.Exists(lc.root.Join(path)) { - return path, nil - } - // Perform localize directory checks. - ldr, err := lc.ldr.New(path) - if err != nil { - return "", errors.WrapPrefixf(err, "invalid chart home") - } - cleaned, err := filesys.ConfirmDir(lc.fSys, ldr.Root()) - if err != nil { - log.Panicf("unable to confirm validated directory %q: %s", ldr.Root(), err) - } - toDst := path - if clean { - toDst, err = filepath.Rel(lc.root.String(), cleaned.String()) - if err != nil { - log.Panicf("no path between scoped directories %q and %q: %s", lc.root, cleaned, err) - } - } - // Note this check does not guarantee that we copied the entire directory. - if dst := filepath.Join(lc.dst, toDst); !lc.fSys.Exists(dst) { - err = lc.copyDir(cleaned, filepath.Join(lc.dst, toDst)) - if err != nil { - return "", errors.WrapPrefixf(err, "unable to copy chart home %q", path) - } - } - return toDst, nil -} - -// copyDir copies src to dst. copyDir does not follow symlinks. -func (lc *localizer) copyDir(src filesys.ConfirmedDir, dst string) error { - err := lc.fSys.Walk(src.String(), - func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - pathToCreate, err := filepath.Rel(src.String(), path) - if err != nil { - log.Panicf("no path from %q to child file %q: %s", src, path, err) - } - pathInDst := filepath.Join(dst, pathToCreate) - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - return nil - } - if info.IsDir() { - err = lc.fSys.MkdirAll(pathInDst) - } else { - var content []byte - content, err = lc.fSys.ReadFile(path) - if err != nil { - return errors.Wrap(err) - } - err = lc.fSys.WriteFile(pathInDst, content) - } - return errors.Wrap(err) - }) - if err != nil { - return errors.WrapPrefixf(err, "unable to copy directory %q", src) - } - return nil -} - -// localizeBuiltinPlugins localizes built-in plugins on kust that can contain file paths. The built-in plugins -// can be inline or in a file. This excludes the HelmChartInflationGenerator. -// -// Note that the localization in this function has not been implemented yet. -func (lc *localizer) localizeBuiltinPlugins(kust *types.Kustomization) error { - for fieldName, entries := range map[string][]string{ - "generators": kust.Generators, - "transformers": kust.Transformers, - "validators": kust.Validators, - } { - for i, entry := range entries { - rm, isPath, err := lc.loadK8sResource(entry) - if err != nil { - return errors.WrapPrefixf(err, "unable to load %s entry", fieldName) - } - err = rm.ApplyFilter(&localizeBuiltinPlugins{lc: lc}) - if err != nil { - return errors.Wrap(err) - } - localizedPlugin, err := rm.AsYaml() - if err != nil { - return errors.WrapPrefixf(err, "unable to serialize localized %s entry %q", fieldName, entry) - } - var localizedEntry string - if isPath { - localizedEntry, err = lc.localizeFileWithContent(entry, localizedPlugin) - if err != nil { - return errors.WrapPrefixf(err, "unable to localize %s entry", fieldName) - } - } else { - localizedEntry = string(localizedPlugin) - } - entries[i] = localizedEntry - } - } - return nil -} - -// localizeK8sResource returns the localized resourceEntry if it is a file -// containing a kubernetes resource. -// localizeK8sResource returns resourceEntry if it is an inline resource. -func (lc *localizer) localizeK8sResource(resourceEntry string) (string, error) { - _, isFile, err := lc.loadK8sResource(resourceEntry) - if err != nil { - return "", err - } - if isFile { - return lc.localizeFile(resourceEntry) - } - return resourceEntry, nil -} - -// loadK8sResource tries to load resourceEntry as a file path or inline -// kubernetes resource. -// On success, loadK8sResource returns the loaded resource map and whether -// resourceEntry is a file path. -func (lc *localizer) loadK8sResource(resourceEntry string) (resmap.ResMap, bool, error) { - rm, inlineErr := lc.rFactory.NewResMapFromBytes([]byte(resourceEntry)) - if inlineErr != nil { - var fileErr error - rm, fileErr = lc.rFactory.FromFile(lc.ldr, resourceEntry) - if fileErr != nil { - err := ResourceLoadError{ - InlineError: inlineErr, - FileError: fileErr, - } - return nil, false, errors.WrapPrefixf(err, "unable to load resource entry %q", resourceEntry) - } - } - return rm, inlineErr != nil, nil -} +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package localizer + +import ( + "io/fs" + "log" + "os" + "path/filepath" + + "sigs.k8s.io/kustomize/api/ifc" + "sigs.k8s.io/kustomize/api/internal/generators" + "sigs.k8s.io/kustomize/api/internal/loader" + "sigs.k8s.io/kustomize/api/internal/target" + "sigs.k8s.io/kustomize/api/provider" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/yaml" +) + +// localizer encapsulates all state needed to localize the root at ldr. +type localizer struct { + fSys filesys.FileSystem + + // underlying type is Loader + ldr ifc.Loader + + // root is at ldr.Root() + root filesys.ConfirmedDir + + rFactory *resmap.Factory + + // destination directory in newDir that mirrors root + dst string +} + +// Run attempts to localize the kustomization root at target with the given localize arguments +// and returns the path to the created newDir. +func Run(target, scope, newDir string, fSys filesys.FileSystem) (string, error) { + ldr, args, err := NewLoader(target, scope, newDir, fSys) + if err != nil { + return "", errors.Wrap(err) + } + defer func() { _ = ldr.Cleanup() }() + + toDst, err := filepath.Rel(args.Scope.String(), args.Target.String()) + if err != nil { + log.Panicf("cannot find path from %q to child directory %q: %s", args.Scope, args.Target, err) + } + dst := args.NewDir.Join(toDst) + if err = fSys.MkdirAll(dst); err != nil { + return "", errors.WrapPrefixf(err, "unable to create directory in localize destination") + } + + err = (&localizer{ + fSys: fSys, + ldr: ldr, + root: args.Target, + rFactory: resmap.NewFactory(provider.NewDepProvider().GetResourceFactory()), + dst: dst, + }).localize() + if err != nil { + errCleanup := fSys.RemoveAll(args.NewDir.String()) + if errCleanup != nil { + log.Printf("unable to clean localize destination: %s", errCleanup) + } + return "", errors.WrapPrefixf(err, "unable to localize target %q", target) + } + return args.NewDir.String(), nil +} + +// localize localizes the root that lc is at +func (lc *localizer) localize() error { + kustomization, kustFileName, err := lc.load() + if err != nil { + return err + } + err = lc.localizeNativeFields(kustomization) + if err != nil { + return err + } + err = lc.localizeBuiltinPlugins(kustomization) + if err != nil { + return err + } + + content, err := yaml.Marshal(kustomization) + if err != nil { + return errors.WrapPrefixf(err, "unable to serialize localized kustomization file") + } + if err = lc.fSys.WriteFile(filepath.Join(lc.dst, kustFileName), content); err != nil { + return errors.WrapPrefixf(err, "unable to write localized kustomization file") + } + return nil +} + +// load returns the kustomization at lc.root and the file name under which it was found +func (lc *localizer) load() (*types.Kustomization, string, error) { + content, kustFileName, err := target.LoadKustFile(lc.ldr) + if err != nil { + return nil, "", errors.Wrap(err) + } + + var kust types.Kustomization + err = (&kust).Unmarshal(content) + if err != nil { + return nil, "", errors.Wrap(err) + } + + // Localize intentionally does not replace legacy fields to return a localized kustomization + // with as much resemblance to the original as possible. + // Localize also intentionally does not enforce fields, as localize does not wish to unnecessarily + // repeat the responsibilities of kustomize build. + + return &kust, kustFileName, nil +} + +// localizeNativeFields localizes paths on kustomize-native fields, like configMapGenerator, that kustomize has a +// built-in understanding of. This excludes helm-related fields, such as `helmGlobals` and `helmCharts`. +func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error { + if path, exists := kust.OpenAPI["path"]; exists { + locPath, err := lc.localizeFile(path) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize openapi path") + } + kust.OpenAPI["path"] = locPath + } + + for fieldName, field := range map[string]struct { + paths []string + locFn func(string) (string, error) + }{ + "bases": { + // Allow use of deprecated field + //nolint:staticcheck + kust.Bases, + lc.localizeRoot, + }, + "components": { + kust.Components, + lc.localizeRoot, + }, + "configurations": { + kust.Configurations, + lc.localizeFile, + }, + "crds": { + kust.Crds, + lc.localizeFile, + }, + "resources": { + kust.Resources, + lc.localizeResource, + }, + } { + for i, path := range field.paths { + locPath, err := field.locFn(path) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize %s entry", fieldName) + } + field.paths[i] = locPath + } + } + + for i := range kust.ConfigMapGenerator { + if err := lc.localizeGenerator(&kust.ConfigMapGenerator[i].GeneratorArgs); err != nil { + return errors.WrapPrefixf(err, "unable to localize configMapGenerator") + } + } + for i := range kust.SecretGenerator { + if err := lc.localizeGenerator(&kust.SecretGenerator[i].GeneratorArgs); err != nil { + return errors.WrapPrefixf(err, "unable to localize secretGenerator") + } + } + if err := lc.localizeHelmInflationGenerator(kust); err != nil { + return err + } + if err := lc.localizeHelmCharts(kust); err != nil { + return err + } + if err := lc.localizePatches(kust.Patches); err != nil { + return errors.WrapPrefixf(err, "unable to localize patches") + } + //nolint:staticcheck + if err := lc.localizePatches(kust.PatchesJson6902); err != nil { + return errors.WrapPrefixf(err, "unable to localize patchesJson6902") + } + //nolint:staticcheck + for i, patch := range kust.PatchesStrategicMerge { + locPath, err := lc.localizeK8sResource(string(patch)) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize patchesStrategicMerge entry") + } + kust.PatchesStrategicMerge[i] = types.PatchStrategicMerge(locPath) + } + for i, replacement := range kust.Replacements { + locPath, err := lc.localizeFile(replacement.Path) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize replacements entry") + } + kust.Replacements[i].Path = locPath + } + return nil +} + +// localizeGenerator localizes the file paths on generator. +func (lc *localizer) localizeGenerator(generator *types.GeneratorArgs) error { + locEnvSrc, err := lc.localizeFile(generator.EnvSource) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize generator env file") + } + locEnvs := make([]string, len(generator.EnvSources)) + for i, env := range generator.EnvSources { + locEnvs[i], err = lc.localizeFile(env) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize generator envs file") + } + } + locFiles := make([]string, len(generator.FileSources)) + for i, file := range generator.FileSources { + locFiles[i], err = lc.localizeFileSource(file) + if err != nil { + return err + } + } + generator.EnvSource = locEnvSrc + generator.EnvSources = locEnvs + generator.FileSources = locFiles + return nil +} + +// localizeFileSource returns the localized file source found in configMap and +// secretGenerators. +func (lc *localizer) localizeFileSource(source string) (string, error) { + key, file, err := generators.ParseFileSource(source) + if err != nil { + return "", errors.Wrap(err) + } + locFile, err := lc.localizeFile(file) + if err != nil { + return "", errors.WrapPrefixf(err, "invalid file source %q", source) + } + var locSource string + if source == file { + locSource = locFile + } else { + locSource = key + "=" + locFile + } + return locSource, nil +} + +// localizeHelmInflationGenerator localizes helmChartInflationGenerator on kust. +// localizeHelmInflationGenerator localizes values files and copies local chart homes. +func (lc *localizer) localizeHelmInflationGenerator(kust *types.Kustomization) error { + for i, chart := range kust.HelmChartInflationGenerator { + locFile, err := lc.localizeFile(chart.Values) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize helmChartInflationGenerator entry %d values", i) + } + kust.HelmChartInflationGenerator[i].Values = locFile + + locDir, err := lc.copyChartHomeEntry(chart.ChartHome) + if err != nil { + return errors.WrapPrefixf(err, "unable to copy helmChartInflationGenerator entry %d", i) + } + kust.HelmChartInflationGenerator[i].ChartHome = locDir + } + return nil +} + +// localizeHelmCharts localizes helmCharts and helmGlobals on kust. +// localizeHelmCharts localizes values files and copies a local chart home. +func (lc *localizer) localizeHelmCharts(kust *types.Kustomization) error { + for i, chart := range kust.HelmCharts { + locFile, err := lc.localizeFile(chart.ValuesFile) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize helmCharts entry %d valuesFile", i) + } + kust.HelmCharts[i].ValuesFile = locFile + + for j, valuesFile := range chart.AdditionalValuesFiles { + locFile, err = lc.localizeFile(valuesFile) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize helmCharts entry %d additionalValuesFiles", i) + } + kust.HelmCharts[i].AdditionalValuesFiles[j] = locFile + } + } + if kust.HelmGlobals != nil { + locDir, err := lc.copyChartHomeEntry(kust.HelmGlobals.ChartHome) + if err != nil { + return errors.WrapPrefixf(err, "unable to copy helmGlobals") + } + kust.HelmGlobals.ChartHome = locDir + } else if len(kust.HelmCharts) > 0 { + _, err := lc.copyChartHomeEntry("") + if err != nil { + return errors.WrapPrefixf(err, "unable to copy default chart home") + } + } + return nil +} + +// localizePatches localizes the file paths on patches if they are non-empty +func (lc *localizer) localizePatches(patches []types.Patch) error { + for i := range patches { + locPath, err := lc.localizeFile(patches[i].Path) + if err != nil { + return err + } + patches[i].Path = locPath + } + return nil +} + +// localizeResource localizes resource path, a file or root, and returns the +// localized path +func (lc *localizer) localizeResource(path string) (string, error) { + var locPath string + + content, fileErr := lc.ldr.Load(path) + // The following check catches the case where path is a repo root. + // Load on a repo will successfully return its README in HTML. + // Because HTML does not follow resource formatting, we then correctly try + // to localize path as a root. + if fileErr == nil { + _, resErr := lc.rFactory.NewResMapFromBytes(content) + if resErr != nil { + fileErr = errors.WrapPrefixf(resErr, "invalid resource at file %q", path) + } else { + locPath, fileErr = lc.localizeFileWithContent(path, content) + } + } + if fileErr != nil { + var redErr *loader.RedirectionError + if errors.As(fileErr, &redErr) { + path = redErr.NewPath + } + + var rootErr error + locPath, rootErr = lc.localizeRoot(path) + if rootErr != nil { + err := PathLocalizeError{ + Path: path, + FileError: fileErr, + RootError: rootErr, + } + return "", err + } + } + return locPath, nil +} + +// localizeFile localizes file path if set and returns the localized path +func (lc *localizer) localizeFile(path string) (string, error) { + // Some localizable fields can be empty, for example, replacements.path. + // We rely on the build command to throw errors for the ones that cannot. + if path == "" { + return "", nil + } + content, err := lc.ldr.Load(path) + if err != nil { + return "", errors.Wrap(err) + } + return lc.localizeFileWithContent(path, content) +} + +// localizeFileWithContent writes content to the localized file path and returns the localized path. +func (lc *localizer) localizeFileWithContent(path string, content []byte) (string, error) { + var locPath string + if loader.IsRemoteFile(path) { + if lc.fSys.Exists(lc.root.Join(LocalizeDir)) { + return "", errors.Errorf("%s already contains %s needed to store file %q", lc.root, LocalizeDir, path) + } + locPath = locFilePath(path) + } else { + // ldr has checked that path must be relative; this is subject to change in beta. + + // We must clean path to: + // 1. avoid symlinks. A `kustomize build` run will fail if we write files to + // symlink paths outside the current root, given that we don't want to recreate + // the symlinks. Even worse, we could be writing files outside the localize destination. + // 2. avoid paths that temporarily traverse outside the current root, + // i.e. ../../../scope/target/current-root. The localized file will be surrounded by + // different directories than its source, and so an uncleaned path may no longer be valid. + locPath = cleanFilePath(lc.fSys, lc.root, path) + } + absPath := filepath.Join(lc.dst, locPath) + if err := lc.fSys.MkdirAll(filepath.Dir(absPath)); err != nil { + return "", errors.WrapPrefixf(err, "unable to create directories to localize file %q", path) + } + if err := lc.fSys.WriteFile(absPath, content); err != nil { + return "", errors.WrapPrefixf(err, "unable to localize file %q", path) + } + return locPath, nil +} + +// localizeRoot localizes root path if set and returns the localized path +func (lc *localizer) localizeRoot(path string) (string, error) { + if path == "" { + return "", nil + } + ldr, err := lc.ldr.New(path) + if err != nil { + return "", errors.Wrap(err) + } + defer func() { _ = ldr.Cleanup() }() + + root, err := filesys.ConfirmDir(lc.fSys, ldr.Root()) + if err != nil { + log.Panicf("unable to establish validated root reference %q: %s", path, err) + } + var locPath string + if repo := ldr.Repo(); repo != "" { + if lc.fSys.Exists(lc.root.Join(LocalizeDir)) { + return "", errors.Errorf("%s already contains %s needed to store root %q", lc.root, LocalizeDir, path) + } + locPath, err = locRootPath(path, repo, root, lc.fSys) + if err != nil { + return "", err + } + } else { + locPath, err = filepath.Rel(lc.root.String(), root.String()) + if err != nil { + log.Panicf("cannot find relative path between scoped localize roots %q and %q: %s", lc.root, root, err) + } + } + newDst := filepath.Join(lc.dst, locPath) + if err = lc.fSys.MkdirAll(newDst); err != nil { + return "", errors.WrapPrefixf(err, "unable to create root %q in localize destination", path) + } + err = (&localizer{ + fSys: lc.fSys, + ldr: ldr, + root: root, + rFactory: lc.rFactory, + dst: newDst, + }).localize() + if err != nil { + return "", errors.WrapPrefixf(err, "unable to localize root %q", path) + } + return locPath, nil +} + +// copyChartHomeEntry copies the helm chart home entry to lc dst +// at the same location relative to the root and returns said relative path. +// If entry is empty, copyChartHomeEntry returns the empty string. +// If entry does not exist, copyChartHome returns entry. +// +// copyChartHomeEntry copies the default home to the same location at dst, +// without following symlinks. An empty entry also indicates the default home. +func (lc *localizer) copyChartHomeEntry(entry string) (string, error) { + path := entry + if entry == "" { + path = types.HelmDefaultHome + } + if filepath.IsAbs(path) { + return "", errors.Errorf("absolute path %q not handled in alpha", path) + } + isDefault := lc.root.Join(path) == lc.root.Join(types.HelmDefaultHome) + locPath, err := lc.copyChartHome(path, !isDefault) + if err != nil { + return "", errors.WrapPrefixf(err, "unable to copy home %q", entry) + } + if entry == "" { + return "", nil + } + return locPath, nil +} + +// copyChartHome copies path relative to lc root to dst and returns the +// copied location relative to dst. If clean is true, copyChartHome uses path's +// delinked location as the copy destination. +// +// If path does not exist, copyChartHome returns path. +func (lc *localizer) copyChartHome(path string, clean bool) (string, error) { + path, err := filepath.Rel(lc.root.String(), lc.root.Join(path)) + if err != nil { + return "", errors.WrapPrefixf(err, "no path to chart home %q", path) + } + // Chart home may serve as untar destination. + // Note that we don't check if path is in scope. + if !lc.fSys.Exists(lc.root.Join(path)) { + return path, nil + } + // Perform localize directory checks. + ldr, err := lc.ldr.New(path) + if err != nil { + return "", errors.WrapPrefixf(err, "invalid chart home") + } + cleaned, err := filesys.ConfirmDir(lc.fSys, ldr.Root()) + if err != nil { + log.Panicf("unable to confirm validated directory %q: %s", ldr.Root(), err) + } + toDst := path + if clean { + toDst, err = filepath.Rel(lc.root.String(), cleaned.String()) + if err != nil { + log.Panicf("no path between scoped directories %q and %q: %s", lc.root, cleaned, err) + } + } + // Note this check does not guarantee that we copied the entire directory. + if dst := filepath.Join(lc.dst, toDst); !lc.fSys.Exists(dst) { + err = lc.copyDir(cleaned, filepath.Join(lc.dst, toDst)) + if err != nil { + return "", errors.WrapPrefixf(err, "unable to copy chart home %q", path) + } + } + return toDst, nil +} + +// copyDir copies src to dst. copyDir does not follow symlinks. +func (lc *localizer) copyDir(src filesys.ConfirmedDir, dst string) error { + err := lc.fSys.Walk(src.String(), + func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + pathToCreate, err := filepath.Rel(src.String(), path) + if err != nil { + log.Panicf("no path from %q to child file %q: %s", src, path, err) + } + pathInDst := filepath.Join(dst, pathToCreate) + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + return nil + } + if info.IsDir() { + err = lc.fSys.MkdirAll(pathInDst) + } else { + var content []byte + content, err = lc.fSys.ReadFile(path) + if err != nil { + return errors.Wrap(err) + } + err = lc.fSys.WriteFile(pathInDst, content) + } + return errors.Wrap(err) + }) + if err != nil { + return errors.WrapPrefixf(err, "unable to copy directory %q", src) + } + return nil +} + +// localizeBuiltinPlugins localizes built-in plugins on kust that can contain file paths. The built-in plugins +// can be inline or in a file. This excludes the HelmChartInflationGenerator. +// +// Note that the localization in this function has not been implemented yet. +func (lc *localizer) localizeBuiltinPlugins(kust *types.Kustomization) error { + for fieldName, entries := range map[string][]string{ + "generators": kust.Generators, + "transformers": kust.Transformers, + "validators": kust.Validators, + } { + for i, entry := range entries { + rm, isPath, err := lc.loadK8sResource(entry) + if err != nil { + return errors.WrapPrefixf(err, "unable to load %s entry", fieldName) + } + err = rm.ApplyFilter(&localizeBuiltinPlugins{lc: lc}) + if err != nil { + return errors.Wrap(err) + } + localizedPlugin, err := rm.AsYaml() + if err != nil { + return errors.WrapPrefixf(err, "unable to serialize localized %s entry %q", fieldName, entry) + } + var localizedEntry string + if isPath { + localizedEntry, err = lc.localizeFileWithContent(entry, localizedPlugin) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize %s entry", fieldName) + } + } else { + localizedEntry = string(localizedPlugin) + } + entries[i] = localizedEntry + } + } + return nil +} + +// localizeK8sResource returns the localized resourceEntry if it is a file +// containing a kubernetes resource. +// localizeK8sResource returns resourceEntry if it is an inline resource. +func (lc *localizer) localizeK8sResource(resourceEntry string) (string, error) { + _, isFile, err := lc.loadK8sResource(resourceEntry) + if err != nil { + return "", err + } + if isFile { + return lc.localizeFile(resourceEntry) + } + return resourceEntry, nil +} + +// loadK8sResource tries to load resourceEntry as a file path or inline +// kubernetes resource. +// On success, loadK8sResource returns the loaded resource map and whether +// resourceEntry is a file path. +func (lc *localizer) loadK8sResource(resourceEntry string) (resmap.ResMap, bool, error) { + rm, inlineErr := lc.rFactory.NewResMapFromBytes([]byte(resourceEntry)) + if inlineErr != nil { + var fileErr error + rm, fileErr = lc.rFactory.FromFile(lc.ldr, resourceEntry) + if fileErr != nil { + err := ResourceLoadError{ + InlineError: inlineErr, + FileError: fileErr, + } + return nil, false, errors.WrapPrefixf(err, "unable to load resource entry %q", resourceEntry) + } + } + return rm, inlineErr != nil, nil +} diff --git a/api/internal/target/kusttarget.go b/api/internal/target/kusttarget.go index c95c89daa6..b2821ea6bd 100644 --- a/api/internal/target/kusttarget.go +++ b/api/internal/target/kusttarget.go @@ -1,590 +1,590 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package target - -import ( - "encoding/json" - "fmt" - "os" - "strings" - - "sigs.k8s.io/kustomize/api/ifc" - "sigs.k8s.io/kustomize/api/internal/accumulator" - "sigs.k8s.io/kustomize/api/internal/builtins" - "sigs.k8s.io/kustomize/api/internal/kusterr" - load "sigs.k8s.io/kustomize/api/internal/loader" - "sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig" - "sigs.k8s.io/kustomize/api/internal/plugins/builtinhelpers" - "sigs.k8s.io/kustomize/api/internal/plugins/loader" - "sigs.k8s.io/kustomize/api/internal/utils" - "sigs.k8s.io/kustomize/api/konfig" - "sigs.k8s.io/kustomize/api/resmap" - "sigs.k8s.io/kustomize/api/resource" - "sigs.k8s.io/kustomize/api/types" - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/openapi" - "sigs.k8s.io/yaml" -) - -// KustTarget encapsulates the entirety of a kustomization build. -type KustTarget struct { - kustomization *types.Kustomization - kustFileName string - ldr ifc.Loader - validator ifc.Validator - rFactory *resmap.Factory - pLdr *loader.Loader - origin *resource.Origin -} - -// NewKustTarget returns a new instance of KustTarget. -func NewKustTarget( - ldr ifc.Loader, - validator ifc.Validator, - rFactory *resmap.Factory, - pLdr *loader.Loader) *KustTarget { - return &KustTarget{ - ldr: ldr, - validator: validator, - rFactory: rFactory, - pLdr: pLdr.LoaderWithWorkingDir(ldr.Root()), - } -} - -// Load attempts to load the target's kustomization file. -func (kt *KustTarget) Load() error { - content, kustFileName, err := LoadKustFile(kt.ldr) - if err != nil { - return err - } - - var k types.Kustomization - if err := k.Unmarshal(content); err != nil { - return err - } - - // show warning message when using deprecated fields. - if warningMessages := k.CheckDeprecatedFields(); warningMessages != nil { - for _, msg := range *warningMessages { - fmt.Fprintf(os.Stderr, "%v\n", msg) - } - } - - k.FixKustomization() - - // check that Kustomization is empty - if err := k.CheckEmpty(); err != nil { - return err - } - - errs := k.EnforceFields() - if len(errs) > 0 { - return fmt.Errorf( - "Failed to read kustomization file under %s:\n"+ - strings.Join(errs, "\n"), kt.ldr.Root()) - } - kt.kustomization = &k - kt.kustFileName = kustFileName - return nil -} - -// Kustomization returns a copy of the immutable, internal kustomization object. -func (kt *KustTarget) Kustomization() types.Kustomization { - var result types.Kustomization - b, _ := json.Marshal(*kt.kustomization) - json.Unmarshal(b, &result) - return result -} - -func LoadKustFile(ldr ifc.Loader) ([]byte, string, error) { - var content []byte - match := 0 - var kustFileName string - for _, kf := range konfig.RecognizedKustomizationFileNames() { - c, err := ldr.Load(kf) - if err == nil { - match += 1 - content = c - kustFileName = kf - } - } - switch match { - case 0: - return nil, "", NewErrMissingKustomization(ldr.Root()) - case 1: - return content, kustFileName, nil - default: - return nil, "", fmt.Errorf( - "Found multiple kustomization files under: %s\n", ldr.Root()) - } -} - -// MakeCustomizedResMap creates a fully customized ResMap -// per the instructions contained in its kustomization instance. -func (kt *KustTarget) MakeCustomizedResMap() (resmap.ResMap, error) { - return kt.makeCustomizedResMap() -} - -func (kt *KustTarget) makeCustomizedResMap() (resmap.ResMap, error) { - var origin *resource.Origin - if len(kt.kustomization.BuildMetadata) != 0 { - origin = &resource.Origin{} - } - kt.origin = origin - ra, err := kt.AccumulateTarget() - if err != nil { - return nil, err - } - - // The following steps must be done last, not as part of - // the recursion implicit in AccumulateTarget. - - err = kt.addHashesToNames(ra) - if err != nil { - return nil, err - } - - // Given that names have changed (prefixs/suffixes added), - // fix all the back references to those names. - err = ra.FixBackReferences() - if err != nil { - return nil, err - } - - // With all the back references fixed, it's OK to resolve Vars. - err = ra.ResolveVars() - if err != nil { - return nil, err - } - - err = kt.IgnoreLocal(ra) - if err != nil { - return nil, err - } - - return ra.ResMap(), nil -} - -func (kt *KustTarget) addHashesToNames( - ra *accumulator.ResAccumulator) error { - p := builtins.NewHashTransformerPlugin() - err := kt.configureBuiltinPlugin(p, nil, builtinhelpers.HashTransformer) - if err != nil { - return err - } - return ra.Transform(p) -} - -// AccumulateTarget returns a new ResAccumulator, -// holding customized resources and the data/rules used -// to do so. The name back references and vars are -// not yet fixed. -// The origin parameter is used through the recursive calls -// to annotate each resource with information about where -// the resource came from, e.g. the file and/or the repository -// it originated from. -// As an entrypoint, one can pass an empty resource.Origin object to -// AccumulateTarget. As AccumulateTarget moves recursively -// through kustomization directories, it updates `origin.path` -// accordingly. When a remote base is found, it updates `origin.repo` -// and `origin.ref` accordingly. -func (kt *KustTarget) AccumulateTarget() ( - ra *accumulator.ResAccumulator, err error) { - return kt.accumulateTarget(accumulator.MakeEmptyAccumulator()) -} - -// ra should be empty when this KustTarget is a Kustomization, or the ra of the parent if this KustTarget is a Component -// (or empty if the Component does not have a parent). -func (kt *KustTarget) accumulateTarget(ra *accumulator.ResAccumulator) ( - resRa *accumulator.ResAccumulator, err error) { - ra, err = kt.accumulateResources(ra, kt.kustomization.Resources) - if err != nil { - return nil, errors.WrapPrefixf(err, "accumulating resources") - } - tConfig, err := builtinconfig.MakeTransformerConfig( - kt.ldr, kt.kustomization.Configurations) - if err != nil { - return nil, err - } - err = ra.MergeConfig(tConfig) - if err != nil { - return nil, errors.WrapPrefixf( - err, "merging config %v", tConfig) - } - crdTc, err := accumulator.LoadConfigFromCRDs(kt.ldr, kt.kustomization.Crds) - if err != nil { - return nil, errors.WrapPrefixf( - err, "loading CRDs %v", kt.kustomization.Crds) - } - err = ra.MergeConfig(crdTc) - if err != nil { - return nil, errors.WrapPrefixf( - err, "merging CRDs %v", crdTc) - } - err = kt.runGenerators(ra) - if err != nil { - return nil, err - } - - // components are expected to execute after reading resources and adding generators ,before applying transformers and validation. - // https://github.com/kubernetes-sigs/kustomize/pull/5170#discussion_r1212101287 - ra, err = kt.accumulateComponents(ra, kt.kustomization.Components) - if err != nil { - return nil, errors.WrapPrefixf(err, "accumulating components") - } - - err = kt.runTransformers(ra) - if err != nil { - return nil, err - } - err = kt.runValidators(ra) - if err != nil { - return nil, err - } - err = ra.MergeVars(kt.kustomization.Vars) - if err != nil { - return nil, errors.WrapPrefixf( - err, "merging vars %v", kt.kustomization.Vars) - } - return ra, nil -} - -// IgnoreLocal drops the local resource by checking the annotation "config.kubernetes.io/local-config". -func (kt *KustTarget) IgnoreLocal(ra *accumulator.ResAccumulator) error { - rf := kt.rFactory.RF() - if rf.IncludeLocalConfigs { - return nil - } - remainRes, err := rf.DropLocalNodes(ra.ResMap().ToRNodeSlice()) - if err != nil { - return err - } - return ra.Intersection(kt.rFactory.FromResourceSlice(remainRes)) -} - -func (kt *KustTarget) runGenerators( - ra *accumulator.ResAccumulator) error { - var generators []*resmap.GeneratorWithProperties - gs, err := kt.configureBuiltinGenerators() - if err != nil { - return err - } - generators = append(generators, gs...) - - gs, err = kt.configureExternalGenerators() - if err != nil { - return errors.WrapPrefixf(err, "loading generator plugins") - } - generators = append(generators, gs...) - for i, g := range generators { - resMap, err := g.Generate() - if err != nil { - return err - } - if resMap != nil { - err = resMap.AddOriginAnnotation(generators[i].Origin) - if err != nil { - return errors.WrapPrefixf(err, "adding origin annotations for generator %v", g) - } - } - err = ra.AbsorbAll(resMap) - if err != nil { - return errors.WrapPrefixf(err, "merging from generator %v", g) - } - } - return nil -} - -func (kt *KustTarget) configureExternalGenerators() ( - []*resmap.GeneratorWithProperties, error) { - ra := accumulator.MakeEmptyAccumulator() - var generatorPaths []string - for _, p := range kt.kustomization.Generators { - // handle inline generators - rm, err := kt.rFactory.NewResMapFromBytes([]byte(p)) - if err != nil { - // not an inline config - generatorPaths = append(generatorPaths, p) - continue - } - // inline config, track the origin - if kt.origin != nil { - resources := rm.Resources() - for _, r := range resources { - r.SetOrigin(kt.origin.Append(kt.kustFileName)) - rm.Replace(r) - } - } - if err = ra.AppendAll(rm); err != nil { - return nil, errors.WrapPrefixf(err, "configuring external generator") - } - } - ra, err := kt.accumulateResources(ra, generatorPaths) - if err != nil { - return nil, err - } - return kt.pLdr.LoadGenerators(kt.ldr, kt.validator, ra.ResMap()) -} - -func (kt *KustTarget) runTransformers(ra *accumulator.ResAccumulator) error { - var r []*resmap.TransformerWithProperties - tConfig := ra.GetTransformerConfig() - lts, err := kt.configureBuiltinTransformers(tConfig) - if err != nil { - return err - } - r = append(r, lts...) - lts, err = kt.configureExternalTransformers(kt.kustomization.Transformers) - if err != nil { - return err - } - r = append(r, lts...) - return ra.Transform(newMultiTransformer(r)) -} - -func (kt *KustTarget) configureExternalTransformers(transformers []string) ([]*resmap.TransformerWithProperties, error) { - ra := accumulator.MakeEmptyAccumulator() - var transformerPaths []string - for _, p := range transformers { - // handle inline transformers - rm, err := kt.rFactory.NewResMapFromBytes([]byte(p)) - if err != nil { - // not an inline config - transformerPaths = append(transformerPaths, p) - continue - } - // inline config, track the origin - if kt.origin != nil { - resources := rm.Resources() - for _, r := range resources { - r.SetOrigin(kt.origin.Append(kt.kustFileName)) - rm.Replace(r) - } - } - - if err = ra.AppendAll(rm); err != nil { - return nil, errors.WrapPrefixf(err, "configuring external transformer") - } - } - ra, err := kt.accumulateResources(ra, transformerPaths) - if err != nil { - return nil, err - } - return kt.pLdr.LoadTransformers(kt.ldr, kt.validator, ra.ResMap()) -} - -func (kt *KustTarget) runValidators(ra *accumulator.ResAccumulator) error { - validators, err := kt.configureExternalTransformers(kt.kustomization.Validators) - if err != nil { - return err - } - for _, v := range validators { - // Validators shouldn't modify the resource map - orignal := ra.ResMap().DeepCopy() - err = v.Transform(ra.ResMap()) - if err != nil { - return err - } - newMap := ra.ResMap().DeepCopy() - if err = kt.removeValidatedByLabel(newMap); err != nil { - return err - } - if err = orignal.ErrorIfNotEqualSets(newMap); err != nil { - return fmt.Errorf("validator shouldn't modify the resource map: %v", err) - } - } - return nil -} - -func (kt *KustTarget) removeValidatedByLabel(rm resmap.ResMap) error { - resources := rm.Resources() - for _, r := range resources { - labels := r.GetLabels() - if _, found := labels[konfig.ValidatedByLabelKey]; !found { - continue - } - delete(labels, konfig.ValidatedByLabelKey) - if err := r.SetLabels(labels); err != nil { - return err - } - } - return nil -} - -// accumulateResources fills the given resourceAccumulator -// with resources read from the given list of paths. -func (kt *KustTarget) accumulateResources( - ra *accumulator.ResAccumulator, paths []string) (*accumulator.ResAccumulator, error) { - for _, path := range paths { - // try loading resource as file then as base (directory or git repository) - if errF := kt.accumulateFile(ra, path); errF != nil { - // not much we can do if the error is an HTTP error so we bail out - if errors.Is(errF, load.ErrHTTP) { - return nil, errF - } - var redErr *load.RedirectionError - if errors.As(errF, &redErr) { - path = redErr.NewPath - } - ldr, err := kt.ldr.New(path) - - if err != nil { - // If accumulateFile found malformed YAML and there was a failure - // loading the resource as a base, then the resource is likely a - // file. The loader failure message is unnecessary, and could be - // confusing. Report only the file load error. - // - // However, a loader timeout implies there is a git repo at the - // path. In that case, both errors could be important. - if kusterr.IsMalformedYAMLError(errF) && !utils.IsErrTimeout(err) { - return nil, errF - } - return nil, errors.WrapPrefixf( - err, "accumulation err='%s'", errF.Error()) - } - // store the origin, we'll need it later - origin := kt.origin.Copy() - if kt.origin != nil { - kt.origin = kt.origin.Append(path) - ra, err = kt.accumulateDirectory(ra, ldr, false) - // after we are done recursing through the directory, reset the origin - kt.origin = &origin - } else { - ra, err = kt.accumulateDirectory(ra, ldr, false) - } - if err != nil { - if kusterr.IsMalformedYAMLError(errF) { // Some error occurred while tyring to decode YAML file - return nil, errF - } - return nil, errors.WrapPrefixf( - err, "accumulation err='%s'", errF.Error()) - } - } - } - return ra, nil -} - -// accumulateResources fills the given resourceAccumulator -// with resources read from the given list of paths. -func (kt *KustTarget) accumulateComponents( - ra *accumulator.ResAccumulator, paths []string) (*accumulator.ResAccumulator, error) { - for _, path := range paths { - // Components always refer to directories - ldr, errL := kt.ldr.New(path) - if errL != nil { - return nil, fmt.Errorf("loader.New %q", errL) - } - var errD error - // store the origin, we'll need it later - origin := kt.origin.Copy() - if kt.origin != nil { - kt.origin = kt.origin.Append(path) - ra, errD = kt.accumulateDirectory(ra, ldr, true) - // after we are done recursing through the directory, reset the origin - kt.origin = &origin - } else { - ra, errD = kt.accumulateDirectory(ra, ldr, true) - } - if errD != nil { - return nil, fmt.Errorf("accumulateDirectory: %q", errD) - } - } - return ra, nil -} - -func (kt *KustTarget) accumulateDirectory( - ra *accumulator.ResAccumulator, ldr ifc.Loader, isComponent bool) (*accumulator.ResAccumulator, error) { - defer ldr.Cleanup() - subKt := NewKustTarget(ldr, kt.validator, kt.rFactory, kt.pLdr) - err := subKt.Load() - if err != nil { - return nil, errors.WrapPrefixf( - err, "couldn't make target for path '%s'", ldr.Root()) - } - subKt.kustomization.BuildMetadata = kt.kustomization.BuildMetadata - subKt.origin = kt.origin - var bytes []byte - if openApiPath, exists := subKt.Kustomization().OpenAPI["path"]; exists { - bytes, err = ldr.Load(openApiPath) - if err != nil { - return nil, err - } - } - err = openapi.SetSchema(subKt.Kustomization().OpenAPI, bytes, false) - if err != nil { - return nil, err - } - if isComponent && subKt.kustomization.Kind != types.ComponentKind { - return nil, fmt.Errorf( - "expected kind '%s' for path '%s' but got '%s'", types.ComponentKind, ldr.Root(), subKt.kustomization.Kind) - } else if !isComponent && subKt.kustomization.Kind == types.ComponentKind { - return nil, fmt.Errorf( - "expected kind != '%s' for path '%s'", types.ComponentKind, ldr.Root()) - } - - var subRa *accumulator.ResAccumulator - if isComponent { - // Components don't create a new accumulator: the kustomization directives are added to the current accumulator - subRa, err = subKt.accumulateTarget(ra) - ra = accumulator.MakeEmptyAccumulator() - } else { - // Child Kustomizations create a new accumulator which resolves their kustomization directives, which will later - // be merged into the current accumulator. - subRa, err = subKt.AccumulateTarget() - } - if err != nil { - return nil, errors.WrapPrefixf( - err, "recursed accumulation of path '%s'", ldr.Root()) - } - err = ra.MergeAccumulator(subRa) - if err != nil { - return nil, errors.WrapPrefixf( - err, "recursed merging from path '%s'", ldr.Root()) - } - return ra, nil -} - -func (kt *KustTarget) accumulateFile( - ra *accumulator.ResAccumulator, path string) error { - resources, err := kt.rFactory.FromFile(kt.ldr, path) - if err != nil { - return errors.WrapPrefixf(err, "accumulating resources from '%s'", path) - } - if kt.origin != nil { - originAnno, err := kt.origin.Append(path).String() - if err != nil { - return errors.WrapPrefixf(err, "cannot add path annotation for '%s'", path) - } - err = resources.AnnotateAll(utils.OriginAnnotationKey, originAnno) - if err != nil || originAnno == "" { - return errors.WrapPrefixf(err, "cannot add path annotation for '%s'", path) - } - } - err = ra.AppendAll(resources) - if err != nil { - return errors.WrapPrefixf(err, "merging resources from '%s'", path) - } - return nil -} - -func (kt *KustTarget) configureBuiltinPlugin( - p resmap.Configurable, c interface{}, bpt builtinhelpers.BuiltinPluginType) (err error) { - var y []byte - if c != nil { - y, err = yaml.Marshal(c) - if err != nil { - return errors.WrapPrefixf( - err, "builtin %s marshal", bpt) - } - } - err = p.Config( - resmap.NewPluginHelpers( - kt.ldr, kt.validator, kt.rFactory, kt.pLdr.Config()), - y) - if err != nil { - return errors.WrapPrefixf( - err, "trouble configuring builtin %s with config: `\n%s`", bpt, string(y)) - } - return nil -} +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package target + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "sigs.k8s.io/kustomize/api/ifc" + "sigs.k8s.io/kustomize/api/internal/accumulator" + "sigs.k8s.io/kustomize/api/internal/builtins" + "sigs.k8s.io/kustomize/api/internal/kusterr" + load "sigs.k8s.io/kustomize/api/internal/loader" + "sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig" + "sigs.k8s.io/kustomize/api/internal/plugins/builtinhelpers" + "sigs.k8s.io/kustomize/api/internal/plugins/loader" + "sigs.k8s.io/kustomize/api/internal/utils" + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/openapi" + "sigs.k8s.io/yaml" +) + +// KustTarget encapsulates the entirety of a kustomization build. +type KustTarget struct { + kustomization *types.Kustomization + kustFileName string + ldr ifc.Loader + validator ifc.Validator + rFactory *resmap.Factory + pLdr *loader.Loader + origin *resource.Origin +} + +// NewKustTarget returns a new instance of KustTarget. +func NewKustTarget( + ldr ifc.Loader, + validator ifc.Validator, + rFactory *resmap.Factory, + pLdr *loader.Loader) *KustTarget { + return &KustTarget{ + ldr: ldr, + validator: validator, + rFactory: rFactory, + pLdr: pLdr.LoaderWithWorkingDir(ldr.Root()), + } +} + +// Load attempts to load the target's kustomization file. +func (kt *KustTarget) Load() error { + content, kustFileName, err := LoadKustFile(kt.ldr) + if err != nil { + return err + } + + var k types.Kustomization + if err := k.Unmarshal(content); err != nil { + return err + } + + // show warning message when using deprecated fields. + if warningMessages := k.CheckDeprecatedFields(); warningMessages != nil { + for _, msg := range *warningMessages { + fmt.Fprintf(os.Stderr, "%v\n", msg) + } + } + + k.FixKustomization() + + // check that Kustomization is empty + if err := k.CheckEmpty(); err != nil { + return err + } + + errs := k.EnforceFields() + if len(errs) > 0 { + return fmt.Errorf( + "Failed to read kustomization file under %s:\n"+ + strings.Join(errs, "\n"), kt.ldr.Root()) + } + kt.kustomization = &k + kt.kustFileName = kustFileName + return nil +} + +// Kustomization returns a copy of the immutable, internal kustomization object. +func (kt *KustTarget) Kustomization() types.Kustomization { + var result types.Kustomization + b, _ := json.Marshal(*kt.kustomization) + json.Unmarshal(b, &result) + return result +} + +func LoadKustFile(ldr ifc.Loader) ([]byte, string, error) { + var content []byte + match := 0 + var kustFileName string + for _, kf := range konfig.RecognizedKustomizationFileNames() { + c, err := ldr.Load(kf) + if err == nil { + match += 1 + content = c + kustFileName = kf + } + } + switch match { + case 0: + return nil, "", NewErrMissingKustomization(ldr.Root()) + case 1: + return content, kustFileName, nil + default: + return nil, "", fmt.Errorf( + "Found multiple kustomization files under: %s\n", ldr.Root()) + } +} + +// MakeCustomizedResMap creates a fully customized ResMap +// per the instructions contained in its kustomization instance. +func (kt *KustTarget) MakeCustomizedResMap() (resmap.ResMap, error) { + return kt.makeCustomizedResMap() +} + +func (kt *KustTarget) makeCustomizedResMap() (resmap.ResMap, error) { + var origin *resource.Origin + if len(kt.kustomization.BuildMetadata) != 0 { + origin = &resource.Origin{} + } + kt.origin = origin + ra, err := kt.AccumulateTarget() + if err != nil { + return nil, err + } + + // The following steps must be done last, not as part of + // the recursion implicit in AccumulateTarget. + + err = kt.addHashesToNames(ra) + if err != nil { + return nil, err + } + + // Given that names have changed (prefixs/suffixes added), + // fix all the back references to those names. + err = ra.FixBackReferences() + if err != nil { + return nil, err + } + + // With all the back references fixed, it's OK to resolve Vars. + err = ra.ResolveVars() + if err != nil { + return nil, err + } + + err = kt.IgnoreLocal(ra) + if err != nil { + return nil, err + } + + return ra.ResMap(), nil +} + +func (kt *KustTarget) addHashesToNames( + ra *accumulator.ResAccumulator) error { + p := builtins.NewHashTransformerPlugin() + err := kt.configureBuiltinPlugin(p, nil, builtinhelpers.HashTransformer) + if err != nil { + return err + } + return ra.Transform(p) +} + +// AccumulateTarget returns a new ResAccumulator, +// holding customized resources and the data/rules used +// to do so. The name back references and vars are +// not yet fixed. +// The origin parameter is used through the recursive calls +// to annotate each resource with information about where +// the resource came from, e.g. the file and/or the repository +// it originated from. +// As an entrypoint, one can pass an empty resource.Origin object to +// AccumulateTarget. As AccumulateTarget moves recursively +// through kustomization directories, it updates `origin.path` +// accordingly. When a remote base is found, it updates `origin.repo` +// and `origin.ref` accordingly. +func (kt *KustTarget) AccumulateTarget() ( + ra *accumulator.ResAccumulator, err error) { + return kt.accumulateTarget(accumulator.MakeEmptyAccumulator()) +} + +// ra should be empty when this KustTarget is a Kustomization, or the ra of the parent if this KustTarget is a Component +// (or empty if the Component does not have a parent). +func (kt *KustTarget) accumulateTarget(ra *accumulator.ResAccumulator) ( + resRa *accumulator.ResAccumulator, err error) { + ra, err = kt.accumulateResources(ra, kt.kustomization.Resources) + if err != nil { + return nil, errors.WrapPrefixf(err, "accumulating resources") + } + tConfig, err := builtinconfig.MakeTransformerConfig( + kt.ldr, kt.kustomization.Configurations) + if err != nil { + return nil, err + } + err = ra.MergeConfig(tConfig) + if err != nil { + return nil, errors.WrapPrefixf( + err, "merging config %v", tConfig) + } + crdTc, err := accumulator.LoadConfigFromCRDs(kt.ldr, kt.kustomization.Crds) + if err != nil { + return nil, errors.WrapPrefixf( + err, "loading CRDs %v", kt.kustomization.Crds) + } + err = ra.MergeConfig(crdTc) + if err != nil { + return nil, errors.WrapPrefixf( + err, "merging CRDs %v", crdTc) + } + err = kt.runGenerators(ra) + if err != nil { + return nil, err + } + + // components are expected to execute after reading resources and adding generators ,before applying transformers and validation. + // https://github.com/kubernetes-sigs/kustomize/pull/5170#discussion_r1212101287 + ra, err = kt.accumulateComponents(ra, kt.kustomization.Components) + if err != nil { + return nil, errors.WrapPrefixf(err, "accumulating components") + } + + err = kt.runTransformers(ra) + if err != nil { + return nil, err + } + err = kt.runValidators(ra) + if err != nil { + return nil, err + } + err = ra.MergeVars(kt.kustomization.Vars) + if err != nil { + return nil, errors.WrapPrefixf( + err, "merging vars %v", kt.kustomization.Vars) + } + return ra, nil +} + +// IgnoreLocal drops the local resource by checking the annotation "config.kubernetes.io/local-config". +func (kt *KustTarget) IgnoreLocal(ra *accumulator.ResAccumulator) error { + rf := kt.rFactory.RF() + if rf.IncludeLocalConfigs { + return nil + } + remainRes, err := rf.DropLocalNodes(ra.ResMap().ToRNodeSlice()) + if err != nil { + return err + } + return ra.Intersection(kt.rFactory.FromResourceSlice(remainRes)) +} + +func (kt *KustTarget) runGenerators( + ra *accumulator.ResAccumulator) error { + var generators []*resmap.GeneratorWithProperties + gs, err := kt.configureBuiltinGenerators() + if err != nil { + return err + } + generators = append(generators, gs...) + + gs, err = kt.configureExternalGenerators() + if err != nil { + return errors.WrapPrefixf(err, "loading generator plugins") + } + generators = append(generators, gs...) + for i, g := range generators { + resMap, err := g.Generate() + if err != nil { + return err + } + if resMap != nil { + err = resMap.AddOriginAnnotation(generators[i].Origin) + if err != nil { + return errors.WrapPrefixf(err, "adding origin annotations for generator %v", g) + } + } + err = ra.AbsorbAll(resMap) + if err != nil { + return errors.WrapPrefixf(err, "merging from generator %v", g) + } + } + return nil +} + +func (kt *KustTarget) configureExternalGenerators() ( + []*resmap.GeneratorWithProperties, error) { + ra := accumulator.MakeEmptyAccumulator() + var generatorPaths []string + for _, p := range kt.kustomization.Generators { + // handle inline generators + rm, err := kt.rFactory.NewResMapFromBytes([]byte(p)) + if err != nil { + // not an inline config + generatorPaths = append(generatorPaths, p) + continue + } + // inline config, track the origin + if kt.origin != nil { + resources := rm.Resources() + for _, r := range resources { + r.SetOrigin(kt.origin.Append(kt.kustFileName)) + rm.Replace(r) + } + } + if err = ra.AppendAll(rm); err != nil { + return nil, errors.WrapPrefixf(err, "configuring external generator") + } + } + ra, err := kt.accumulateResources(ra, generatorPaths) + if err != nil { + return nil, err + } + return kt.pLdr.LoadGenerators(kt.ldr, kt.validator, ra.ResMap()) +} + +func (kt *KustTarget) runTransformers(ra *accumulator.ResAccumulator) error { + var r []*resmap.TransformerWithProperties + tConfig := ra.GetTransformerConfig() + lts, err := kt.configureBuiltinTransformers(tConfig) + if err != nil { + return err + } + r = append(r, lts...) + lts, err = kt.configureExternalTransformers(kt.kustomization.Transformers) + if err != nil { + return err + } + r = append(r, lts...) + return ra.Transform(newMultiTransformer(r)) +} + +func (kt *KustTarget) configureExternalTransformers(transformers []string) ([]*resmap.TransformerWithProperties, error) { + ra := accumulator.MakeEmptyAccumulator() + var transformerPaths []string + for _, p := range transformers { + // handle inline transformers + rm, err := kt.rFactory.NewResMapFromBytes([]byte(p)) + if err != nil { + // not an inline config + transformerPaths = append(transformerPaths, p) + continue + } + // inline config, track the origin + if kt.origin != nil { + resources := rm.Resources() + for _, r := range resources { + r.SetOrigin(kt.origin.Append(kt.kustFileName)) + rm.Replace(r) + } + } + + if err = ra.AppendAll(rm); err != nil { + return nil, errors.WrapPrefixf(err, "configuring external transformer") + } + } + ra, err := kt.accumulateResources(ra, transformerPaths) + if err != nil { + return nil, err + } + return kt.pLdr.LoadTransformers(kt.ldr, kt.validator, ra.ResMap()) +} + +func (kt *KustTarget) runValidators(ra *accumulator.ResAccumulator) error { + validators, err := kt.configureExternalTransformers(kt.kustomization.Validators) + if err != nil { + return err + } + for _, v := range validators { + // Validators shouldn't modify the resource map + orignal := ra.ResMap().DeepCopy() + err = v.Transform(ra.ResMap()) + if err != nil { + return err + } + newMap := ra.ResMap().DeepCopy() + if err = kt.removeValidatedByLabel(newMap); err != nil { + return err + } + if err = orignal.ErrorIfNotEqualSets(newMap); err != nil { + return fmt.Errorf("validator shouldn't modify the resource map: %v", err) + } + } + return nil +} + +func (kt *KustTarget) removeValidatedByLabel(rm resmap.ResMap) error { + resources := rm.Resources() + for _, r := range resources { + labels := r.GetLabels() + if _, found := labels[konfig.ValidatedByLabelKey]; !found { + continue + } + delete(labels, konfig.ValidatedByLabelKey) + if err := r.SetLabels(labels); err != nil { + return err + } + } + return nil +} + +// accumulateResources fills the given resourceAccumulator +// with resources read from the given list of paths. +func (kt *KustTarget) accumulateResources( + ra *accumulator.ResAccumulator, paths []string) (*accumulator.ResAccumulator, error) { + for _, path := range paths { + // try loading resource as file then as base (directory or git repository) + if errF := kt.accumulateFile(ra, path); errF != nil { + // not much we can do if the error is an HTTP error so we bail out + if errors.Is(errF, load.ErrHTTP) { + return nil, errF + } + var redErr *load.RedirectionError + if errors.As(errF, &redErr) { + path = redErr.NewPath + } + ldr, err := kt.ldr.New(path) + + if err != nil { + // If accumulateFile found malformed YAML and there was a failure + // loading the resource as a base, then the resource is likely a + // file. The loader failure message is unnecessary, and could be + // confusing. Report only the file load error. + // + // However, a loader timeout implies there is a git repo at the + // path. In that case, both errors could be important. + if kusterr.IsMalformedYAMLError(errF) && !utils.IsErrTimeout(err) { + return nil, errF + } + return nil, errors.WrapPrefixf( + err, "accumulation err='%s'", errF.Error()) + } + // store the origin, we'll need it later + origin := kt.origin.Copy() + if kt.origin != nil { + kt.origin = kt.origin.Append(path) + ra, err = kt.accumulateDirectory(ra, ldr, false) + // after we are done recursing through the directory, reset the origin + kt.origin = &origin + } else { + ra, err = kt.accumulateDirectory(ra, ldr, false) + } + if err != nil { + if kusterr.IsMalformedYAMLError(errF) { // Some error occurred while tyring to decode YAML file + return nil, errF + } + return nil, errors.WrapPrefixf( + err, "accumulation err='%s'", errF.Error()) + } + } + } + return ra, nil +} + +// accumulateResources fills the given resourceAccumulator +// with resources read from the given list of paths. +func (kt *KustTarget) accumulateComponents( + ra *accumulator.ResAccumulator, paths []string) (*accumulator.ResAccumulator, error) { + for _, path := range paths { + // Components always refer to directories + ldr, errL := kt.ldr.New(path) + if errL != nil { + return nil, fmt.Errorf("loader.New %q", errL) + } + var errD error + // store the origin, we'll need it later + origin := kt.origin.Copy() + if kt.origin != nil { + kt.origin = kt.origin.Append(path) + ra, errD = kt.accumulateDirectory(ra, ldr, true) + // after we are done recursing through the directory, reset the origin + kt.origin = &origin + } else { + ra, errD = kt.accumulateDirectory(ra, ldr, true) + } + if errD != nil { + return nil, fmt.Errorf("accumulateDirectory: %q", errD) + } + } + return ra, nil +} + +func (kt *KustTarget) accumulateDirectory( + ra *accumulator.ResAccumulator, ldr ifc.Loader, isComponent bool) (*accumulator.ResAccumulator, error) { + defer ldr.Cleanup() + subKt := NewKustTarget(ldr, kt.validator, kt.rFactory, kt.pLdr) + err := subKt.Load() + if err != nil { + return nil, errors.WrapPrefixf( + err, "couldn't make target for path '%s'", ldr.Root()) + } + subKt.kustomization.BuildMetadata = kt.kustomization.BuildMetadata + subKt.origin = kt.origin + var bytes []byte + if openApiPath, exists := subKt.Kustomization().OpenAPI["path"]; exists { + bytes, err = ldr.Load(openApiPath) + if err != nil { + return nil, err + } + } + err = openapi.SetSchema(subKt.Kustomization().OpenAPI, bytes, false) + if err != nil { + return nil, err + } + if isComponent && subKt.kustomization.Kind != types.ComponentKind { + return nil, fmt.Errorf( + "expected kind '%s' for path '%s' but got '%s'", types.ComponentKind, ldr.Root(), subKt.kustomization.Kind) + } else if !isComponent && subKt.kustomization.Kind == types.ComponentKind { + return nil, fmt.Errorf( + "expected kind != '%s' for path '%s'", types.ComponentKind, ldr.Root()) + } + + var subRa *accumulator.ResAccumulator + if isComponent { + // Components don't create a new accumulator: the kustomization directives are added to the current accumulator + subRa, err = subKt.accumulateTarget(ra) + ra = accumulator.MakeEmptyAccumulator() + } else { + // Child Kustomizations create a new accumulator which resolves their kustomization directives, which will later + // be merged into the current accumulator. + subRa, err = subKt.AccumulateTarget() + } + if err != nil { + return nil, errors.WrapPrefixf( + err, "recursed accumulation of path '%s'", ldr.Root()) + } + err = ra.MergeAccumulator(subRa) + if err != nil { + return nil, errors.WrapPrefixf( + err, "recursed merging from path '%s'", ldr.Root()) + } + return ra, nil +} + +func (kt *KustTarget) accumulateFile( + ra *accumulator.ResAccumulator, path string) error { + resources, err := kt.rFactory.FromFile(kt.ldr, path) + if err != nil { + return errors.WrapPrefixf(err, "accumulating resources from '%s'", path) + } + if kt.origin != nil { + originAnno, err := kt.origin.Append(path).String() + if err != nil { + return errors.WrapPrefixf(err, "cannot add path annotation for '%s'", path) + } + err = resources.AnnotateAll(utils.OriginAnnotationKey, originAnno) + if err != nil || originAnno == "" { + return errors.WrapPrefixf(err, "cannot add path annotation for '%s'", path) + } + } + err = ra.AppendAll(resources) + if err != nil { + return errors.WrapPrefixf(err, "merging resources from '%s'", path) + } + return nil +} + +func (kt *KustTarget) configureBuiltinPlugin( + p resmap.Configurable, c interface{}, bpt builtinhelpers.BuiltinPluginType) (err error) { + var y []byte + if c != nil { + y, err = yaml.Marshal(c) + if err != nil { + return errors.WrapPrefixf( + err, "builtin %s marshal", bpt) + } + } + err = p.Config( + resmap.NewPluginHelpers( + kt.ldr, kt.validator, kt.rFactory, kt.pLdr.Config()), + y) + if err != nil { + return errors.WrapPrefixf( + err, "trouble configuring builtin %s with config: `\n%s`", bpt, string(y)) + } + return nil +}