diff --git a/cmd/tk/tool.go b/cmd/tk/tool.go index 7e19bf0c4..2b389bbf5 100644 --- a/cmd/tk/tool.go +++ b/cmd/tk/tool.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/go-clix/cli" + "github.com/posener/complete" "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/jsonnet/jpath" @@ -23,6 +24,7 @@ func toolCmd() *cli.Command { cmd.AddCommand( jpathCmd(), importsCmd(), + importersCmd(), chartsCmd(), ) return cmd @@ -130,6 +132,42 @@ func importsCmd() *cli.Command { return cmd } +func importersCmd() *cli.Command { + cmd := &cli.Command{ + Use: "importers ", + Short: "list all environments that either directly or transitively import the given files", + Args: cli.Args{ + Validator: cli.ArgsMin(1), + Predictor: complete.PredictFiles("*"), + }, + } + + root := cmd.Flags().String("root", ".", "root directory to search for environments") + cmd.Run = func(cmd *cli.Command, args []string) error { + root, err := filepath.Abs(*root) + if err != nil { + return fmt.Errorf("resolving root: %w", err) + } + + for _, f := range args { + if _, err := os.Stat(f); os.IsNotExist(err) { + return fmt.Errorf("file %q does not exist", f) + } + } + + envs, err := jsonnet.FindImporterForFiles(root, args, nil) + if err != nil { + return fmt.Errorf("resolving imports: %s", err) + } + + fmt.Println(strings.Join(envs, "\n")) + + return nil + } + + return cmd +} + func gitRoot() (string, error) { s, err := git("rev-parse", "--show-toplevel") return strings.TrimRight(s, "\n"), err diff --git a/pkg/jsonnet/find_importers.go b/pkg/jsonnet/find_importers.go new file mode 100644 index 000000000..61f91ebb6 --- /dev/null +++ b/pkg/jsonnet/find_importers.go @@ -0,0 +1,239 @@ +package jsonnet + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/grafana/tanka/pkg/jsonnet/jpath" +) + +// FindImporterForFiles finds the entrypoints (main.jsonnet files) that import the given files. +// It looks through imports transitively, so if a file is imported through a chain, it will still be reported. +// If the given file is a main.jsonnet file, it will be returned as well. +func FindImporterForFiles(root string, files []string, chain map[string]struct{}) ([]string, error) { + if chain == nil { + chain = make(map[string]struct{}) + } + + var err error + root, err = filepath.Abs(root) + if err != nil { + return nil, err + } + + importers := map[string]struct{}{} + + if len(chain) == 0 { + for i := range files { + files[i], err = filepath.Abs(files[i]) + if err != nil { + return nil, err + } + + symlink, err := evalSymlinks(files[i]) + if err != nil { + return nil, err + } + if symlink != files[i] { + files = append(files, symlink) + } + + symlinks, err := findSymlinks(root, files[i]) + if err != nil { + return nil, err + } + files = append(files, symlinks...) + } + + files = uniqueStringSlice(files) + } + + for _, file := range files { + if filepath.Base(file) == jpath.DefaultEntrypoint { + importers[file] = struct{}{} + } + + newImporters, err := findImporters(root, file, chain) + if err != nil { + return nil, err + } + for _, importer := range newImporters { + importers[importer] = struct{}{} + } + } + + var importersSlice []string + for importer := range importers { + importersSlice = append(importersSlice, importer) + } + + sort.Strings(importersSlice) + + return importersSlice, nil +} + +type cachedJsonnetFile struct { + Base string + Imports []string + Content string + IsMainFile bool +} + +var jsonnetFilesMap = make(map[string]map[string]*cachedJsonnetFile) +var symlinkCache = make(map[string]string) + +func evalSymlinks(path string) (string, error) { + var err error + eval, ok := symlinkCache[path] + if !ok { + eval, err = filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + symlinkCache[path] = eval + } + return eval, nil +} + +func findSymlinks(root, file string) ([]string, error) { + var symlinks []string + + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + eval, err := evalSymlinks(path) + if err != nil { + return err + } + if strings.Contains(file, eval) { + symlinks = append(symlinks, strings.Replace(file, eval, path, 1)) + } + } + + return nil + }) + + return symlinks, err +} + +func findImporters(root string, searchForFile string, chain map[string]struct{}) ([]string, error) { + if _, ok := chain[searchForFile]; ok { + return nil, nil + } + chain[searchForFile] = struct{}{} + + if _, ok := jsonnetFilesMap[root]; !ok { + jsonnetFilesMap[root] = make(map[string]*cachedJsonnetFile) + + files, err := FindFiles(root, nil) + if err != nil { + return nil, err + } + for _, file := range files { + content, err := os.ReadFile(file) + if err != nil { + return nil, err + } + matches := importsRegexp.FindAllStringSubmatch(string(content), -1) + + cachedObj := &cachedJsonnetFile{ + Content: string(content), + IsMainFile: strings.HasSuffix(file, jpath.DefaultEntrypoint), + } + for _, match := range matches { + cachedObj.Imports = append(cachedObj.Imports, match[2]) + } + jsonnetFilesMap[root][file] = cachedObj + } + } + jsonnetFiles := jsonnetFilesMap[root] + + var importers []string + var intermediateImporters []string + + for jsonnetFilePath, jsonnetFileContent := range jsonnetFiles { + isImporter := false + for _, importPath := range jsonnetFileContent.Imports { + if filepath.Base(importPath) != filepath.Base(searchForFile) { // If the filename is not the same as the file we are looking for, skip + continue + } + + // Match on relative imports with .. + // Jsonnet also matches all intermediary paths for some reason, so we look at them too + doubleDotCount := strings.Count(importPath, "..") + if doubleDotCount > 0 { + importPath = strings.ReplaceAll(importPath, "../", "") + for i := 0; i <= doubleDotCount; i++ { + dir := filepath.Dir(jsonnetFilePath) + for j := 0; j < i; j++ { + dir = filepath.Dir(dir) + } + testImportPath := filepath.Join(dir, importPath) + isImporter = pathMatches(searchForFile, testImportPath) + } + } + + // Match on imports to lib/ or vendor/ + if !isImporter { + importPath = strings.ReplaceAll(importPath, "./", "") + isImporter = pathMatches(searchForFile, filepath.Join(root, "vendor", importPath)) || pathMatches(searchForFile, filepath.Join(root, "lib", importPath)) + } + + // Match on imports to the base dir where the file is located (e.g. in the env dir) + if !isImporter { + if jsonnetFileContent.Base == "" { + base, err := jpath.FindBase(jsonnetFilePath, root) + if err != nil { + return nil, err + } + jsonnetFileContent.Base = base + } + isImporter = strings.HasPrefix(searchForFile, jsonnetFileContent.Base) && strings.HasSuffix(searchForFile, importPath) + } + + if isImporter { + if jsonnetFileContent.IsMainFile { + importers = append(importers, jsonnetFilePath) + } else { + intermediateImporters = append(intermediateImporters, jsonnetFilePath) + } + break + } + } + } + + if len(intermediateImporters) > 0 { + newImporters, err := FindImporterForFiles(root, intermediateImporters, chain) + if err != nil { + return nil, err + } + importers = append(importers, newImporters...) + } + + return importers, nil +} + +func pathMatches(path1, path2 string) bool { + if path1 == path2 { + return true + } + + var err error + + evalPath1, err := evalSymlinks(path1) + if err != nil { + return false + } + + evalPath2, err := evalSymlinks(path2) + if err != nil { + return false + } + + return evalPath1 == evalPath2 +} diff --git a/pkg/jsonnet/find_importers_test.go b/pkg/jsonnet/find_importers_test.go new file mode 100644 index 000000000..5ba1a2d4f --- /dev/null +++ b/pkg/jsonnet/find_importers_test.go @@ -0,0 +1,115 @@ +package jsonnet + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFindImportersForFiles(t *testing.T) { + cases := []struct { + name string + files []string + expectedImporters []string + expectedErr error + }{ + { + name: "no files", + files: []string{}, + }, + { + name: "invalid file", + files: []string{"testdata/findImporters/does-not-exist.jsonnet"}, + expectedErr: fmt.Errorf("lstat %s: no such file or directory", absPath(t, "testdata/findImporters/does-not-exist.jsonnet")), + }, + { + name: "project with no imports", + files: []string{"testdata/findImporters/environments/no-imports/main.jsonnet"}, + expectedImporters: []string{absPath(t, "testdata/findImporters/environments/no-imports/main.jsonnet")}, // itself only + }, + { + name: "local import", + files: []string{"testdata/findImporters/environments/imports-locals-and-vendored/local-file1.libsonnet"}, + expectedImporters: []string{absPath(t, "testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet")}, + }, + { + name: "local import with relative path", + files: []string{"testdata/findImporters/environments/imports-locals-and-vendored/local-file2.libsonnet"}, + expectedImporters: []string{absPath(t, "testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet")}, + }, + { + name: "lib imported through chain", + files: []string{"testdata/findImporters/lib/lib1/main.libsonnet"}, + expectedImporters: []string{ + absPath(t, "testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet"), + }, + }, + { + name: "vendored lib imported through chain + directly", + files: []string{"testdata/findImporters/vendor/vendored/main.libsonnet"}, + expectedImporters: []string{ + absPath(t, "testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet"), + absPath(t, "testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet"), + absPath(t, "testdata/findImporters/environments/imports-symlinked-vendor/main.jsonnet"), + }, + }, + { + name: "vendored lib found through symlink", // expect same result as previous test + files: []string{"testdata/findImporters/vendor/vendor-symlinked/main.libsonnet"}, + expectedImporters: []string{ + absPath(t, "testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet"), + absPath(t, "testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet"), + absPath(t, "testdata/findImporters/environments/imports-symlinked-vendor/main.jsonnet"), + }, + }, + { + name: "text file", + files: []string{"testdata/findImporters/vendor/vendored/text-file.txt"}, + expectedImporters: []string{ + absPath(t, "testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet"), + absPath(t, "testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet"), + absPath(t, "testdata/findImporters/environments/imports-symlinked-vendor/main.jsonnet"), + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + importers, err := FindImporterForFiles("testdata/findImporters", c.files, nil) + + if c.expectedErr != nil { + require.EqualError(t, err, c.expectedErr.Error()) + } else { + require.NoError(t, err) + require.Equal(t, c.expectedImporters, importers) + } + }) + } + +} + +func BenchmarkFindImporters(b *testing.B) { + // Create a very large and complex project + tempDir := b.TempDir() + generateTestProject(b, tempDir, 100, false) + + // Run the benchmark + expectedImporters := []string{filepath.Join(tempDir, "main.jsonnet")} + b.ResetTimer() + for i := 0; i < b.N; i++ { + importers, err := FindImporterForFiles(tempDir, []string{filepath.Join(tempDir, "file10.libsonnet")}, nil) + + require.NoError(b, err) + require.Equal(b, expectedImporters, importers) + } +} + +func absPath(t *testing.T, path string) string { + t.Helper() + + abs, err := filepath.Abs(path) + require.NoError(t, err) + return abs +} diff --git a/pkg/jsonnet/imports.go b/pkg/jsonnet/imports.go index cbe2acb2a..38e69fff7 100644 --- a/pkg/jsonnet/imports.go +++ b/pkg/jsonnet/imports.go @@ -218,3 +218,17 @@ func findImportRecursiveRegexp(list map[string]bool, vm *jsonnet.VM, filename, c } return nil } + +func uniqueStringSlice(s []string) []string { + seen := make(map[string]struct{}, len(s)) + j := 0 + for _, v := range s { + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + s[j] = v + j++ + } + return s[:j] +} diff --git a/pkg/jsonnet/imports_test.go b/pkg/jsonnet/imports_test.go index 7c3b5e736..7c7787d32 100644 --- a/pkg/jsonnet/imports_test.go +++ b/pkg/jsonnet/imports_test.go @@ -48,7 +48,7 @@ func BenchmarkGetSnippetHash(b *testing.B) { b.Run(tc.name, func(b *testing.B) { // Create a very large and complex project tempDir := b.TempDir() - generateTestProject(b, tempDir, 10000, tc.importFromMain) + generateTestProject(b, tempDir, 1000, tc.importFromMain) // Create a VM. It's important to reuse the same VM // While there is a caching mechanism that normally shouldn't be shared in a benchmark iteration, @@ -118,7 +118,7 @@ func generateTestProject(t testing.TB, dir string, depth int, importAllFromMain var allFiles []string var mainContentSplit []string - for i := 0; i < 1000; i++ { + for i := 0; i < depth; i++ { mainContentSplit = append(mainContentSplit, fmt.Sprintf("(import 'file%d.libsonnet')", i)) filePath := filepath.Join(dir, fmt.Sprintf("file%d.libsonnet", i)) err := os.WriteFile( @@ -134,8 +134,8 @@ func generateTestProject(t testing.TB, dir string, depth int, importAllFromMain } require.NoError(t, os.WriteFile(filepath.Join(dir, "main.jsonnet"), []byte(strings.Join(mainContentSplit, " + ")), 0644)) allFiles = append(allFiles, filepath.Join(dir, "main.jsonnet")) - require.NoError(t, os.WriteFile(filepath.Join(dir, "file1000.libsonnet"), []byte(`"a string"`), 0644)) - allFiles = append(allFiles, filepath.Join(dir, "file1000.libsonnet")) + require.NoError(t, os.WriteFile(filepath.Join(dir, fmt.Sprintf("file%d.libsonnet", depth)), []byte(`"a string"`), 0644)) + allFiles = append(allFiles, filepath.Join(dir, fmt.Sprintf("file%d.libsonnet", depth))) return allFiles } diff --git a/pkg/jsonnet/jpath/jpath.go b/pkg/jsonnet/jpath/jpath.go index e99f0bc2e..e5fd7590a 100644 --- a/pkg/jsonnet/jpath/jpath.go +++ b/pkg/jsonnet/jpath/jpath.go @@ -5,7 +5,7 @@ import ( "path/filepath" ) -const defaultEntrypoint = "main.jsonnet" +const DefaultEntrypoint = "main.jsonnet" // Resolve the given path and resolves the jPath around it. This means it: // - figures out the project root (the one with .jsonnetfile, vendor/ and lib/) @@ -49,7 +49,7 @@ func Filename(path string) (string, error) { } if fi.IsDir() { - return defaultEntrypoint, nil + return DefaultEntrypoint, nil } return filepath.Base(fi.Name()), nil diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/chain1.libsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/chain1.libsonnet new file mode 100644 index 000000000..5a030f0bf --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/chain1.libsonnet @@ -0,0 +1,3 @@ +{ + chain: import 'chain2.libsonnet', +} diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/chain2.libsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/chain2.libsonnet new file mode 100644 index 000000000..baf659c74 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/chain2.libsonnet @@ -0,0 +1,3 @@ +{ + chain: import 'lib1/main.libsonnet', +} diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet new file mode 100644 index 000000000..010b81fff --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-lib-and-vendored-through-chain/main.jsonnet @@ -0,0 +1,4 @@ +{ + chain: import 'chain1.libsonnet', + +} diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/local-file1.libsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/local-file1.libsonnet new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/local-file1.libsonnet @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/local-file2.libsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/local-file2.libsonnet new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/local-file2.libsonnet @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet new file mode 100644 index 000000000..8faea36e4 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-locals-and-vendored/main.jsonnet @@ -0,0 +1,5 @@ +{ + attr1: import 'local-file1.libsonnet', + attr2: import '././../imports-locals-and-vendored/./local-file2.libsonnet', + vendored1: import 'vendored/main.libsonnet', +} diff --git a/pkg/jsonnet/testdata/findImporters/environments/imports-symlinked-vendor/main.jsonnet b/pkg/jsonnet/testdata/findImporters/environments/imports-symlinked-vendor/main.jsonnet new file mode 100644 index 000000000..da56ba9ea --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/imports-symlinked-vendor/main.jsonnet @@ -0,0 +1,3 @@ +{ + symlinked: import 'vendor-symlinked/main.libsonnet', +} diff --git a/pkg/jsonnet/testdata/findImporters/environments/no-imports/main.jsonnet b/pkg/jsonnet/testdata/findImporters/environments/no-imports/main.jsonnet new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/environments/no-imports/main.jsonnet @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/findImporters/lib/lib1/main.libsonnet b/pkg/jsonnet/testdata/findImporters/lib/lib1/main.libsonnet new file mode 100644 index 000000000..635fab2c2 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/lib/lib1/main.libsonnet @@ -0,0 +1,3 @@ +{ + vendored: import 'vendored/main.libsonnet', +} diff --git a/pkg/jsonnet/testdata/findImporters/lib/lib2/main.libsonnet b/pkg/jsonnet/testdata/findImporters/lib/lib2/main.libsonnet new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/lib/lib2/main.libsonnet @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/findImporters/lib/unimported-lib/main.libsonnet b/pkg/jsonnet/testdata/findImporters/lib/unimported-lib/main.libsonnet new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/lib/unimported-lib/main.libsonnet @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/findImporters/tkrc.yaml b/pkg/jsonnet/testdata/findImporters/tkrc.yaml new file mode 100644 index 000000000..6cb4483b5 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/tkrc.yaml @@ -0,0 +1 @@ +# This is the root! \ No newline at end of file diff --git a/pkg/jsonnet/testdata/findImporters/vendor/unimported-vendor/main.libsonnet b/pkg/jsonnet/testdata/findImporters/vendor/unimported-vendor/main.libsonnet new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/vendor/unimported-vendor/main.libsonnet @@ -0,0 +1 @@ +{} diff --git a/pkg/jsonnet/testdata/findImporters/vendor/vendor-symlinked b/pkg/jsonnet/testdata/findImporters/vendor/vendor-symlinked new file mode 120000 index 000000000..0bb55907d --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/vendor/vendor-symlinked @@ -0,0 +1 @@ +vendored \ No newline at end of file diff --git a/pkg/jsonnet/testdata/findImporters/vendor/vendored/main.libsonnet b/pkg/jsonnet/testdata/findImporters/vendor/vendored/main.libsonnet new file mode 100644 index 000000000..fd71b56df --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/vendor/vendored/main.libsonnet @@ -0,0 +1,3 @@ +{ + text: importstr 'text-file.txt', +} diff --git a/pkg/jsonnet/testdata/findImporters/vendor/vendored/text-file.txt b/pkg/jsonnet/testdata/findImporters/vendor/vendored/text-file.txt new file mode 100644 index 000000000..e87177c96 --- /dev/null +++ b/pkg/jsonnet/testdata/findImporters/vendor/vendored/text-file.txt @@ -0,0 +1 @@ +I am text \ No newline at end of file