From 6a4f83fd1a450e6bd90ca96b652bb7c3157ea514 Mon Sep 17 00:00:00 2001 From: Julien Duchesne Date: Thu, 24 Aug 2023 13:53:35 -0400 Subject: [PATCH] Hide the Jsonnet Go implementation behind an interface (#914) * Hide the Jsonnet Go implementation behind an interface Tanka currently evals jsonnet using the Go native code. However, some other implementations have come up in the past years that could be worth using (ex: https://github.com/CertainLach/jrsonnet, which is much faster) In this PR is the first step: I create an interface where all the jsonnet eval code happens. The Go Jsonnet implementation is now hidden behind this interface. The setting can either be passed as a global flag or as an env spec attribute to be used when exporting (`spec.exportJsonnetImplementation`) * PR review changes `VM` -> `Evaluator` Add some docstrings Add some comments * Select the Jsonnet implementation when constructing the loader Add the `path` as a struct member of the Go-jsonnet implementation. We have this available when we construct the loader. This lets us then drop the same argument from `EvaluateAnonymousSnippet()` in the interface - as not all implementations need it. --------- Co-authored-by: Iain Lane --- cmd/tk/flags.go | 8 ++- pkg/jsonnet/eval.go | 71 +++++++------------ pkg/jsonnet/eval_test.go | 19 ++--- pkg/jsonnet/find_importers_test.go | 2 +- pkg/jsonnet/implementations/goimpl/impl.go | 32 +++++++++ .../{ => implementations/goimpl}/importer.go | 14 ++-- .../goimpl}/tk.libsonnet.go | 2 +- pkg/jsonnet/implementations/goimpl/vm.go | 33 +++++++++ pkg/jsonnet/implementations/types/types.go | 13 ++++ pkg/jsonnet/imports.go | 9 +-- pkg/jsonnet/imports_test.go | 3 +- pkg/jsonnet/jpath/jpath_test.go | 5 +- pkg/jsonnet/lint.go | 4 +- pkg/process/data_test.go | 6 +- pkg/spec/v1alpha1/environment.go | 17 ++--- pkg/tanka/evaluators.go | 9 +-- pkg/tanka/evaluators_test.go | 7 +- pkg/tanka/inline.go | 7 +- pkg/tanka/load.go | 36 ++++++++-- pkg/tanka/parallel.go | 4 ++ pkg/tanka/static.go | 7 +- 21 files changed, 201 insertions(+), 107 deletions(-) create mode 100644 pkg/jsonnet/implementations/goimpl/impl.go rename pkg/jsonnet/{ => implementations/goimpl}/importer.go (87%) rename pkg/jsonnet/{ => implementations/goimpl}/tk.libsonnet.go (89%) create mode 100644 pkg/jsonnet/implementations/goimpl/vm.go create mode 100644 pkg/jsonnet/implementations/types/types.go diff --git a/cmd/tk/flags.go b/cmd/tk/flags.go index 79c32df88..4d77bbb77 100644 --- a/cmd/tk/flags.go +++ b/cmd/tk/flags.go @@ -58,12 +58,14 @@ func labelSelectorFlag(fs *pflag.FlagSet) func() labels.Selector { func jsonnetFlags(fs *pflag.FlagSet) func() tanka.JsonnetOpts { getExtCode, getTLACode := cliCodeParser(fs) maxStack := fs.Int("max-stack", 0, "Jsonnet VM max stack. The default value is the value set in the go-jsonnet library. Increase this if you get: max stack frames exceeded") + jsonnetImplementation := fs.String("jsonnet-implementation", "go", "Only go is supported for now.") return func() tanka.JsonnetOpts { return tanka.JsonnetOpts{ - MaxStack: *maxStack, - ExtCode: getExtCode(), - TLACode: getTLACode(), + MaxStack: *maxStack, + ExtCode: getExtCode(), + TLACode: getTLACode(), + JsonnetImplementation: *jsonnetImplementation, } } } diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go index cbf647801..d9e9b71ad 100644 --- a/pkg/jsonnet/eval.go +++ b/pkg/jsonnet/eval.go @@ -9,8 +9,9 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" + "github.com/grafana/tanka/pkg/jsonnet/implementations/types" "github.com/grafana/tanka/pkg/jsonnet/jpath" - "github.com/grafana/tanka/pkg/jsonnet/native" ) // Modifier allows to set optional parameters on the Jsonnet VM. @@ -31,12 +32,13 @@ func (i *InjectedCode) Set(key, value string) { // Opts are additional properties for the Jsonnet VM type Opts struct { - MaxStack int - ExtCode InjectedCode - TLACode InjectedCode - ImportPaths []string - EvalScript string - CachePath string + JsonnetImplementation string + MaxStack int + ExtCode InjectedCode + TLACode InjectedCode + ImportPaths []string + EvalScript string + CachePath string CachePathRegexes []*regexp.Regexp } @@ -75,76 +77,51 @@ func (o Opts) Clone() Opts { } } -// MakeVM returns a Jsonnet VM with some extensions of Tanka, including: -// - extended importer -// - extCode and tlaCode applied -// - native functions registered -func MakeVM(opts Opts) *jsonnet.VM { - vm := jsonnet.MakeVM() - vm.Importer(NewExtendedImporter(opts.ImportPaths)) - - for k, v := range opts.ExtCode { - vm.ExtCode(k, v) - } - for k, v := range opts.TLACode { - vm.TLACode(k, v) - } - - for _, nf := range native.Funcs() { - vm.NativeFunction(nf) - } - - if opts.MaxStack > 0 { - vm.MaxStack = opts.MaxStack - } - - return vm -} - // EvaluateFile evaluates the Jsonnet code in the given file and returns the // result in JSON form. It disregards opts.ImportPaths in favor of automatically // resolving these according to the specified file. -func EvaluateFile(jsonnetFile string, opts Opts) (string, error) { - evalFunc := func(vm *jsonnet.VM) (string, error) { - return vm.EvaluateFile(jsonnetFile) +func EvaluateFile(impl types.JsonnetImplementation, jsonnetFile string, opts Opts) (string, error) { + evalFunc := func(evaluator types.JsonnetEvaluator) (string, error) { + return evaluator.EvaluateFile(jsonnetFile) } data, err := os.ReadFile(jsonnetFile) if err != nil { return "", err } - return evaluateSnippet(evalFunc, jsonnetFile, string(data), opts) + return evaluateSnippet(impl, evalFunc, jsonnetFile, string(data), opts) } // Evaluate renders the given jsonnet into a string // If cache options are given, a hash from the data will be computed and // the resulting string will be cached for future retrieval -func Evaluate(path, data string, opts Opts) (string, error) { - evalFunc := func(vm *jsonnet.VM) (string, error) { - return vm.EvaluateAnonymousSnippet(path, data) +func Evaluate(path string, impl types.JsonnetImplementation, data string, opts Opts) (string, error) { + evalFunc := func(evaluator types.JsonnetEvaluator) (string, error) { + return evaluator.EvaluateAnonymousSnippet(data) } - return evaluateSnippet(evalFunc, path, data, opts) + return evaluateSnippet(impl, evalFunc, path, data, opts) } -type evalFunc func(vm *jsonnet.VM) (string, error) +type evalFunc func(evaluator types.JsonnetEvaluator) (string, error) -func evaluateSnippet(evalFunc evalFunc, path, data string, opts Opts) (string, error) { +func evaluateSnippet(jsonnetImpl types.JsonnetImplementation, evalFunc evalFunc, path, data string, opts Opts) (string, error) { var cache *FileEvalCache if opts.CachePath != "" && opts.PathIsCached(path) { cache = NewFileEvalCache(opts.CachePath) } - // Create VM jpath, _, _, err := jpath.Resolve(path, false) if err != nil { return "", errors.Wrap(err, "resolving import paths") } opts.ImportPaths = jpath - vm := MakeVM(opts) + evaluator := jsonnetImpl.MakeEvaluator(opts.ImportPaths, opts.ExtCode, opts.TLACode, opts.MaxStack) + // We're using the go implementation to deal with imports because we're not evaluating, we're reading the AST + importVM := goimpl.MakeRawVM(opts.ImportPaths, opts.ExtCode, opts.TLACode, opts.MaxStack) var hash string if cache != nil { startTime := time.Now() - if hash, err = getSnippetHash(vm, path, data); err != nil { + if hash, err = getSnippetHash(importVM, path, data); err != nil { return "", err } cacheLog := log.Debug().Str("path", path).Str("hash", hash).Dur("duration_ms", time.Since(startTime)) @@ -157,7 +134,7 @@ func evaluateSnippet(evalFunc evalFunc, path, data string, opts Opts) (string, e cacheLog.Bool("cache_hit", false).Msg("computed snippet hash") } - content, err := evalFunc(vm) + content, err := evalFunc(evaluator) if err != nil { return "", err } diff --git a/pkg/jsonnet/eval_test.go b/pkg/jsonnet/eval_test.go index 13a83b8e8..db15b3def 100644 --- a/pkg/jsonnet/eval_test.go +++ b/pkg/jsonnet/eval_test.go @@ -5,10 +5,13 @@ import ( "path/filepath" "testing" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var jsonnetImpl = &goimpl.JsonnetGoImplementation{} + const importTreeResult = `[ { "breed": "apple", @@ -51,13 +54,13 @@ const thisFileResult = `{ // To be consistent with the jsonnet executable, // when evaluating a file, `std.thisFile` should point to the given path func TestEvaluateFile(t *testing.T) { - result, err := EvaluateFile("testdata/thisFile/main.jsonnet", Opts{}) + result, err := EvaluateFile(jsonnetImpl, "testdata/thisFile/main.jsonnet", Opts{}) assert.NoError(t, err) assert.Equal(t, thisFileResult, result) } func TestEvaluateFileDoesntExist(t *testing.T) { - result, err := EvaluateFile("testdata/doesnt-exist/main.jsonnet", Opts{}) + result, err := EvaluateFile(jsonnetImpl, "testdata/doesnt-exist/main.jsonnet", Opts{}) assert.EqualError(t, err, "open testdata/doesnt-exist/main.jsonnet: no such file or directory") assert.Equal(t, "", result) } @@ -69,10 +72,10 @@ func TestEvaluateFileWithCaching(t *testing.T) { cachePath := filepath.Join(tmp, "cache") // Should be created during caching // Evaluate two files - result, err := EvaluateFile("testdata/thisFile/main.jsonnet", Opts{CachePath: cachePath}) + result, err := EvaluateFile(jsonnetImpl, "testdata/thisFile/main.jsonnet", Opts{CachePath: cachePath}) assert.NoError(t, err) assert.Equal(t, thisFileResult, result) - result, err = EvaluateFile("testdata/importTree/main.jsonnet", Opts{CachePath: cachePath}) + result, err = EvaluateFile(jsonnetImpl, "testdata/importTree/main.jsonnet", Opts{CachePath: cachePath}) assert.NoError(t, err) assert.Equal(t, importTreeResult, result) @@ -82,10 +85,10 @@ func TestEvaluateFileWithCaching(t *testing.T) { assert.Len(t, readCache, 2) // Evaluate two files again, same result - result, err = EvaluateFile("testdata/thisFile/main.jsonnet", Opts{CachePath: cachePath}) + result, err = EvaluateFile(jsonnetImpl, "testdata/thisFile/main.jsonnet", Opts{CachePath: cachePath}) assert.NoError(t, err) assert.Equal(t, thisFileResult, result) - result, err = EvaluateFile("testdata/importTree/main.jsonnet", Opts{CachePath: cachePath}) + result, err = EvaluateFile(jsonnetImpl, "testdata/importTree/main.jsonnet", Opts{CachePath: cachePath}) assert.NoError(t, err) assert.Equal(t, importTreeResult, result) @@ -95,10 +98,10 @@ func TestEvaluateFileWithCaching(t *testing.T) { } // Evaluate two files again, modified cache is returned instead of the actual result - result, err = EvaluateFile("testdata/thisFile/main.jsonnet", Opts{CachePath: cachePath}) + result, err = EvaluateFile(jsonnetImpl, "testdata/thisFile/main.jsonnet", Opts{CachePath: cachePath}) assert.NoError(t, err) assert.Equal(t, "BYfdlr1ZOVwiOfbd89JYTcK-eRQh05bi8ky3k1vVW5o=.json", result) - result, err = EvaluateFile("testdata/importTree/main.jsonnet", Opts{CachePath: cachePath}) + result, err = EvaluateFile(jsonnetImpl, "testdata/importTree/main.jsonnet", Opts{CachePath: cachePath}) assert.NoError(t, err) assert.Equal(t, "R_3hy-dRfOwXN-fezQ50ZF4dnrFcBcbQ9LztR_XWzJA=.json", result) } diff --git a/pkg/jsonnet/find_importers_test.go b/pkg/jsonnet/find_importers_test.go index 8f3765059..147a5955b 100644 --- a/pkg/jsonnet/find_importers_test.go +++ b/pkg/jsonnet/find_importers_test.go @@ -228,7 +228,7 @@ func TestFindImportersForFiles(t *testing.T) { if filepath.Base(file) != jpath.DefaultEntrypoint { continue } - _, err := EvaluateFile(file, Opts{}) + _, err := EvaluateFile(jsonnetImpl, file, Opts{}) require.NoError(t, err, "failed to eval %s", file) } diff --git a/pkg/jsonnet/implementations/goimpl/impl.go b/pkg/jsonnet/implementations/goimpl/impl.go new file mode 100644 index 000000000..53e1a5869 --- /dev/null +++ b/pkg/jsonnet/implementations/goimpl/impl.go @@ -0,0 +1,32 @@ +package goimpl + +import ( + "github.com/google/go-jsonnet" + "github.com/grafana/tanka/pkg/jsonnet/implementations/types" +) + +type JsonnetGoVM struct { + vm *jsonnet.VM + + path string +} + +func (vm *JsonnetGoVM) EvaluateAnonymousSnippet(snippet string) (string, error) { + return vm.vm.EvaluateAnonymousSnippet(vm.path, snippet) +} + +func (vm *JsonnetGoVM) EvaluateFile(filename string) (string, error) { + return vm.vm.EvaluateFile(filename) +} + +type JsonnetGoImplementation struct { + Path string +} + +func (i *JsonnetGoImplementation) MakeEvaluator(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int) types.JsonnetEvaluator { + return &JsonnetGoVM{ + vm: MakeRawVM(importPaths, extCode, tlaCode, maxStack), + + path: i.Path, + } +} diff --git a/pkg/jsonnet/importer.go b/pkg/jsonnet/implementations/goimpl/importer.go similarity index 87% rename from pkg/jsonnet/importer.go rename to pkg/jsonnet/implementations/goimpl/importer.go index c8b45113d..70ced2b0e 100644 --- a/pkg/jsonnet/importer.go +++ b/pkg/jsonnet/implementations/goimpl/importer.go @@ -1,4 +1,4 @@ -package jsonnet +package goimpl import ( "path/filepath" @@ -8,10 +8,10 @@ import ( const locationInternal = "" -// ExtendedImporter wraps jsonnet.FileImporter to add additional functionality: +// extendedImporter wraps jsonnet.FileImporter to add additional functionality: // - `import "file.yaml"` // - `import "tk"` -type ExtendedImporter struct { +type extendedImporter struct { loaders []importLoader // for loading jsonnet from somewhere. First one that returns non-nil is used processors []importProcessor // for post-processing (e.g. yaml -> json) } @@ -24,10 +24,10 @@ type importLoader func(importedFrom, importedPath string) (c *jsonnet.Contents, // further type importProcessor func(contents, foundAt string) (c *jsonnet.Contents, err error) -// NewExtendedImporter returns a new instance of ExtendedImporter with the +// newExtendedImporter returns a new instance of ExtendedImporter with the // correct jpaths set up -func NewExtendedImporter(jpath []string) *ExtendedImporter { - return &ExtendedImporter{ +func newExtendedImporter(jpath []string) *extendedImporter { + return &extendedImporter{ loaders: []importLoader{ tkLoader, newFileLoader(&jsonnet.FileImporter{ @@ -38,7 +38,7 @@ func NewExtendedImporter(jpath []string) *ExtendedImporter { } // Import implements the functionality offered by the ExtendedImporter -func (i *ExtendedImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { +func (i *extendedImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { // load using loader for _, loader := range i.loaders { c, f, err := loader(importedFrom, importedPath) diff --git a/pkg/jsonnet/tk.libsonnet.go b/pkg/jsonnet/implementations/goimpl/tk.libsonnet.go similarity index 89% rename from pkg/jsonnet/tk.libsonnet.go rename to pkg/jsonnet/implementations/goimpl/tk.libsonnet.go index 9c2780fce..260f9c08b 100644 --- a/pkg/jsonnet/tk.libsonnet.go +++ b/pkg/jsonnet/implementations/goimpl/tk.libsonnet.go @@ -1,4 +1,4 @@ -package jsonnet +package goimpl import jsonnet "github.com/google/go-jsonnet" diff --git a/pkg/jsonnet/implementations/goimpl/vm.go b/pkg/jsonnet/implementations/goimpl/vm.go new file mode 100644 index 000000000..cafb79d7b --- /dev/null +++ b/pkg/jsonnet/implementations/goimpl/vm.go @@ -0,0 +1,33 @@ +package goimpl + +import ( + "github.com/google/go-jsonnet" + "github.com/grafana/tanka/pkg/jsonnet/native" +) + +// MakeRawVM returns a Jsonnet VM with some extensions of Tanka, including: +// - extended importer +// - extCode and tlaCode applied +// - native functions registered +// This is exposed because Go is used for advanced use cases, like finding transitive imports or linting. +func MakeRawVM(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int) *jsonnet.VM { + vm := jsonnet.MakeVM() + vm.Importer(newExtendedImporter(importPaths)) + + for k, v := range extCode { + vm.ExtCode(k, v) + } + for k, v := range tlaCode { + vm.TLACode(k, v) + } + + for _, nf := range native.Funcs() { + vm.NativeFunction(nf) + } + + if maxStack > 0 { + vm.MaxStack = maxStack + } + + return vm +} diff --git a/pkg/jsonnet/implementations/types/types.go b/pkg/jsonnet/implementations/types/types.go new file mode 100644 index 000000000..36d80255c --- /dev/null +++ b/pkg/jsonnet/implementations/types/types.go @@ -0,0 +1,13 @@ +package types + +// JsonnetEvaluator represents a struct that can evaluate Jsonnet code +// It is configured with import paths, external code and top-level arguments +type JsonnetEvaluator interface { + EvaluateAnonymousSnippet(snippet string) (string, error) + EvaluateFile(filename string) (string, error) +} + +// JsonnetImplementation is a factory for JsonnetEvaluator +type JsonnetImplementation interface { + MakeEvaluator(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int) JsonnetEvaluator +} diff --git a/pkg/jsonnet/imports.go b/pkg/jsonnet/imports.go index cbe2acb2a..59c27171d 100644 --- a/pkg/jsonnet/imports.go +++ b/pkg/jsonnet/imports.go @@ -15,8 +15,8 @@ import ( "github.com/google/go-jsonnet/toolutils" "github.com/pkg/errors" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/grafana/tanka/pkg/jsonnet/jpath" - "github.com/grafana/tanka/pkg/jsonnet/native" ) var importsRegexp = regexp.MustCompile(`import(str)?\s+['"]([^'"%()]+)['"]`) @@ -48,12 +48,7 @@ func TransitiveImports(dir string) ([]string, error) { return nil, errors.Wrap(err, "resolving JPATH") } - vm := jsonnet.MakeVM() - vm.Importer(NewExtendedImporter(jpath)) - for _, nf := range native.Funcs() { - vm.NativeFunction(nf) - } - + vm := goimpl.MakeRawVM(jpath, nil, nil, 0) node, err := jsonnet.SnippetToAST(filepath.Base(entrypoint), string(sonnet)) if err != nil { return nil, errors.Wrap(err, "creating Jsonnet AST") diff --git a/pkg/jsonnet/imports_test.go b/pkg/jsonnet/imports_test.go index 7c7787d32..24e0ccd26 100644 --- a/pkg/jsonnet/imports_test.go +++ b/pkg/jsonnet/imports_test.go @@ -8,6 +8,7 @@ import ( "sync" "testing" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -53,7 +54,7 @@ func BenchmarkGetSnippetHash(b *testing.B) { // Create a VM. It's important to reuse the same VM // While there is a caching mechanism that normally shouldn't be shared in a benchmark iteration, // it's useful to evaluate its impact here, because the caching will also improve the evaluation performance afterwards. - vm := MakeVM(Opts{ImportPaths: []string{tempDir}}) + vm := goimpl.MakeRawVM([]string{tempDir}, nil, nil, 0) content, err := os.ReadFile(filepath.Join(tempDir, "main.jsonnet")) require.NoError(b, err) diff --git a/pkg/jsonnet/jpath/jpath_test.go b/pkg/jsonnet/jpath/jpath_test.go index 0c21d8ffd..31e31c33e 100644 --- a/pkg/jsonnet/jpath/jpath_test.go +++ b/pkg/jsonnet/jpath/jpath_test.go @@ -8,10 +8,13 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/tanka/pkg/jsonnet" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" ) +var jsonnetImpl = &goimpl.JsonnetGoImplementation{} + func TestResolvePrecedence(t *testing.T) { - s, err := jsonnet.EvaluateFile("./testdata/precedence/environments/default/main.jsonnet", jsonnet.Opts{}) + s, err := jsonnet.EvaluateFile(jsonnetImpl, "./testdata/precedence/environments/default/main.jsonnet", jsonnet.Opts{}) require.NoError(t, err) want := map[string]string{ diff --git a/pkg/jsonnet/lint.go b/pkg/jsonnet/lint.go index d44effa7d..98989ad90 100644 --- a/pkg/jsonnet/lint.go +++ b/pkg/jsonnet/lint.go @@ -10,6 +10,7 @@ import ( "github.com/gobwas/glob" "github.com/google/go-jsonnet/linter" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -106,14 +107,13 @@ func lintWithRecover(file string) (buf bytes.Buffer, success bool) { return } - vm := MakeVM(Opts{}) jpaths, _, _, err := jpath.Resolve(file, true) if err != nil { fmt.Fprintf(&buf, "got an error getting jpath for %s: %v\n\n", file, err) return } + vm := goimpl.MakeRawVM(jpaths, nil, nil, 0) - vm.Importer(NewExtendedImporter(jpaths)) failed := linter.LintSnippet(vm, &buf, []linter.Snippet{{FileName: file, Code: string(content)}}) return buf, !failed } diff --git a/pkg/process/data_test.go b/pkg/process/data_test.go index ea226570b..89e8e7024 100644 --- a/pkg/process/data_test.go +++ b/pkg/process/data_test.go @@ -5,7 +5,7 @@ import ( "fmt" "path/filepath" - "github.com/grafana/tanka/pkg/jsonnet" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/grafana/tanka/pkg/kubernetes/manifest" ) @@ -18,9 +18,7 @@ type testData struct { func loadFixture(name string) testData { filename := filepath.Join("./testdata", name) - vm := jsonnet.MakeVM(jsonnet.Opts{ - ImportPaths: []string{"./testdata"}, - }) + vm := goimpl.MakeRawVM([]string{"./testdata"}, nil, nil, 0) data, err := vm.EvaluateFile(filename) if err != nil { diff --git a/pkg/spec/v1alpha1/environment.go b/pkg/spec/v1alpha1/environment.go index a477e974c..0a8ce4329 100644 --- a/pkg/spec/v1alpha1/environment.go +++ b/pkg/spec/v1alpha1/environment.go @@ -57,14 +57,15 @@ func (m Metadata) NameLabel() string { // Spec defines Kubernetes properties type Spec struct { - APIServer string `json:"apiServer,omitempty"` - ContextNames []string `json:"contextNames,omitempty"` - Namespace string `json:"namespace"` - DiffStrategy string `json:"diffStrategy,omitempty"` - ApplyStrategy string `json:"applyStrategy,omitempty"` - InjectLabels bool `json:"injectLabels,omitempty"` - ResourceDefaults ResourceDefaults `json:"resourceDefaults"` - ExpectVersions ExpectVersions `json:"expectVersions"` + APIServer string `json:"apiServer,omitempty"` + ContextNames []string `json:"contextNames,omitempty"` + Namespace string `json:"namespace"` + DiffStrategy string `json:"diffStrategy,omitempty"` + ApplyStrategy string `json:"applyStrategy,omitempty"` + InjectLabels bool `json:"injectLabels,omitempty"` + ResourceDefaults ResourceDefaults `json:"resourceDefaults"` + ExpectVersions ExpectVersions `json:"expectVersions"` + ExportJsonnetImplementation string `json:"exportJsonnetImplementation,omitempty"` } // ExpectVersions holds semantic version constraints diff --git a/pkg/tanka/evaluators.go b/pkg/tanka/evaluators.go index 64c761cb6..e8ed91980 100644 --- a/pkg/tanka/evaluators.go +++ b/pkg/tanka/evaluators.go @@ -7,11 +7,12 @@ import ( "github.com/pkg/errors" "github.com/grafana/tanka/pkg/jsonnet" + "github.com/grafana/tanka/pkg/jsonnet/implementations/types" "github.com/grafana/tanka/pkg/jsonnet/jpath" ) // EvalJsonnet evaluates the jsonnet environment at the given file system path -func evalJsonnet(path string, opts jsonnet.Opts) (raw string, err error) { +func evalJsonnet(path string, impl types.JsonnetImplementation, opts jsonnet.Opts) (raw string, err error) { entrypoint, err := jpath.Entrypoint(path) if err != nil { return "", err @@ -37,14 +38,14 @@ function(%s) `, tlaJoin, entrypoint, tlaJoin, opts.EvalScript) } - raw, err = jsonnet.Evaluate(path, evalScript, opts) + raw, err = jsonnet.Evaluate(path, impl, evalScript, opts) if err != nil { - return "", errors.Wrap(err, "evaluating jsonnet") + return "", fmt.Errorf("evaluating jsonnet in path '%s': %w", path, err) } return raw, nil } - raw, err = jsonnet.EvaluateFile(entrypoint, opts) + raw, err = jsonnet.EvaluateFile(impl, entrypoint, opts) if err != nil { return "", errors.Wrap(err, "evaluating jsonnet") } diff --git a/pkg/tanka/evaluators_test.go b/pkg/tanka/evaluators_test.go index e5767b089..f6ce72417 100644 --- a/pkg/tanka/evaluators_test.go +++ b/pkg/tanka/evaluators_test.go @@ -5,9 +5,12 @@ import ( "testing" "github.com/grafana/tanka/pkg/jsonnet" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/stretchr/testify/assert" ) +var jsonnetImpl = &goimpl.JsonnetGoImplementation{} + func TestEvalJsonnet(t *testing.T) { var tlaCode jsonnet.InjectedCode // Pass in the mandatory parameters as TLA codes, but note that only `foo` @@ -27,7 +30,7 @@ func TestEvalJsonnet(t *testing.T) { // This will fail intermittently if TLAs are passed as positional // parameters. - json, err := evalJsonnet("testdata/cases/withtlas", opts) + json, err := evalJsonnet("testdata/cases/withtlas", jsonnetImpl, opts) assert.NoError(t, err) assert.Equal(t, `"foovalue"`, strings.TrimSpace(json)) } @@ -43,7 +46,7 @@ func TestEvalJsonnetWithExpression(t *testing.T) { // This will fail intermittently if TLAs are passed as positional // parameters. - json, err := evalJsonnet("testdata/cases/object", opts) + json, err := evalJsonnet("testdata/cases/object", jsonnetImpl, opts) assert.NoError(t, err) assert.Equal(t, `"object"`, strings.TrimSpace(json)) }) diff --git a/pkg/tanka/inline.go b/pkg/tanka/inline.go index 71f5182a2..c50532d10 100644 --- a/pkg/tanka/inline.go +++ b/pkg/tanka/inline.go @@ -6,6 +6,7 @@ import ( "path/filepath" "sort" + "github.com/grafana/tanka/pkg/jsonnet/implementations/types" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/kubernetes/manifest" "github.com/grafana/tanka/pkg/process" @@ -16,7 +17,9 @@ import ( // InlineLoader loads an environment that is specified inline from within // Jsonnet. The Jsonnet output is expected to hold a tanka.dev/Environment type, // Kubernetes resources are expected at the `data` key of this very type -type InlineLoader struct{} +type InlineLoader struct { + jsonnetImpl types.JsonnetImplementation +} func (i *InlineLoader) Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { if opts.Name != "" { @@ -110,7 +113,7 @@ func (i *InlineLoader) Eval(path string, opts LoaderOpts) (interface{}, error) { // Can't provide env as extVar, as we need to evaluate Jsonnet first to know it opts.ExtCode.Set(environmentExtCode, `error "Using tk.env and std.extVar('tanka.dev/environment') is only supported for static environments. Directly access this data using standard Jsonnet instead."`) - raw, err := evalJsonnet(path, opts.JsonnetOpts) + raw, err := evalJsonnet(path, i.jsonnetImpl, opts.JsonnetOpts) if err != nil { return nil, err } diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index 31fd0a921..21b635d98 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -5,6 +5,8 @@ import ( "os" "path/filepath" + "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" + "github.com/grafana/tanka/pkg/jsonnet/implementations/types" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/kubernetes" "github.com/grafana/tanka/pkg/kubernetes/manifest" @@ -51,7 +53,7 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { return nil, err } - loader, err := DetectLoader(path) + loader, err := DetectLoader(path, opts) if err != nil { return nil, err } @@ -80,7 +82,7 @@ func LoadManifests(env *v1alpha1.Environment, filters process.Matchers) (*LoadRe // Peek loads the metadata of the environment at path. To get resources as well, // use Load func Peek(path string, opts Opts) (*v1alpha1.Environment, error) { - loader, err := DetectLoader(path) + loader, err := DetectLoader(path, opts) if err != nil { return nil, err } @@ -92,7 +94,7 @@ func Peek(path string, opts Opts) (*v1alpha1.Environment, error) { // loaded. List can be used to deal with multiple inline environments, by first // listing them, choosing the right one and then only loading that one func List(path string, opts Opts) ([]*v1alpha1.Environment, error) { - loader, err := DetectLoader(path) + loader, err := DetectLoader(path, opts) if err != nil { return nil, err } @@ -100,9 +102,20 @@ func List(path string, opts Opts) ([]*v1alpha1.Environment, error) { return loader.List(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) } +func getJsonnetImplementation(path string, opts Opts) (types.JsonnetImplementation, error) { + switch opts.JsonnetOpts.JsonnetImplementation { + case "go", "": + return &goimpl.JsonnetGoImplementation{ + Path: path, + }, nil + default: + return nil, fmt.Errorf("unknown jsonnet implementation: %s", opts.JsonnetOpts.JsonnetImplementation) + } +} + // Eval returns the raw evaluated Jsonnet func Eval(path string, opts Opts) (interface{}, error) { - loader, err := DetectLoader(path) + loader, err := DetectLoader(path, opts) if err != nil { return nil, err } @@ -112,21 +125,30 @@ func Eval(path string, opts Opts) (interface{}, error) { // DetectLoader detects whether the environment is inline or static and picks // the approriate loader -func DetectLoader(path string) (Loader, error) { +func DetectLoader(path string, opts Opts) (Loader, error) { _, base, err := jpath.Dirs(path) if err != nil { return nil, err } + jsonnetImpl, err := getJsonnetImplementation(base, opts) + if err != nil { + return nil, err + } + // check if spec.json exists _, err = os.Stat(filepath.Join(base, spec.Specfile)) if os.IsNotExist(err) { - return &InlineLoader{}, nil + return &InlineLoader{ + jsonnetImpl: jsonnetImpl, + }, nil } else if err != nil { return nil, err } - return &StaticLoader{}, nil + return &StaticLoader{ + jsonnetImpl: jsonnetImpl, + }, nil } // Loader is an abstraction over the process of loading Environments diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index c7bba5350..123173cc7 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -49,6 +49,10 @@ func parallelLoadEnvironments(envs []*v1alpha1.Environment, opts parallelOpts) ( // to Tanka workflow thus being able to handle such cases o.JsonnetOpts = o.JsonnetOpts.Clone() + if o.JsonnetOpts.JsonnetImplementation == "" { + o.JsonnetOpts.JsonnetImplementation = env.Spec.ExportJsonnetImplementation + } + o.Name = env.Metadata.Name path := env.Metadata.Namespace rootDir, err := jpath.FindRoot(path) diff --git a/pkg/tanka/static.go b/pkg/tanka/static.go index 68c677e97..d978b283c 100644 --- a/pkg/tanka/static.go +++ b/pkg/tanka/static.go @@ -3,6 +3,7 @@ package tanka import ( "encoding/json" + "github.com/grafana/tanka/pkg/jsonnet/implementations/types" "github.com/grafana/tanka/pkg/spec" "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/rs/zerolog/log" @@ -10,7 +11,9 @@ import ( // StaticLoader loads an environment from a static file called `spec.json`. // Jsonnet is evaluated as normal -type StaticLoader struct{} +type StaticLoader struct { + jsonnetImpl types.JsonnetImplementation +} func (s StaticLoader) Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { config, err := s.Peek(path, opts) @@ -57,7 +60,7 @@ func (s *StaticLoader) Eval(path string, opts LoaderOpts) (interface{}, error) { } opts.ExtCode.Set(environmentExtCode, envCode) - raw, err := evalJsonnet(path, opts.JsonnetOpts) + raw, err := evalJsonnet(path, s.jsonnetImpl, opts.JsonnetOpts) if err != nil { return nil, err }