feat: Import post-processing

Adds a second `interface` for importing, `ImportProcessor`. If set on a
VM (non-nil), it is asked to perhaps convert the Importer result before
importCache stores it.

This can for example be used to enhance what `import` is capable of, so
that it also reads `.yaml` files, etc.

This is a feature we once had in Tanka, but had to disable as we were
not able to differentiate between `import` and `importstr` based on the
`Importer` interface. This change allows us to do that, as
`ImportProcessor` is only ever considered for `import`.

Author: tombrk

imports.go

@@ -50,6 +50,13 @@ type Importer interface {
 	Import(importedFrom, importedPath string) (contents Contents, foundAt string, err error)
 }
 
+// ImportProcessor allows to dynamically modify the result returned by the
+// Importer, for example to convert other formats to a Jsonnet snippet before
+// evaluating.
+type ImportProcessor interface {
+	Process(contents Contents, foundAt string) (Contents, error)
+}
+
 // Contents is a representation of imported data. It is a simple
 // string wrapper, which makes it easier to enforce the caching policy.
 type Contents struct {
@@ -79,12 +86,14 @@ type importCache struct {
 	astCache            map[string]ast.Node
 	codeCache           map[string]potentialValue
 	importer            Importer
+	processor           ImportProcessor
 }
 
 // makeImportCache creates an importCache using an Importer.
-func makeImportCache(importer Importer) *importCache {
+func makeImportCache(importer Importer, processor ImportProcessor) *importCache {
 	return &importCache{
 		importer:            importer,
+		processor:           processor,
 		foundAtVerification: make(map[string]Contents),
 		astCache:            make(map[string]ast.Node),
 		codeCache:           make(map[string]potentialValue),
@@ -115,9 +124,19 @@ func (cache *importCache) importAST(importedFrom, importedPath string) (ast.Node
 	if err != nil {
 		return nil, "", err
 	}
+
 	if cachedNode, isCached := cache.astCache[foundAt]; isCached {
 		return cachedNode, foundAt, nil
 	}
+
+	if cache.processor != nil {
+		c, err := cache.processor.Process(contents, foundAt)
+		if err != nil {
+			return nil, "", err
+		}
+		contents = c
+	}
+
 	node, err := program.SnippetToAST(foundAt, contents.String())
 	cache.astCache[foundAt] = node
 	return node, foundAt, err

vm.go

@@ -31,14 +31,15 @@ import (
 // VM is the core interpreter and is the touchpoint used to parse and execute
 // Jsonnet.
 type VM struct {
-	MaxStack       int
-	ext            vmExtMap
-	tla            vmExtMap
-	nativeFuncs    map[string]*NativeFunction
-	importer       Importer
-	ErrorFormatter ErrorFormatter
-	StringOutput   bool
-	importCache    *importCache
+	MaxStack        int
+	ext             vmExtMap
+	tla             vmExtMap
+	nativeFuncs     map[string]*NativeFunction
+	importer        Importer
+	importProcessor ImportProcessor
+	ErrorFormatter  ErrorFormatter
+	StringOutput    bool
+	importCache     *importCache
 }
 
 // External variable or top level argument provided before execution
@@ -56,20 +57,21 @@ type vmExtMap map[string]vmExt
 func MakeVM() *VM {
 	defaultImporter := &FileImporter{}
 	return &VM{
-		MaxStack:       500,
-		ext:            make(vmExtMap),
-		tla:            make(vmExtMap),
-		nativeFuncs:    make(map[string]*NativeFunction),
-		ErrorFormatter: &termErrorFormatter{pretty: false, maxStackTraceSize: 20},
-		importer:       &FileImporter{},
-		importCache:    makeImportCache(defaultImporter),
+		MaxStack:        500,
+		ext:             make(vmExtMap),
+		tla:             make(vmExtMap),
+		nativeFuncs:     make(map[string]*NativeFunction),
+		ErrorFormatter:  &termErrorFormatter{pretty: false, maxStackTraceSize: 20},
+		importer:        &FileImporter{},
+		importProcessor: nil,
+		importCache:     makeImportCache(defaultImporter, nil),
 	}
 }
 
 // Fully flush cache. This should be executed when we are no longer sure that the source files
 // didn't change, for example when the importer changed.
 func (vm *VM) flushCache() {
-	vm.importCache = makeImportCache(vm.importer)
+	vm.importCache = makeImportCache(vm.importer, vm.importProcessor)
 }
 
 // Flush value cache. This should be executed when calculated values may no longer be up to date,
@@ -110,6 +112,13 @@ func (vm *VM) Importer(i Importer) {
 	vm.flushCache()
 }
 
+// ImportProcessor sets the ImportProcesor to use during evaluation
+// (post-processing callback)
+func (vm *VM) ImportProcessor(i ImportProcessor) {
+	vm.importProcessor = i
+	vm.flushCache()
+}
+
 // NativeFunction registers a native function.
 func (vm *VM) NativeFunction(f *NativeFunction) {
 	vm.nativeFuncs[f.Name] = f