diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f6e86c4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019 Volumental AB + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e5d1987 --- /dev/null +++ b/README.md @@ -0,0 +1,170 @@ +# JSONTemplate + +`jsontemplate` is a JSON transformation and templating language and library implemented in Go. + +In simple terms, it renders arbitrary JSON structures based on a JSON-like template definition, populating it with data from an some other JSON structure. + +## Feature overview + +- Low-clutter template syntax, aimed to look similar to the final output. +- [JSONPath](https://goessner.net/articles/JsonPath/) expressions to fetch values from the input. +- Array generator expressions with subtemplates, allowing mapping of arrays of objects. +- Ability to call Go functions from within a template. + +## Getting started + +The following is a complete but minimal program that loads a template and transforms an input JSON object using it. + +```go +package main + +import ( + "os" + "strings" + + "github.com/Volumental/jsontemplate" +) + +const input = `{ "snakeCase": 123 }` + +func main() { + template, _ := jsontemplate.ParseString(`{ "CamelCase": $.snakeCase }`, nil) + template.RenderJSON(os.Stdout, strings.NewReader(input)) + os.Stdout.Sync() +} +``` + +Running the above program will output: +``` +{"CamelCase":123} +``` + +## Features by example + +This example illustrates some of the features in `jsontemplate`. For further details, please see the library documentation. + +Consider the following input JSON structure: + +```json +{ + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + } +} +``` + +When fed through the following template: + +``` +{ + # Pick an invidual field. + "bicycle_color": $.store.bicycle.color, + + "book_info": { + # Slice an array, taking the first three elements. + "top_three": $.store.book[:3], + + # Map a list of objects. + "price_list": range $.store.book[*] [ + { + "title": $.title, + "price": $.price, + } + ], + }, + + # Calculate the average of all price fields by calling a Go function. + "avg_price": Avg($..price), +} +``` + +...the following output is yielded: + +```json +{ + "avg_price": 14.774000000000001, + "bicycle_color": "red", + "book_info": { + "price_list": [ + { + "price": 8.95, + "title": "Sayings of the Century" + }, + { + "price": 12.99, + "title": "Sword of Honour" + }, + { + "price": 8.99, + "title": "Moby Dick" + }, + { + "price": 22.99, + "title": "The Lord of the Rings" + } + ], + "top_three": [ + { + "author": "Nigel Rees", + "category": "reference", + "price": 8.95, + "title": "Sayings of the Century" + }, + { + "author": "Evelyn Waugh", + "category": "fiction", + "price": 12.99, + "title": "Sword of Honour" + }, + { + "author": "Herman Melville", + "category": "fiction", + "isbn": "0-553-21311-3", + "price": 8.99, + "title": "Moby Dick" + } + ] + } +} +``` + +## Performance + +`jsontemplate` has first and foremost been designed with correctness and ease of use in mind. As such, optimum performance has not been the primary objective. Nevertheless, you can expect to see in the order of 10 MB/s on a single CPU core, around half of which is JSON parsing/encoding. We expect this to be more than adequate for most production use-cases. + +## Maturity + +`jsontemplate` is provided as-is, and you should assume it has bugs. That said, at the time of writing, the library is being used for production workloads at [Volumental](https://www.volumental.com). + +Until a 1.0 release is made, incompatible changes may occur, though we will generally strive to maintain full backwards compatibility. Incompatible changes to the template definition format are unlikely to be introduced at this point. diff --git a/benchmark_test.go b/benchmark_test.go new file mode 100644 index 0000000..8717e1b --- /dev/null +++ b/benchmark_test.go @@ -0,0 +1,107 @@ +package jsontemplate_test + +import ( + "encoding/json" + "io/ioutil" + "strings" + "testing" + + "github.com/Volumental/jsontemplate" +) + +// Store example from JSONPath. +const benchmarkInput = ` +{ + "foo": { + "bar": [ + { + "text": "this is a benchmark test", + "number": 12345.6789, + "bool": true, + "array": [1, 2, 3, "hello", true] + }, + { + "text": "short" + }, + { + "text": "loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooonger", + "number": 0, + "bool": false, + "array": [1, 2, 3, "hello", true] + }, + { + "text": "this is a second benchmark test", + "number": 4711, + "bool": null + }, + { + "text": "this is the final benchmark test", + "array": ["nice", 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3] + } + ] + } +} +` + +const benchmarkTemplate = ` +{ + "arrays": range $..bar [$.array[0]], + "mapping": range $.foo.bar[:4] [ + { + "string": ToUpper($.text), + "num": $.number, + } + ], + "a_text": $.foo.bar[3].text, +} +` + +var result interface{} + +func Benchmark_core(b *testing.B) { + var funcs = jsontemplate.FunctionMap{"ToUpper": strings.ToUpper} + var template, err = jsontemplate.ParseString(benchmarkTemplate, funcs) + if err != nil { + panic(err) + } + + b.SetBytes(int64(len(benchmarkInput))) + + var input interface{} + if err := json.Unmarshal([]byte(benchmarkInput), &input); err != nil { + panic(err) + } + + var output interface{} + for n := 0; n < b.N; n++ { + var err error + output, err = template.Render(input) + if err != nil { + panic(err) + } + } + result = output +} + +func Benchmark_full(b *testing.B) { + var funcs = jsontemplate.FunctionMap{"ToUpper": strings.ToUpper} + var template, err = jsontemplate.ParseString(benchmarkTemplate, funcs) + if err != nil { + panic(err) + } + + b.SetBytes(int64(len(benchmarkInput))) + + var input interface{} + if err := json.Unmarshal([]byte(benchmarkInput), &input); err != nil { + panic(err) + } + + var output interface{} + for n := 0; n < b.N; n++ { + if err = template.RenderJSON(ioutil.Discard, strings.NewReader(benchmarkInput)); err != nil { + panic(err) + } + } + result = output +} diff --git a/examples_test.go b/examples_test.go new file mode 100644 index 0000000..bbef1be --- /dev/null +++ b/examples_test.go @@ -0,0 +1,93 @@ +package jsontemplate_test + +import ( + "os" + "strings" + + "github.com/Volumental/jsontemplate" +) + +// Store example from JSONPath. +const Input = ` +{ + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + } +} +` + +const Template = ` +{ + # Pick an invidual field. + "bicycle_color": $.store.bicycle.color, + + "book_info": { + # Slice an array, taking the first three elements. + "top_three": $.store.book[:3], + + # Map a list of objects. + "price_list": range $.store.book[*] [ + { + "title": $.title, + "price": $.price, + } + ], + }, + + # Calculate the average of all price fields. + "avg_price": Avg($..price), +} +` + +// Helper function we'll use in the template. +func Avg(values []interface{}) float64 { + var sum = 0.0 + var cnt = 0 + for _, val := range values { + if num, ok := val.(float64); ok { + sum += num + cnt += 1 + } + } + return sum / float64(cnt) +} + +func Example() { + var funcs = jsontemplate.FunctionMap{"Avg": Avg} + var template, _ = jsontemplate.ParseString(Template, funcs) + + template.RenderJSON(os.Stdout, strings.NewReader(Input)) + os.Stdout.Sync() + // Output: {"avg_price":14.774000000000001,"bicycle_color":"red","book_info":{"price_list":[{"price":8.95,"title":"Sayings of the Century"},{"price":12.99,"title":"Sword of Honour"},{"price":8.99,"title":"Moby Dick"},{"price":22.99,"title":"The Lord of the Rings"}],"top_three":[{"author":"Nigel Rees","category":"reference","price":8.95,"title":"Sayings of the Century"},{"author":"Evelyn Waugh","category":"fiction","price":12.99,"title":"Sword of Honour"},{"author":"Herman Melville","category":"fiction","isbn":"0-553-21311-3","price":8.99,"title":"Moby Dick"}]}} +} diff --git a/functions.go b/functions.go new file mode 100644 index 0000000..f5090c7 --- /dev/null +++ b/functions.go @@ -0,0 +1 @@ +package jsontemplate diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a2a2feb --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/Volumental/jsontemplate + +go 1.13 + +require ( + github.com/alecthomas/participle v0.3.0 + k8s.io/client-go v11.0.0+incompatible +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..85d9316 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/alecthomas/participle v0.3.0 h1:e8vhrYR1nDjzDxyDwpLO27TWOYWilaT+glkwbPadj50= +github.com/alecthomas/participle v0.3.0/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= +k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= diff --git a/internal/parse/grammar.go b/internal/parse/grammar.go new file mode 100644 index 0000000..058f8d8 --- /dev/null +++ b/internal/parse/grammar.go @@ -0,0 +1,70 @@ +package parse + +import ( + "github.com/alecthomas/participle" + "github.com/alecthomas/participle/lexer" + "github.com/alecthomas/participle/lexer/ebnf" +) + +type Generator struct { + Range string `"range" @JSONPath` + SubTemplate Value `"[" @@ "]"` +} + +type AnnotatedField struct { + Annotation string `("@" @Ident)?` + Key string `@String ":"` + Value Value `@@` +} + +type Object struct { + Fields []AnnotatedField `"{" (@@ ("," @@)* ","?)? "}"` +} + +type Function struct { + Name string `@Ident` + Args []Value `"(" (@@ ("," @@)*)? ")"` +} + +type Value struct { + // These are standard JSON fields. + String *string ` @String` + Number *float64 `| @Number` + Object *Object `| @@` + Array []Value `| "[" (@@ ("," @@)* ","?)? "]"` + Bool *bool `| (@"true" | "false")` + Null bool `| @"null"` + + // These are template elements generating JSON fields. + Generator *Generator `| @@` + Extractor *string `| @JSONPath` + Function *Function `| @@` +} + +type Template struct { + Root Value `@@` +} + +var lex = lexer.Must(ebnf.New(` + Comment = "#" { "\u0000"…"\uffff"-"\n" } . + Ident = (alpha | "_") { "_" | alpha | digit } . + String = "\"" { "\u0000"…"\uffff"-"\""-"\\" | "\\" any } "\"" . + Number = Int | Float . + Int = [ "-" ] digit { digit } . + Float = [ "-" ] [ digit ] "." digit { digit } . + JSONPath = "$" { "." { "." } JSONPathExpr } . + JSONPathExpr = "*" | (Ident { "[" { "\u0000"…"\uffff"-"]" } "]" }) . + Punct = "!"…"/" | ":"…"@" | "["…` + "\"`\"" + ` | "{"…"~" . + Whitespace = " " | "\t" | "\n" | "\r" . + + alpha = "a"…"z" | "A"…"Z" . + digit = "0"…"9" . + any = "\u0000"…"\uffff" . +`)) + +var Parser = participle.MustBuild( + &Template{}, + participle.Lexer(lex), + participle.Unquote("String"), + participle.Elide("Whitespace", "Comment"), +) diff --git a/internal/parse/grammar_test.go b/internal/parse/grammar_test.go new file mode 100644 index 0000000..4e6275e --- /dev/null +++ b/internal/parse/grammar_test.go @@ -0,0 +1,207 @@ +package parse + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func numberValue(f float64) Value { return Value{Number: &f} } +func stringValue(s string) Value { return Value{String: &s} } +func boolValue(b bool) Value { return Value{Bool: &b} } +func extractorValue(jsonPath string) Value { return Value{Extractor: &jsonPath} } + +func TestTextRenderer_Render(t *testing.T) { + tests := []struct { + name string + definition string + wantOut Template + wantErr bool + }{ + { + name: "empty", + definition: "", + wantErr: true, + }, + { + name: "number", + definition: "1", + wantOut: Template{ + Root: numberValue(1), + }, + }, + { + name: "string", + definition: `"foo"`, + wantOut: Template{ + Root: stringValue("foo"), + }, + }, + { + name: "array", + definition: `[1, 2, true, "foo"]`, + wantOut: Template{ + Root: Value{Array: []Value{ + numberValue(1), + numberValue(2), + boolValue(true), + stringValue("foo"), + }}, + }, + }, + { + name: "object", + definition: ` + { + "foo": true, + "bar": 123, # Traliing comma is allowed, unlike JSON. + } + `, + wantOut: Template{ + Root: Value{Object: &Object{ + Fields: []AnnotatedField{ + { + Key: "foo", + Value: boolValue(true), + }, + { + Key: "bar", + Value: numberValue(123), + }, + }, + }}, + }, + }, + { + name: "function", + definition: `compare("foo", "bar")`, + wantOut: Template{ + Root: Value{Function: &Function{ + Name: "compare", + Args: []Value{ + stringValue("foo"), + stringValue("bar"), + }, + }}, + }, + }, + { + name: "annotation", + definition: `{@foobar "foo": true}`, + wantOut: Template{ + Root: Value{Object: &Object{ + Fields: []AnnotatedField{ + { + Annotation: "foobar", + Key: "foo", + Value: boolValue(true), + }, + }, + }}, + }, + }, + { + name: "jsonpath", + definition: `{"foo": $.foo.bar[234]..baz[*]}`, + wantOut: Template{ + Root: Value{Object: &Object{ + Fields: []AnnotatedField{ + { + Key: "foo", + Value: extractorValue("$.foo.bar[234]..baz[*]"), + }, + }, + }}, + }, + }, + { + name: "complex", + definition: ` + { + "foo": [ + 123, + { + @deprecated "baz": range $..stuff [ + { + "x": null, + "y": $.hello[1:5] + } + ], + "something": "with trailing comma", + } + ] + } + `, + wantOut: Template{ + Root: Value{Object: &Object{Fields: []AnnotatedField{ + { + Key: "foo", + Value: Value{ + Array: []Value{ + numberValue(123), + Value{ + Object: &Object{Fields: []AnnotatedField{ + { + Annotation: "deprecated", + Key: "baz", + Value: Value{ + Generator: &Generator{ + Range: "$..stuff", + SubTemplate: Value{ + Object: &Object{Fields: []AnnotatedField{ + { + Key: "x", + Value: Value{Null: true}, + }, + { + Key: "y", + Value: extractorValue("$.hello[1:5]"), + }, + }}, + }, + }, + }, + }, + { + Key: "something", + Value: stringValue("with trailing comma"), + }, + }}, + }, + }, + }, + }, + }, + }}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var out Template + if err := Parser.ParseString(tt.definition, &out); (err != nil) != tt.wantErr { + // Print the lexer output in case of a parser error (for debug help). + var l, _ = lex.Lex(strings.NewReader(tt.definition)) + var symbolNames = map[rune]string{} + for k, v := range lex.Symbols() { + symbolNames[v] = k + } + for i := 0; i < 10000; i++ { + if tok, err := l.Next(); err != nil || tok.EOF() { + break + } else { + if strings.TrimSpace(tok.String()) != "" { + fmt.Println(tok, symbolNames[tok.Type]) + } + } + } + t.Errorf("Parser.ParseString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(out, tt.wantOut) { + t.Errorf("Parser.ParseString() = %v, want %v", out, tt.wantOut) + } + }) + } +} diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..c9c3cd8 --- /dev/null +++ b/parse.go @@ -0,0 +1,173 @@ +package jsontemplate + +import ( + "fmt" + "io" + "reflect" + "strings" + + "github.com/Volumental/jsontemplate/internal/parse" + "k8s.io/client-go/util/jsonpath" +) + +type builder struct { + funcs FunctionMap +} + +func (b *builder) buildObject(o *parse.Object) object { + var res = object{} + for _, f := range o.Fields { + res[f.Key] = field{ + value: b.buildValue(&f.Value), + annotation: f.Annotation, + } + } + return res +} + +func (b *builder) buildQuery(q *string) query { + var jp = jsonpath.New("template-query") + if err := jp.Parse(fmt.Sprintf("{%s}", *q)); err != nil { + panic(fmt.Errorf("jsontemplate: invalid jsonpath: %v", err)) + } + return query{expression: jp} +} + +func (b *builder) buildFunction(node *parse.Function) template { + var res = function{ + name: node.Name, + args: make([]template, len(node.Args)), + } + var ok bool + if res.function, ok = b.funcs[node.Name]; !ok { + panic(fmt.Errorf("jsontemplate: no such function: %s", node.Name)) + } else if fun := reflect.ValueOf(res.function); fun.Kind() != reflect.Func { + panic(fmt.Sprintf("%s is not a function", node.Name)) // Actual panic. + } + // TODO(josef): Verify that the signature has a single return value. + for i, v := range node.Args { + res.args[i] = b.buildValue(&v) + } + return res +} + +func (b *builder) buildValue(v *parse.Value) template { + switch { + case v.String != nil: + return stringConstant(*v.String) + case v.Number != nil: + return numberConstant(*v.Number) + case v.Bool != nil: + return boolConstant(*v.Bool) + case v.Null: + return nullConstant{} + case v.Object != nil: + return b.buildObject(v.Object) + case v.Array != nil: + var res = make(array, len(v.Array)) + for i, v := range v.Array { + res[i] = b.buildValue(&v) + } + return res + case v.Generator != nil: + return generator{ + over: b.buildQuery(&v.Generator.Range), + template: b.buildValue(&v.Generator.SubTemplate), + } + case v.Extractor != nil: + return b.buildQuery(v.Extractor) + case v.Function != nil: + return b.buildFunction(v.Function) + default: + panic("unhandled case") + } +} + +// FunctionMap is a map of named functions that may be called from within a +// template. +type FunctionMap map[string]interface{} + +// ParseString works like Parse, but takes a string as input rather than a +// Reader. +func ParseString(s string, funcs FunctionMap) (*Template, error) { + return Parse(strings.NewReader(s), funcs) +} + +// Parse reads a template definition from a textual format. +// +// The template definition format has a grammar similar to a regular JSON value, +// with some additions that allow interpolation and transformation. These are +// outlined below. As a special case, a regular JSON file is interpreted as a +// template taking no inputs. +// +// Queries +// +// A template can pull data from the input data using JSONPath expressions (see +// https://goessner.net/articles/JsonPath/). Each query expression can evaluate +// to either a JSON value, which will then be inserted in its place, or a range +// of values, which will yield an array. +// { +// "single_x": $.foo.x, +// "array_of_all_x_recursively": $..x +// } +// +// Generators +// +// Generators is a mechanism that allows repeating a sub-template within an +// array. It takes the format of the keyword `range` followed an array +// expression and a sub-template, as such: +// range $.some_array_of_xy[*] [ +// { "foo": $.x, "bar": $.y } +// ] +// Inside the sub-template, the `$` refers to the root of each element in the +// input array. Thus, the example above maps the fields `x` and `y` to `foo` and +// `bar`, respectively, in the objects in the output array. +// +// Functions +// +// Regular Go functions can be exposed to and called from within the template. +// This allows more complex transformations. For example, by adding a Coalesce +// function that returns the first non-nil argument, fallback defaults can be +// introduced as follows: +// { "foo": Coalesce($.some_input, "default value") } +// +// Field annotations +// +// Members in objects can be prefixed with an annotation, starting with an `@` +// character. +// { +// @deprecated "field": "value" +// } +// Annotations are stripped from the final output. However, future versions of +// this library may allow some control over how annotated fields are rendered. +// For example, it could be used to elide deprecated fields. +// +// Other differences +// +// To help users clarify and document intentions, the template format allows +// comments in the template definition. These are preceded by a `#` character. +// Anything from and including this character to the end of the line will be +// ignored when parsing the template. +// +// Finally, unlike JSON, the template format tolerates trailing commas after the +// last element of objects and arrays. +func Parse(r io.Reader, funcs FunctionMap) (t *Template, err error) { + var ast parse.Template + if err := parse.Parser.Parse(r, &ast); err != nil { + return nil, fmt.Errorf("jsontemplate: parse error: %v", err) + } + // We handle errors in the recurstion using panics that stop here. + // This is similar to how the json library does it. + defer func() { + var r = recover() + switch r := r.(type) { + case nil: // Nothing. + case error: + err = r + default: + panic(fmt.Sprintf("jsontemplate: panic during parsing: %v", r)) + } + }() + var b = builder{funcs: funcs} + return &Template{definition: b.buildValue(&ast.Root)}, nil +} diff --git a/parse_test.go b/parse_test.go new file mode 100644 index 0000000..1faa990 --- /dev/null +++ b/parse_test.go @@ -0,0 +1,157 @@ +package jsontemplate + +import ( + "fmt" + "reflect" + "testing" + + "k8s.io/client-go/util/jsonpath" +) + +func mustParseJSONPath(s string) *jsonpath.JSONPath { + var jp = jsonpath.New("template-query") + if err := jp.Parse(fmt.Sprintf("{%s}", s)); err != nil { + panic("tests broken") + } + return jp +} + +func TestParseString(t *testing.T) { + tests := []struct { + name string + definition string + wantOut *Template + wantErr bool + }{ + { + name: "empty", + definition: "", + wantErr: true, + }, + { + name: "number", + definition: "1", + wantOut: &Template{ + definition: numberConstant(1), + }, + }, + { + name: "string", + definition: `"foo"`, + wantOut: &Template{ + definition: stringConstant("foo"), + }, + }, + { + name: "array", + definition: `[1, 2, true, "foo"]`, + wantOut: &Template{ + definition: array{ + numberConstant(1), + numberConstant(2), + boolConstant(true), + stringConstant("foo"), + }, + }, + }, + { + name: "object", + definition: ` + { + "foo": true, + "bar": 123, # Traliing comma is allowed, unlike JSON. + } + `, + wantOut: &Template{ + definition: object{ + "foo": field{value: boolConstant(true)}, + "bar": field{value: numberConstant(123)}, + }, + }, + }, + { + name: "annotation", + definition: `{@deprecated "foo": true}`, + wantOut: &Template{ + definition: object{ + "foo": field{ + value: boolConstant(true), + annotation: "deprecated", + }, + }, + }, + }, + // Note: Functions are not comparable in Go, so it we can't test using + // one here in any way that isn't already covered elsewhere. But + // we can test the error handling of missing ones. + { + name: "missing function", + definition: `missing("foo", "bar")`, + wantErr: true, + }, + { + name: "jsonpath", + definition: `{"foo": $.foo.bar[234]..baz[*]}`, + wantOut: &Template{ + definition: object{ + "foo": field{ + value: query{ + expression: mustParseJSONPath("$.foo.bar[234]..baz[*]"), + }, + }, + }, + }, + }, + { + name: "complex", + definition: ` + { + "foo": [ + 123, + { + @deprecated "baz": range $..stuff [ + { + "x": null, + "y": $.hello[1:5] + } + ], + "something": "with trailing comma", + } + ] + } + `, + wantOut: &Template{ + definition: object{ + "foo": field{value: array{ + numberConstant(123), + object{ + "baz": field{ + value: generator{ + over: query{expression: mustParseJSONPath("$..stuff")}, + template: object{ + "x": field{value: nullConstant{}}, + "y": field{value: query{expression: mustParseJSONPath("$.hello[1:5]")}}, + }, + }, + annotation: "deprecated", + }, + "something": field{value: stringConstant("with trailing comma")}, + }, + }}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotT, err := ParseString(tt.definition, nil) + if (err != nil) != tt.wantErr { + t.Errorf("ParseString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotT, tt.wantOut) { + t.Errorf("ParseString() = %v, want %v", gotT, tt.wantOut) + } + }) + } +} diff --git a/render.go b/render.go new file mode 100644 index 0000000..480246f --- /dev/null +++ b/render.go @@ -0,0 +1,249 @@ +package jsontemplate + +import ( + "encoding/json" + "fmt" + "io" + "reflect" + "runtime" + "runtime/debug" + + "k8s.io/client-go/util/jsonpath" +) + +// MissingKeyPolicy discates how the rendering should handle references to keys +// that are missing from the input data. +type MissingKeyPolicy int + +const ( + // NullOnMissing makes query expressions referencing missing values evaluate + // to null. + NullOnMissing MissingKeyPolicy = iota + + // ErrorOnMissing causes the renderer to return an error if a query + // expression references a missing value. + ErrorOnMissing +) + +type options struct { + MissingKeys MissingKeyPolicy +} + +type template interface { + interpolate(data interface{}, opt options) interface{} +} + +type stringConstant string +type boolConstant bool +type numberConstant float64 +type nullConstant struct{} + +type object map[string]field + +type field struct { + value template + annotation string +} + +type array []template + +type query struct { + expression *jsonpath.JSONPath +} + +type generator struct { + over query + template template +} + +type function struct { + name string // For giving informative error messages. + function interface{} // Must be a function with a single return value. + args []template +} + +func (s stringConstant) interpolate(data interface{}, opt options) interface{} { return string(s) } +func (b boolConstant) interpolate(data interface{}, opt options) interface{} { return bool(b) } +func (n numberConstant) interpolate(data interface{}, opt options) interface{} { return float64(n) } +func (n nullConstant) interpolate(data interface{}, opt options) interface{} { return nil } + +func (o object) interpolate(data interface{}, opt options) interface{} { + var res = make(map[string]interface{}) + for name, field := range o { + res[name] = field.value.interpolate(data, opt) + } + return res +} + +func (a array) interpolate(data interface{}, opt options) interface{} { + var res = make([]interface{}, len(a)) + for i, templ := range a { + res[i] = templ.interpolate(data, opt) + } + return res +} + +func (q query) interpolate(data interface{}, opt options) interface{} { + if data == nil { + switch opt.MissingKeys { + case NullOnMissing: + return nil + case ErrorOnMissing: + panic(fmt.Errorf("jsontemplate: cannot execute query, input is null")) + } + } + q.expression.AllowMissingKeys(opt.MissingKeys == NullOnMissing) + var hits, err = q.expression.FindResults(data) + if err != nil { + panic(fmt.Errorf("jsontemplate: error executing query: %v", err)) + } + switch len(hits[0]) { + case 0: + return nil + case 1: + return hits[0][0].Interface() + default: // Many, make an array. + var res = make([]interface{}, len(hits[0])) + for i, v := range hits[0] { + res[i] = v.Interface() + } + return res + } +} + +func (g generator) interpolate(data interface{}, opt options) interface{} { + if data == nil { + switch opt.MissingKeys { + case NullOnMissing: + return nil + case ErrorOnMissing: + panic(fmt.Errorf("jsontemplate: cannot generate array, input is null")) + } + } + g.over.expression.AllowMissingKeys(opt.MissingKeys == NullOnMissing) + var hits, err = g.over.expression.FindResults(data) + if err != nil { + panic(err) + } + var res = make([]interface{}, len(hits[0])) + for i, v := range hits[0] { + var inner interface{} + if v.IsValid() { + inner = v.Interface() + } + res[i] = g.template.interpolate(inner, opt) + } + return res +} + +func (f function) interpolate(data interface{}, opt options) interface{} { + var args = make([]reflect.Value, len(f.args)) + var ftype = reflect.TypeOf(f.function) + for i, templ := range f.args { + // The Call function of the reflect library doesn't handle nil + // interfaces the way we want (it will create an invalid Value) so we + // need some special handling of nil arguments here. + var val = templ.interpolate(data, opt) + var expected reflect.Type + if ftype.IsVariadic() && i >= ftype.NumIn()-1 { + // Variadic arguments are represented as a final array argument. + expected = ftype.In(ftype.NumIn() - 1).Elem() + } else { + expected = ftype.In(i) + } + if val == nil { + switch expected.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, + reflect.Map, reflect.Ptr, reflect.Slice: + // These types can be assigned 'nil'. + args[i] = reflect.Zero(expected) + default: + panic(fmt.Errorf("jsontemplate: cannot pass nil as argument %d of %s, expecting %v", i+1, f.name, expected)) + } + continue + } + // If the value is not nil, we check that it matches the argument of + // the function, to give a more informative error message than Call + // would give us. + var rval = reflect.ValueOf(val) + var actual = rval.Type() + if expected.Kind() == reflect.Ptr && actual.Kind() != reflect.Ptr { + // If the function wants a pointer and we have a value, we make a + // pointer to a copy and pass that. This allows declaring + // functions taking nullable arguments by means of pointers. + var pointer = reflect.New(actual) + pointer.Elem().Set(rval) + actual = pointer.Type() + rval = pointer + } + if !actual.AssignableTo(expected) { + panic(fmt.Errorf("jsontemplate: cannot pass %v (%v) as argument %d of %s, expecting %v", val, reflect.TypeOf(val), i+1, f.name, expected)) + } + args[i] = rval + } + // The parser should already have asserted that this is valid, yielding a + // nicer and earlier panic than we could do here, so no extra checks here. + return reflect.ValueOf(f.function).Call(args)[0].Interface() +} + +// Template represents a transformation from one JSON-like structure to another. +type Template struct { + definition template + + // MissingKeys defines the policy for how to handle keys referenced in + // queries that are absent in the input data. The default is to substitute + // them with null. + MissingKeys MissingKeyPolicy +} + +// Render generates a JSON-like structure based on the template definition, +// using the passed `data` as source data for query expressions. +func (t *Template) Render(data interface{}) (res interface{}, err error) { + // We handle errors in the recurstion using panics that stop here. + // This is similar to how the json library does it. + defer func() { + var r = recover() + switch r := r.(type) { + case nil: // Nothing. + case runtime.Error: + err = fmt.Errorf("jsontemplate: while rendering: %v\n%s", r, debug.Stack()) + case error: + fmt.Println("was here!") + err = r + default: + err = fmt.Errorf("jsontemplate: panic during interpolation: %v", r) + } + }() + res = t.definition.interpolate(data, options{MissingKeys: t.MissingKeys}) + return +} + +// RenderJSON generates JSON output based on the template definition, using JSON +// input as source data for query expressions. +// +// Note that RenderJSON will only attempt to read a single JSON value from the +// input stream. If the stream contains multiple white-space delimited JSON +// values that you wish to transform, RenderJSON can be called repeatedly with +// the same arguments. +// +// If EOF is encountered on the input stream before the start of a JSON value, +// RenderJSON will return io.EOF. +func (t *Template) RenderJSON(out io.Writer, in io.Reader) error { + var dec = json.NewDecoder(in) + var input interface{} + if err := dec.Decode(&input); err != nil { + if err == io.EOF { + return err + } + return fmt.Errorf("jsontemplate: invalid input: %v", err) + } + var output, err = t.Render(input) + if err != nil { + return err + } + var enc = json.NewEncoder(out) + if err := enc.Encode(&output); err != nil { + return fmt.Errorf("jsontemplate: error writing output: %v", err) + } + return nil +} diff --git a/render_test.go b/render_test.go new file mode 100644 index 0000000..e0fdc41 --- /dev/null +++ b/render_test.go @@ -0,0 +1,627 @@ +package jsontemplate + +import ( + "bytes" + "fmt" + "os" + "reflect" + "strings" + "testing" +) + +func Test_stringConstant_interpolate(t *testing.T) { + tests := []struct { + name string + s stringConstant + want interface{} + }{ + { + name: "empty", + s: "", + want: "", + }, + { + name: "not empty", + s: "foo", + want: "foo", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.s.interpolate(nil, options{}); !reflect.DeepEqual(got, tt.want) { + t.Errorf("stringConstant.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_boolConstant_interpolate(t *testing.T) { + tests := []struct { + name string + b boolConstant + want interface{} + }{ + { + name: "true", + b: true, + want: true, + }, + { + name: "false", + b: false, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.b.interpolate(nil, options{}); !reflect.DeepEqual(got, tt.want) { + t.Errorf("boolConstant.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_numberConstant_interpolate(t *testing.T) { + tests := []struct { + name string + n numberConstant + want interface{} + }{ + { + name: "arbitrary", + n: 4711, + want: float64(4711), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.n.interpolate(nil, options{}); !reflect.DeepEqual(got, tt.want) { + t.Errorf("numberConstant.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_nullConstant_interpolate(t *testing.T) { + var n nullConstant + if n.interpolate(nil, options{}) != nil { + t.Errorf("nullConstant.interpolate() != nil") + } +} + +func Test_object_interpolate(t *testing.T) { + type args struct { + data interface{} + opt options + } + tests := []struct { + name string + o object + args args + want interface{} + }{ + { + name: "empty", + o: object{}, + want: map[string]interface{}{}, + }, + { + name: "simple", + o: object{ + "x": field{value: numberConstant(123)}, + }, + want: map[string]interface{}{ + "x": float64(123), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.o.interpolate(tt.args.data, tt.args.opt); !reflect.DeepEqual(got, tt.want) { + t.Errorf("object.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_array_interpolate(t *testing.T) { + tests := []struct { + name string + a array + want interface{} + }{ + { + name: "empty", + a: array{}, + want: []interface{}{}, + }, + { + name: "not empty", + a: array{ + numberConstant(1), + numberConstant(2), + boolConstant(true), + stringConstant("foo"), + }, + want: []interface{}{float64(1), float64(2), true, "foo"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.a.interpolate(nil, options{}); !reflect.DeepEqual(got, tt.want) { + t.Errorf("array.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_query_interpolate(t *testing.T) { + type args struct { + data interface{} + opt options + } + tests := []struct { + name string + expression string + args args + want interface{} + wantPanic bool + }{ + { + name: "trivial", + expression: "$", + args: args{data: 123}, + want: 123, + }, + { + name: "array", + expression: "$.*", + args: args{ + data: struct{ X, Y int }{X: 1, Y: 1}, + }, + want: []interface{}{1, 1}, + }, + { + name: "nested", + expression: "$.X.Y", + args: args{ + data: struct{ X interface{} }{X: struct{ Y int }{Y: 123}}, + }, + want: 123, + }, + { + name: "search", + expression: "$..Y", + args: args{ + data: struct{ X interface{} }{X: struct{ Y int }{Y: 123}}, + }, + want: 123, + }, + { + name: "missing chain", + expression: "$.a.b.c", + args: args{ + data: map[string]int{"a": 123}, + }, + want: nil, + }, + { + name: "missing chain error", + expression: "$.a.b.c", + args: args{ + data: map[string]int{"a": 123}, + opt: options{MissingKeys: ErrorOnMissing}, + }, + want: nil, + wantPanic: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q := query{ + expression: mustParseJSONPath(tt.expression), + } + if tt.wantPanic { + defer func() { + if r := recover(); r == nil { + t.Errorf("query.interpolate() did not panic as expected") + } + }() + } + if got := q.interpolate(tt.args.data, tt.args.opt); !reflect.DeepEqual(got, tt.want) { + t.Errorf("query.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_generator_interpolate(t *testing.T) { + type args struct { + data interface{} + opt options + } + tests := []struct { + name string + g generator + args args + want interface{} + wantPanic bool + }{ + { + name: "trivial (no input)", + g: generator{ + over: query{expression: mustParseJSONPath("$")}, + template: query{expression: mustParseJSONPath("$")}, + }, + want: nil, + }, + { + name: "trivial (no input, error)", + g: generator{ + over: query{expression: mustParseJSONPath("$")}, + template: query{expression: mustParseJSONPath("$")}, + }, + args: args{opt: options{MissingKeys: ErrorOnMissing}}, + want: nil, + wantPanic: true, + }, + { + name: "trivial (input)", + g: generator{ + over: query{expression: mustParseJSONPath("$")}, + template: query{expression: mustParseJSONPath("$")}, + }, + args: args{ + data: "foo", + }, + want: []interface{}{"foo"}, + }, + { + name: "list", + g: generator{ + over: query{expression: mustParseJSONPath("$.*")}, + template: query{expression: mustParseJSONPath("$")}, + }, + args: args{ + data: []string{"foo", "bar", "baz"}, + }, + want: []interface{}{"foo", "bar", "baz"}, + }, + { + name: "search", + g: generator{ + over: query{expression: mustParseJSONPath("$..Inner")}, + template: query{expression: mustParseJSONPath("$")}, + }, + args: args{ + data: []struct { + Inner string + }{ + {Inner: "hello"}, + {Inner: "world"}, + }, + }, + want: []interface{}{"hello", "world"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantPanic { + defer func() { + if r := recover(); r == nil { + t.Errorf("query.interpolate() did not panic as expected") + } + }() + } + if got := tt.g.interpolate(tt.args.data, tt.args.opt); !reflect.DeepEqual(got, tt.want) { + t.Errorf("generator.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_function_interpolate(t *testing.T) { + var hello = func() string { return "hello" } + var fancy = func(n float64, np *float64, bs []byte, i interface{}, more ...interface{}) string { return "ok" } + tests := []struct { + name string + f function + want interface{} + wantPanic bool + }{ + { + name: "nullary", + f: function{ + name: "hello", + function: hello, + }, + want: "hello", + }, + { + name: "binary", + f: function{ + name: "compare", + function: strings.Compare, + args: []template{ + stringConstant("foo"), + stringConstant("bar"), + }, + }, + want: 1, + }, + { + name: "fancy", + f: function{ + name: "fancy", + function: fancy, + args: []template{ + numberConstant(1), + nullConstant{}, + nullConstant{}, + nullConstant{}, + }, + }, + want: "ok", + }, + { + name: "null to float", + f: function{ + name: "fancy", + function: fancy, + args: []template{ + nullConstant{}, + nullConstant{}, + nullConstant{}, + nullConstant{}, + }, + }, + wantPanic: true, + }, + { + name: "number to interface", + f: function{ + name: "fancy", + function: fancy, + args: []template{ + numberConstant(1), + nullConstant{}, + nullConstant{}, + numberConstant(1), + }, + }, + want: "ok", + }, + { + name: "variadic", + f: function{ + name: "fancy", + function: fancy, + args: []template{ + numberConstant(1), + nullConstant{}, + nullConstant{}, + numberConstant(1), + numberConstant(2), + nullConstant{}, + numberConstant(3), + }, + }, + want: "ok", + }, + { + name: "number to pointer", + f: function{ + name: "fancy", + function: fancy, + args: []template{ + numberConstant(1), + numberConstant(1), + nullConstant{}, + nullConstant{}, + }, + }, + want: "ok", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantPanic { + defer func() { + if r := recover(); r == nil { + t.Errorf("query.interpolate() did not panic as expected") + } + }() + } + if got := tt.f.interpolate(nil, options{}); !reflect.DeepEqual(got, tt.want) { + t.Errorf("array.interpolate() = %v, want %v", got, tt.want) + } + }) + } +} + +var testData = map[string]interface{}{ + "string": "hello world", + "number": 123, + "bool": true, + "nil": nil, + "object": map[string]interface{}{ + "first": "hello", + "second": "world", + }, + "array": []interface{}{ + "text", + 123, + true, + nil, + }, + "nested": map[string]interface{}{ + "first": map[string]interface{}{ + "a": 123, + "b": true, + }, + "second": map[string]interface{}{ + "a": 321, + "b": true, + }, + }, + "array_of_objects": []interface{}{ + map[string]interface{}{ + "n": 123, + }, + map[string]interface{}{ + "n": 321, + }, + }, +} + +func TestTemplate_Render(t *testing.T) { + var funcMap = map[string]interface{}{ + "to_upper": strings.ToUpper, + } + type args struct { + data interface{} + opt options + } + tests := []struct { + name string + definition string + args args + wantRes interface{} + wantErr bool + }{ + { + name: "trivial", + definition: "1", + wantRes: float64(1), + }, + { + name: "static", + definition: `{"foo": "hello", "bar": [1, 2, 3]}`, + wantRes: map[string]interface{}{ + "foo": "hello", + "bar": []interface{}{float64(1), float64(2), float64(3)}, + }, + }, + { + name: "query field", + definition: `$.number`, + wantRes: 123, + args: args{data: testData}, + }, + { + name: "query object", + definition: `$.object`, + wantRes: map[string]interface{}{ + "first": "hello", + "second": "world", + }, + args: args{data: testData}, + }, + { + name: "query recursive", + definition: `$..b`, + wantRes: []interface{}{true, true}, + args: args{data: testData}, + }, + { + name: "composed", + definition: ` + { + "foo": $.array[2:], + "bar": range $.array_of_objects[*] [ + { "x": $.n } + ], + "greeting": to_upper($.string) + } + `, + wantRes: map[string]interface{}{ + "foo": []interface{}{true, nil}, + "bar": []interface{}{ + map[string]interface{}{"x": 123}, + map[string]interface{}{"x": 321}, + }, + "greeting": "HELLO WORLD", + }, + args: args{data: testData}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + templ, err := ParseString(tt.definition, funcMap) + if err != nil { + panic(fmt.Sprintf("broken test: %v", err)) + } + templ.MissingKeys = tt.args.opt.MissingKeys + gotRes, err := templ.Render(tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Template.Render() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotRes, tt.wantRes) { + t.Errorf("Template.Render() = %v, want %v", gotRes, tt.wantRes) + } + }) + } +} + +func TestTemplate_RenderJSON(t *testing.T) { + var funcMap = map[string]interface{}{ + "to_upper": strings.ToUpper, + } + tests := []struct { + name string + definition string + input string + wantOut string + wantErr bool + }{ + { + name: "composed", + definition: ` + { + "foo": $.array[2:], + "bar": range $.array_of_objects[*] [ + { "x": $.n } + ], + "greeting": to_upper($.string) + } + `, + input: ` + { + "array": [1, 2, "hello", 3], + "array_of_objects": [ + { "n": 123, "m": 321 }, + { "n": true, "m": false }, + { "n": "A", "m": "B" } + ], + "string": "hello world" + } + `, + wantOut: `{"bar":[{"x":123},{"x":true},{"x":"A"}],"foo":["hello",3],"greeting":"HELLO WORLD"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + templ, err := ParseString(tt.definition, funcMap) + if err != nil { + panic(fmt.Sprintf("broken test: %v", err)) + } + out := &bytes.Buffer{} + if err := templ.RenderJSON(out, strings.NewReader(tt.input)); (err != nil) != tt.wantErr { + t.Errorf("Template.RenderJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotOut := out.String(); strings.TrimSpace(gotOut) != tt.wantOut { + t.Errorf("Template.RenderJSON() = %v, want %v", gotOut, tt.wantOut) + } + }) + } +} + +func ExampleTemplate_RenderJSON() { + const input = `{ "snakeCase": 123 }` + template, _ := ParseString(`{ "CamelCase": $.snakeCase }`, nil) + template.RenderJSON(os.Stdout, strings.NewReader(input)) + os.Stdout.Sync() + // Output: {"CamelCase":123} +}