From 04d98a299622eea58dc114280c9bdbc3921efdfd Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Fri, 29 Dec 2017 22:22:53 +0100 Subject: [PATCH] Refactoring template processing Signed-off-by: Dave Henderson --- gomplate.go | 61 ++++------------- gomplate_test.go | 2 +- process.go | 144 ---------------------------------------- process_test.go | 65 ------------------ template.go | 169 +++++++++++++++++++++++++++++++++++++++++++++++ template_test.go | 52 +++++++++++++++ 6 files changed, 236 insertions(+), 257 deletions(-) delete mode 100644 process.go delete mode 100644 process_test.go create mode 100644 template.go create mode 100644 template_test.go diff --git a/gomplate.go b/gomplate.go index 6bb3243a5..860a7cda5 100644 --- a/gomplate.go +++ b/gomplate.go @@ -2,16 +2,11 @@ package main import ( "io" - "path/filepath" "text/template" "github.com/hairyhenderson/gomplate/data" ) -func (g *Gomplate) createTemplate(name string) *template.Template { - return template.New(name).Funcs(g.funcMap).Option("missingkey=error") -} - // Gomplate - type Gomplate struct { funcMap template.FuncMap @@ -20,13 +15,19 @@ type Gomplate struct { } // RunTemplate - -func (g *Gomplate) RunTemplate(in *input, out io.Writer) error { +func (g *Gomplate) RunTemplate(t *input) error { context := &Context{} - tmpl, err := g.createTemplate(in.name).Delims(g.leftDelim, g.rightDelim).Parse(in.contents) + tmpl, err := t.toGoTemplate(g) if err != nil { return err } - err = tmpl.Execute(out, context) + + switch t.target.(type) { + case io.Closer: + // nolint: errcheck + defer t.target.(io.Closer).Close() + } + err = tmpl.Execute(t.target, context) return err } @@ -39,12 +40,6 @@ func NewGomplate(d *data.Data, leftDelim, rightDelim string) *Gomplate { } } -// input - models an input file... -type input struct { - name string - contents string -} - func runTemplate(o *GomplateOpts) error { defer runCleanupHooks() d := data.NewData(o.dataSources, o.dataSourceHeaders) @@ -52,42 +47,14 @@ func runTemplate(o *GomplateOpts) error { g := NewGomplate(d, o.lDelim, o.rDelim) - excludeList, err := executeCombinedGlob(o.excludeGlob) + tmpl, err := gatherTemplates(o) if err != nil { return err } - - if o.inputDir != "" { - return processInputDir(o.inputDir, o.outputDir, excludeList, g) - } - - return processInputFiles(o.input, o.inputFiles, o.outputFiles, excludeList, g) -} - -// Called from process.go ... -func renderTemplate(g *Gomplate, in *input, outPath string) error { - outFile, err := openOutFile(outPath) - if err != nil { - return err - } - // nolint: errcheck - defer outFile.Close() - err = g.RunTemplate(in, outFile) - return err -} - -// takes an array of glob strings and executes it as a whole, -// returning a merged list of globbed files -func executeCombinedGlob(globArray []string) ([]string, error) { - var combinedExcludes []string - for _, glob := range globArray { - excludeList, err := filepath.Glob(glob) - if err != nil { - return nil, err + for _, t := range tmpl { + if err := g.RunTemplate(t); err != nil { + return err } - - combinedExcludes = append(combinedExcludes, excludeList...) } - - return combinedExcludes, nil + return nil } diff --git a/gomplate_test.go b/gomplate_test.go index d3d23cc69..d9805b32c 100644 --- a/gomplate_test.go +++ b/gomplate_test.go @@ -17,7 +17,7 @@ import ( func testTemplate(g *Gomplate, template string) string { var out bytes.Buffer - g.RunTemplate(&input{"testtemplate", template}, &out) + g.RunTemplate(&input{name: "testtemplate", contents: template, target: &out}) return out.String() } diff --git a/process.go b/process.go deleted file mode 100644 index 46dd9e63c..000000000 --- a/process.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" -) - -// == Direct input processing ======================================== - -func processInputFiles(stringTemplate string, input []string, output []string, excludeList []string, g *Gomplate) error { - ins, err := readInputs(stringTemplate, input) - if err != nil { - return err - } - - if len(output) == 0 { - output = []string{"-"} - } - - for n, in := range ins { - if err := renderTemplate(g, in, output[n]); err != nil { - return err - } - } - return nil -} - -// == Recursive input dir processing ====================================== - -func processInputDir(input string, output string, excludeList []string, g *Gomplate) error { - input = filepath.Clean(input) - output = filepath.Clean(output) - - // assert tha input path exists - si, err := os.Stat(input) - if err != nil { - return err - } - - // read directory - entries, err := ioutil.ReadDir(input) - if err != nil { - return err - } - - // ensure output directory - if err = os.MkdirAll(output, si.Mode()); err != nil { - return err - } - - // process or dive in again - for _, entry := range entries { - nextInPath := filepath.Join(input, entry.Name()) - nextOutPath := filepath.Join(output, entry.Name()) - - if inList(excludeList, nextInPath) { - continue - } - - if entry.IsDir() { - err := processInputDir(nextInPath, nextOutPath, excludeList, g) - if err != nil { - return err - } - } else { - in, err := readInput(nextInPath) - if err != nil { - return err - } - if err := renderTemplate(g, in, nextOutPath); err != nil { - return err - } - } - } - return nil -} - -func inList(list []string, entry string) bool { - for _, file := range list { - if file == entry { - return true - } - } - - return false -} - -// == File handling ================================================ - -func readInputs(inString string, files []string) ([]*input, error) { - if inString != "" { - return []*input{{ - name: "", - contents: inString, - }}, nil - } - if len(files) == 0 { - files = []string{"-"} - } - ins := make([]*input, len(files)) - - for n, filename := range files { - in, err := readInput(filename) - if err != nil { - return nil, err - } - ins[n] = in - } - return ins, nil -} - -func readInput(filename string) (*input, error) { - var err error - var inFile *os.File - if filename == "-" { - inFile = os.Stdin - } else { - inFile, err = os.Open(filename) - if err != nil { - return nil, fmt.Errorf("failed to open %s\n%v", filename, err) - } - // nolint: errcheck - defer inFile.Close() - } - bytes, err := ioutil.ReadAll(inFile) - if err != nil { - err = fmt.Errorf("read failed for %s\n%v", filename, err) - return nil, err - } - in := &input{ - name: filename, - contents: string(bytes), - } - return in, nil -} - -func openOutFile(filename string) (out *os.File, err error) { - if filename == "-" { - return os.Stdout, nil - } - return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) -} diff --git a/process_test.go b/process_test.go deleted file mode 100644 index 7441724c2..000000000 --- a/process_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// +build !windows - -package main - -import ( - "io/ioutil" - "os" - "testing" - - "path/filepath" - - "log" - - "github.com/hairyhenderson/gomplate/data" - "github.com/stretchr/testify/assert" -) - -func TestReadInput(t *testing.T) { - actual, err := readInputs("foo", nil) - assert.Nil(t, err) - assert.Equal(t, &input{"", "foo"}, actual[0]) - - // stdin is "" because during tests it's given /dev/null - actual, err = readInputs("", []string{"-"}) - assert.Nil(t, err) - assert.Equal(t, &input{"-", ""}, actual[0]) - - actual, err = readInputs("", []string{"process_test.go"}) - assert.Nil(t, err) - thisFile, _ := os.Open("process_test.go") - expected, _ := ioutil.ReadAll(thisFile) - assert.Equal(t, &input{"process_test.go", string(expected)}, actual[0]) -} - -func TestInputDir(t *testing.T) { - outDir, err := ioutil.TempDir(filepath.Join("test", "files", "input-dir"), "out-temp-") - assert.Nil(t, err) - defer (func() { - if cerr := os.RemoveAll(outDir); cerr != nil { - log.Fatalf("Error while removing temporary directory %s : %v", outDir, cerr) - } - })() - - src, err := data.ParseSource("config=test/files/input-dir/config.yml") - assert.Nil(t, err) - - d := &data.Data{ - Sources: map[string]*data.Source{"config": src}, - } - gomplate := NewGomplate(d, "{{", "}}") - err = processInputDir(filepath.Join("test", "files", "input-dir", "in"), outDir, []string{"**/*.exclude.txt"}, gomplate) - assert.Nil(t, err) - - top, err := ioutil.ReadFile(filepath.Join(outDir, "top.txt")) - assert.Nil(t, err) - assert.Equal(t, "eins", string(top)) - - inner, err := ioutil.ReadFile(filepath.Join(outDir, "inner/nested.txt")) - assert.Nil(t, err) - assert.Equal(t, "zwei", string(inner)) - - // excluded file should not exist in out dir - _, err = ioutil.ReadFile(filepath.Join(outDir, "inner/exclude.txt")) - assert.NotEmpty(t, err) -} diff --git a/template.go b/template.go new file mode 100644 index 000000000..9ba8c6b95 --- /dev/null +++ b/template.go @@ -0,0 +1,169 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "text/template" +) + +// input - models an input file... +type input struct { + name string + target io.Writer + contents string +} + +func (t *input) toGoTemplate(g *Gomplate) (*template.Template, error) { + tmpl := template.New(t.name) + tmpl.Option("missingkey=error") + tmpl.Funcs(g.funcMap) + tmpl.Delims(g.leftDelim, g.rightDelim) + return tmpl.Parse(t.contents) +} + +// gatherTemplates - gather and prepare input template(s) and output file(s) for rendering +func gatherTemplates(o *GomplateOpts) (ins []*input, err error) { + // the arg-provided input string gets a special name + if o.input != "" { + o.inputFiles = []string{""} + } + + // input dirs presume output dirs are set too + if o.inputDir != "" { + excludes, err := executeCombinedGlob(o.excludeGlob) + if err != nil { + return nil, err + } + + o.inputFiles, o.outputFiles, err = walkDir(o.inputDir, o.outputDir, excludes) + if err != nil { + return nil, err + } + } + + if len(o.outputFiles) == 0 { + o.outputFiles = []string{"-"} + } + + for i, filename := range o.inputFiles { + contents, err := readInput(filename) + if err != nil { + return nil, err + } + target, err := openOutFile(o.outputFiles[i]) + if err != nil { + return nil, err + } + addCleanupHook(func() { + // nolint: errcheck + target.Close() + }) + ins = append(ins, &input{ + name: filename, + target: target, + contents: contents, + }) + } + + return nil, nil +} + +func walkDir(dir, outDir string, excludes []string) ([]string, []string, error) { + dir = filepath.Clean(dir) + outDir = filepath.Clean(outDir) + + si, err := os.Stat(dir) + if err != nil { + return nil, nil, err + } + + entries, err := ioutil.ReadDir(dir) + if err != nil { + return nil, nil, err + } + + if err = os.MkdirAll(outDir, si.Mode()); err != nil { + return nil, nil, err + } + + inFiles := []string{} + outFiles := []string{} + for _, entry := range entries { + nextInPath := filepath.Join(dir, entry.Name()) + nextOutPath := filepath.Join(outDir, entry.Name()) + + if inList(excludes, nextInPath) { + continue + } + + if entry.IsDir() { + i, o, err := walkDir(nextInPath, nextOutPath, excludes) + if err != nil { + return nil, nil, err + } + inFiles = append(inFiles, i...) + outFiles = append(outFiles, o...) + } else { + inFiles = append(inFiles, nextInPath) + outFiles = append(outFiles, nextOutPath) + } + } + return inFiles, outFiles, nil +} + +func inList(list []string, entry string) bool { + for _, file := range list { + if file == entry { + return true + } + } + + return false +} + +func openOutFile(filename string) (out *os.File, err error) { + if filename == "-" { + return os.Stdout, nil + } + return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) +} + +func readInput(filename string) (string, error) { + var err error + var inFile *os.File + if filename == "-" { + inFile = os.Stdin + } else { + inFile, err = os.Open(filename) + if err != nil { + return "", fmt.Errorf("failed to open %s\n%v", filename, err) + } + // nolint: errcheck + defer inFile.Close() + } + bytes, err := ioutil.ReadAll(inFile) + if err != nil { + err = fmt.Errorf("read failed for %s\n%v", filename, err) + return "", err + } + return string(bytes), nil +} + +// takes an array of glob strings and executes it as a whole, +// returning a merged list of globbed files +func executeCombinedGlob(globArray []string) ([]string, error) { + var combinedExcludes []string + for _, glob := range globArray { + excludeList, err := filepath.Glob(glob) + if err != nil { + return nil, err + } + + combinedExcludes = append(combinedExcludes, excludeList...) + } + + return combinedExcludes, nil +} diff --git a/template_test.go b/template_test.go new file mode 100644 index 000000000..d678a3e0d --- /dev/null +++ b/template_test.go @@ -0,0 +1,52 @@ +// +build !windows + +package main + +// func TestReadInput(t *testing.T) { +// actual, err := readInputs("foo", nil) +// assert.Nil(t, err) +// assert.Equal(t, &input{"", "foo"}, actual[0]) + +// // stdin is "" because during tests it's given /dev/null +// actual, err = readInputs("", []string{"-"}) +// assert.Nil(t, err) +// assert.Equal(t, &input{"-", ""}, actual[0]) + +// actual, err = readInputs("", []string{"template_test.go"}) +// assert.Nil(t, err) +// thisFile, _ := os.Open("template_test.go") +// expected, _ := ioutil.ReadAll(thisFile) +// assert.Equal(t, &input{"template_test.go", string(expected)}, actual[0]) +// } + +// func TestInputDir(t *testing.T) { +// outDir, err := ioutil.TempDir(filepath.Join("test", "files", "input-dir"), "out-temp-") +// assert.Nil(t, err) +// defer (func() { +// if cerr := os.RemoveAll(outDir); cerr != nil { +// log.Fatalf("Error while removing temporary directory %s : %v", outDir, cerr) +// } +// })() + +// src, err := data.ParseSource("config=test/files/input-dir/config.yml") +// assert.Nil(t, err) + +// d := &data.Data{ +// Sources: map[string]*data.Source{"config": src}, +// } +// gomplate := NewGomplate(d, "{{", "}}") +// err = processInputDir(filepath.Join("test", "files", "input-dir", "in"), outDir, []string{"**/*.exclude.txt"}, gomplate) +// assert.Nil(t, err) + +// top, err := ioutil.ReadFile(filepath.Join(outDir, "top.txt")) +// assert.Nil(t, err) +// assert.Equal(t, "eins", string(top)) + +// inner, err := ioutil.ReadFile(filepath.Join(outDir, "inner/nested.txt")) +// assert.Nil(t, err) +// assert.Equal(t, "zwei", string(inner)) + +// // excluded file should not exist in out dir +// _, err = ioutil.ReadFile(filepath.Join(outDir, "inner/exclude.txt")) +// assert.NotEmpty(t, err) +// }