From b5c2992ecb4b5fab8bde2a54dd5fc1f5bcaca45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Mon, 24 Apr 2017 09:54:47 +0200 Subject: [PATCH] Add support for datasource directories When the argument given with `--datasource` point to a directory, every file in this directory is processed with the plain-file logic for datasource (e.g. name of ds == filebasename, type determined by file extension). Related to #117 --- README.md | 4 +- data.go | 54 ++++++++++++++++++------ data_test.go | 3 +- process_test.go | 25 +++++++++++ test/files/datasource-dir/ds/french.json | 4 ++ test/files/datasource-dir/ds/german.yml | 2 + test/files/datasource-dir/in/test.txt | 1 + 7 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 test/files/datasource-dir/ds/french.json create mode 100644 test/files/datasource-dir/ds/german.yml create mode 100644 test/files/datasource-dir/in/test.txt diff --git a/README.md b/README.md index 9ca740bd2..a9a3fe1e8 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,9 @@ A few different forms are valid: - Create a data source named `mydata` which is read from `file.json` (in the current working directory). This form is only valid for files in the current directory. - `mydata.json` - This form infers the name from the file name (without extension). Only valid for files in the current directory. - +- `mydatadir/` + - If a plain directory is given then any plain file within the directory is treated as a data file as described above. + #### Overriding the template delimiters Sometimes it's necessary to override the default template delimiters (`{{`/`}}`). diff --git a/data.go b/data.go index e2b996877..049526118 100644 --- a/data.go +++ b/data.go @@ -62,13 +62,15 @@ func NewData(datasourceArgs []string, headerArgs []string) *Data { sources := make(map[string]*Source) headers := parseHeaderArgs(headerArgs) for _, v := range datasourceArgs { - s, err := ParseSource(v) - if err != nil { - log.Fatalf("error parsing datasource %v", err) - return nil + for _, arg := range fanOutDatasourceArgs(v) { + s, err := ParseSource(arg) + if err != nil { + log.Fatalf("error parsing datasource %s: %v", arg, err) + return nil + } + s.Header = headers[s.Alias] + sources[s.Alias] = s } - s.Header = headers[s.Alias] - sources[s.Alias] = s } return &Data{ Sources: sources, @@ -119,13 +121,13 @@ func ParseSource(value string) (*Source, error) { ) parts := strings.SplitN(value, "=", 2) if len(parts) == 1 { - f := parts[0] - alias = strings.SplitN(value, ".", 2)[0] - if path.Base(f) != f { - err := fmt.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory", value) + srcURL = absURL(value) + if srcURL.Scheme == "file" { + alias = strings.SplitN(path.Base(value), ".", 2)[0] + } else { + err := fmt.Errorf("Invalid datasource (%s). Must provide an alias name for non-file datasources", value) return nil, err } - srcURL = absURL(f) } else if len(parts) == 2 { alias = parts[0] var err error @@ -144,6 +146,14 @@ func ParseSource(value string) (*Source, error) { } func absURL(value string) *url.URL { + parsedURL, err := url.Parse(value) + if err != nil || parsedURL.Scheme == "" || !parsedURL.IsAbs() { + return fileToURL(value) + } + return parsedURL +} + +func fileToURL(path string) *url.URL { cwd, err := os.Getwd() if err != nil { log.Fatalf("Can't get working directory: %s", err) @@ -153,7 +163,7 @@ func absURL(value string) *url.URL { Path: cwd + "/", } relURL := &url.URL{ - Path: value, + Path: path, } return baseURL.ResolveReference(relURL) } @@ -212,6 +222,26 @@ func (d *Data) ReadSource(fs vfs.Filesystem, source *Source, args ...string) ([] return nil, nil } +// If argument is a plain directory, use dir contents as datasource files +func fanOutDatasourceArgs(arg string) []string { + if _, err := statDir(arg); err == nil { + files, err := ioutil.ReadDir(arg) + if err != nil { + log.Fatalf("error parsing datasource directory %s %v", arg, err) + return nil + } + + var paths []string + for _, file := range files { + if !file.IsDir() { + paths = append(paths, path.Join(arg, file.Name())) + } + } + return paths + } + return []string{arg} +} + func readFile(source *Source, args ...string) ([]byte, error) { if source.FS == nil { source.FS = vfs.OS() diff --git a/data_test.go b/data_test.go index d4f2000d1..e19106a9b 100644 --- a/data_test.go +++ b/data_test.go @@ -81,7 +81,8 @@ func TestParseSourceNoAlias(t *testing.T) { assert.Equal(t, "foo", s.Alias) _, err = ParseSource("../foo.json") - assert.Error(t, err) + assert.NoError(t, err) + assert.Equal(t, "foo", s.Alias) _, err = ParseSource("ftp://example.com/foo.yml") assert.Error(t, err) diff --git a/process_test.go b/process_test.go index 5b7fc812c..01ec27630 100644 --- a/process_test.go +++ b/process_test.go @@ -9,9 +9,34 @@ import ( "log" + "path" + "github.com/stretchr/testify/assert" ) +func TestDatasourceDir(t *testing.T) { + outDir, err := ioutil.TempDir("test/files/datasource-dir", "out-temp-") + assert.Nil(t, err) + defer (func() { + if cerr := os.RemoveAll(outDir); cerr != nil { + log.Fatalf("Error while removing temporary directory %s : %v", outDir, cerr) + } + })() + + data := NewData([]string{"test/files/datasource-dir/ds"}, []string{}) + gomplate := NewGomplate(data, "{{", "}}") + err = processInputFiles( + "", + []string{"test/files/datasource-dir/in/test.txt"}, + []string{path.Join(outDir, "out.txt")}, + gomplate) + assert.NoError(t, err) + + out, err := ioutil.ReadFile(filepath.Join(outDir, "out.txt")) + assert.NoError(t, err) + assert.Equal(t, "eins-deux", string(out)) +} + func TestReadInput(t *testing.T) { actual, err := readInputs("foo", nil) assert.Nil(t, err) diff --git a/test/files/datasource-dir/ds/french.json b/test/files/datasource-dir/ds/french.json new file mode 100644 index 000000000..140c55701 --- /dev/null +++ b/test/files/datasource-dir/ds/french.json @@ -0,0 +1,4 @@ +{ + "one": "un", + "two": "deux" +} diff --git a/test/files/datasource-dir/ds/german.yml b/test/files/datasource-dir/ds/german.yml new file mode 100644 index 000000000..239f065c9 --- /dev/null +++ b/test/files/datasource-dir/ds/german.yml @@ -0,0 +1,2 @@ +one: eins +two: zwei \ No newline at end of file diff --git a/test/files/datasource-dir/in/test.txt b/test/files/datasource-dir/in/test.txt new file mode 100644 index 000000000..08b98bb01 --- /dev/null +++ b/test/files/datasource-dir/in/test.txt @@ -0,0 +1 @@ +{{ (datasource "german").one }}-{{ (datasource "french").two }} \ No newline at end of file