Adding vault list support

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

data/datasource.go

@@ -23,7 +23,10 @@ import (
 	"github.com/hairyhenderson/gomplate/vault"
 )
 
-var jsonMimetype = "application/json"
+const (
+	plaintext    = "text/plain"
+	jsonMimetype = "application/json"
+)
 
 // stdin - for overriding in tests
 var stdin io.Reader
@@ -226,8 +229,6 @@ func (d *Data) DatasourceExists(alias string) bool {
 	return ok
 }
 
-const plaintext = "text/plain"
-
 // Datasource -
 func (d *Data) Datasource(alias string, args ...string) (interface{}, error) {
 	source, ok := d.Sources[alias]
@@ -239,22 +240,24 @@ func (d *Data) Datasource(alias string, args ...string) (interface{}, error) {
 		return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", alias)
 	}
 	s := string(b)
-	if source.Type == jsonMimetype {
-		return JSON(s), nil
-	}
-	if source.Type == "application/yaml" {
-		return YAML(s), nil
-	}
-	if source.Type == "text/csv" {
-		return CSV(s), nil
-	}
-	if source.Type == "application/toml" {
-		return TOML(s), nil
-	}
-	if source.Type == plaintext {
-		return s, nil
-	}
-	return nil, errors.Errorf("Datasources of type %s not yet supported", source.Type)
+	var out interface{}
+	switch source.Type {
+	case jsonMimetype:
+		out = JSON(s)
+	case "application/array+json":
+		out = JSONArray(s)
+	case "application/yaml":
+		out = YAML(s)
+	case "text/csv":
+		out = CSV(s)
+	case "application/toml":
+		out = TOML(s)
+	case plaintext:
+		out = s
+	default:
+		return nil, errors.Errorf("Datasources of type %s not yet supported", source.Type)
+	}
+	return out, nil
 }
 
 // DatasourceReachable - Determines if the named datasource is reachable with
@@ -379,54 +382,6 @@ func readHTTP(source *Source, args ...string) ([]byte, error) {
 	return body, nil
 }
 
-func readVault(source *Source, args ...string) ([]byte, error) {
-	if source.VC == nil {
-		source.VC = vault.New(source.URL)
-		source.VC.Login()
-	}
-
-	params := make(map[string]interface{})
-
-	p := source.URL.Path
-
-	for key, val := range source.URL.Query() {
-		params[key] = strings.Join(val, " ")
-	}
-
-	if len(args) == 1 {
-		parsed, err := url.Parse(args[0])
-		if err != nil {
-			return nil, err
-		}
-
-		if parsed.Path != "" {
-			p = p + "/" + parsed.Path
-		}
-
-		for key, val := range parsed.Query() {
-			params[key] = strings.Join(val, " ")
-		}
-	}
-
-	var data []byte
-	var err error
-
-	if len(params) > 0 {
-		data, err = source.VC.Write(p, params)
-	} else {
-		data, err = source.VC.Read(p)
-	}
-	if err != nil {
-		return nil, err
-	}
-	if len(data) == 0 {
-		return nil, errors.Errorf("no value found for path %s", p)
-	}
-	source.Type = "application/json"
-
-	return data, nil
-}
-
 func readConsul(source *Source, args ...string) ([]byte, error) {
 	if source.KV == nil {
 		source.KV = libkv.NewConsul(source.URL)

data/datasource_vault.go

@@ -0,0 +1,66 @@
+package data
+
+import (
+	"net/url"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/hairyhenderson/gomplate/vault"
+)
+
+func parseVaultParams(sourceURL *url.URL, args []string) (params map[string]interface{}, p string, err error) {
+	p = sourceURL.Path
+	params = make(map[string]interface{})
+	for key, val := range sourceURL.Query() {
+		params[key] = strings.Join(val, " ")
+	}
+
+	if len(args) == 1 {
+		parsed, err := url.Parse(args[0])
+		if err != nil {
+			return nil, "", err
+		}
+
+		if parsed.Path != "" {
+			p = p + "/" + parsed.Path
+		}
+
+		for key, val := range parsed.Query() {
+			params[key] = strings.Join(val, " ")
+		}
+	}
+	return params, p, nil
+}
+
+func readVault(source *Source, args ...string) ([]byte, error) {
+	if source.VC == nil {
+		source.VC = vault.New(source.URL)
+		source.VC.Login()
+	}
+
+	params, p, err := parseVaultParams(source.URL, args)
+	if err != nil {
+		return nil, err
+	}
+
+	var data []byte
+
+	source.Type = "application/json"
+	if len(params) > 0 {
+		data, err = source.VC.Write(p, params)
+	} else if strings.HasSuffix(p, "/") {
+		source.Type = "application/array+json"
+		data, err = source.VC.List(p)
+	} else {
+		data, err = source.VC.Read(p)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(data) == 0 {
+		return nil, errors.Errorf("no value found for path %s", p)
+	}
+
+	return data, nil
+}

data/datasource_vault_test.go

@@ -0,0 +1,48 @@
+package data
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/hairyhenderson/gomplate/vault"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadVault(t *testing.T) {
+	expected := "{\"value\":\"foo\"}\n"
+	server, v := vault.MockServer(200, `{"data":`+expected+`}`)
+	defer server.Close()
+
+	source := &Source{
+		Alias: "foo",
+		URL:   &url.URL{Scheme: "vault", Path: "/secret/foo"},
+		Ext:   "",
+		Type:  "text/plain",
+		VC:    v,
+	}
+
+	r, err := readVault(source)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(expected), r)
+
+	r, err = readVault(source, "bar")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(expected), r)
+
+	r, err = readVault(source, "?param=value")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(expected), r)
+
+	source.URL, _ = url.Parse("vault:///secret/foo?param1=value1&param2=value2")
+	r, err = readVault(source)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(expected), r)
+
+	expected = "[\"one\",\"two\"]\n"
+	server, source.VC = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`)
+	defer server.Close()
+	source.URL, _ = url.Parse("vault:///secret/foo/")
+	r, err = readVault(source)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte(expected), r)
+}

docs/content/functions/data.md

@@ -13,7 +13,7 @@ Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d)
 
 Currently, `file://`, `stdin://`, `http://`, `https://`, `vault://`, and `boltdb://` URLs are supported.
 
-Currently-supported formats are JSON, YAML, TOML, and CSV.
+Currently-supported formats are JSON, YAML, TOML, and CSV. Plain-text datasources can also be specified, but can only be safely accessed with the [`include`](#include) function.
 
 ### Basic usage
 
@@ -231,6 +231,8 @@ The `vault+http://` URL scheme can be used to indicate that request must be sent
 over regular unencrypted HTTP, while `vault+https://` and `vault://` are equivalent,
 and indicate that requests must be sent over HTTPS.
 
+List support is also available when the URL ends with a `/` character. In order for this to work correctly, the authenticated token must have permission to use the [`list` capability](https://www.vaultproject.io/docs/concepts/policies.html#list) for the given path.
+
 This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence:
 
 | auth backend | configuration |
@@ -345,8 +347,7 @@ Alias to [`datasource`](#datasource)
 
 Includes the content of a given datasource (provided by the [`--datasource/-d`](../usage/#datasource-d) argument).
 
-This is similar to [`datasource`](#datasource),
-except that the data is not parsed.
+This is similar to [`datasource`](#datasource), except that the data is not parsed. There is no restriction on the type of data included, except that it should be textual.
 
 ### Usage
 

test/integration/datasources_vault_test.go

@@ -37,11 +37,15 @@ func (s *VaultDatasourcesSuite) SetUpSuite(c *C) {
 	handle(c, err)
 
 	err = s.v.vc.Sys().PutPolicy("writepol", `path "*" {
-  policy = "write"
+  capabilities = ["create","update","delete"]
 }`)
 	handle(c, err)
 	err = s.v.vc.Sys().PutPolicy("readpol", `path "*" {
-  policy = "read"
+  capabilities = ["read","delete"]
+}`)
+	handle(c, err)
+	err = s.v.vc.Sys().PutPolicy("listPol", `path "*" {
+  capabilities = ["read","list","delete"]
 }`)
 	handle(c, err)
 }
@@ -359,3 +363,33 @@ func (s *VaultDatasourcesSuite) TestDynamicAuth(c *C) {
 		result.Assert(c, icmd.Expected{ExitCode: 0, Out: "10.1.2.3"})
 	}
 }
+
+func (s *VaultDatasourcesSuite) TestList(c *C) {
+	s.v.vc.Logical().Write("secret/dir/foo", map[string]interface{}{"value": "one"})
+	s.v.vc.Logical().Write("secret/dir/bar", map[string]interface{}{"value": "two"})
+	defer s.v.vc.Logical().Delete("secret/dir/foo")
+	defer s.v.vc.Logical().Delete("secret/dir/bar")
+	tok, err := s.v.tokenCreate("listpol", 5)
+	handle(c, err)
+
+	result := icmd.RunCmd(icmd.Command(GomplateBin,
+		"-d", "vault=vault:///secret/dir/",
+		"-i", `{{ range (ds "vault" ) }}{{ . }}: {{ (ds "vault" .).value }} {{end}}`,
+	), func(c *icmd.Cmd) {
+		c.Env = []string{
+			"VAULT_ADDR=http://" + s.v.addr,
+			"VAULT_TOKEN=" + tok,
+		}
+	})
+	result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar: two foo: one"})
+
+	result = icmd.RunCmd(icmd.Command(GomplateBin,
+		"-d", "vault=vault+http://"+s.v.addr+"/secret",
+		"-i", `{{ range (ds "vault" "dir/" ) }}{{ . }} {{end}}`,
+	), func(c *icmd.Cmd) {
+		c.Env = []string{
+			"VAULT_TOKEN=" + tok,
+		}
+	})
+	result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar foo"})
+}

vault/vault.go

@@ -6,6 +6,8 @@ import (
 	"log"
 	"net/url"
 
+	"github.com/pkg/errors"
+
 	vaultapi "github.com/hashicorp/vault/api"
 )
 
@@ -90,3 +92,26 @@ func (v *Vault) Write(path string, data map[string]interface{}) ([]byte, error)
 	}
 	return buf.Bytes(), nil
 }
+
+// List -
+func (v *Vault) List(path string) ([]byte, error) {
+	secret, err := v.client.Logical().List(path)
+	if err != nil {
+		return nil, err
+	}
+	if secret == nil {
+		return nil, nil
+	}
+
+	keys, ok := secret.Data["keys"]
+	if !ok {
+		return nil, errors.Errorf("keys param missing from vault list")
+	}
+
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	if err := enc.Encode(keys); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}