From 64041c1f7038d4ddbf8b78ca130d00b6e564b5bb Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Fri, 18 May 2018 15:43:24 -0400 Subject: [PATCH] Adding vault list support Signed-off-by: Dave Henderson --- data/datasource.go | 89 ++++++---------------- data/datasource_vault.go | 66 ++++++++++++++++ data/datasource_vault_test.go | 48 ++++++++++++ docs/content/functions/data.md | 7 +- test/integration/datasources_vault_test.go | 38 ++++++++- vault/vault.go | 25 ++++++ 6 files changed, 201 insertions(+), 72 deletions(-) create mode 100644 data/datasource_vault.go create mode 100644 data/datasource_vault_test.go diff --git a/data/datasource.go b/data/datasource.go index 6370bd5fa..8f9857b43 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -23,7 +23,10 @@ import ( "github.com/hairyhenderson/gomplate/vault" ) -var jsonMimetype = "application/json" +const ( + plaintext = "text/plain" + jsonMimetype = "application/json" +) // stdin - for overriding in tests var stdin io.Reader @@ -226,8 +229,6 @@ func (d *Data) DatasourceExists(alias string) bool { return ok } -const plaintext = "text/plain" - // Datasource - func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { source, ok := d.Sources[alias] @@ -239,22 +240,24 @@ func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", alias) } s := string(b) - if source.Type == jsonMimetype { - return JSON(s), nil - } - if source.Type == "application/yaml" { - return YAML(s), nil - } - if source.Type == "text/csv" { - return CSV(s), nil - } - if source.Type == "application/toml" { - return TOML(s), nil - } - if source.Type == plaintext { - return s, nil - } - return nil, errors.Errorf("Datasources of type %s not yet supported", source.Type) + var out interface{} + switch source.Type { + case jsonMimetype: + out = JSON(s) + case "application/array+json": + out = JSONArray(s) + case "application/yaml": + out = YAML(s) + case "text/csv": + out = CSV(s) + case "application/toml": + out = TOML(s) + case plaintext: + out = s + default: + return nil, errors.Errorf("Datasources of type %s not yet supported", source.Type) + } + return out, nil } // DatasourceReachable - Determines if the named datasource is reachable with @@ -379,54 +382,6 @@ func readHTTP(source *Source, args ...string) ([]byte, error) { return body, nil } -func readVault(source *Source, args ...string) ([]byte, error) { - if source.VC == nil { - source.VC = vault.New(source.URL) - source.VC.Login() - } - - params := make(map[string]interface{}) - - p := source.URL.Path - - for key, val := range source.URL.Query() { - params[key] = strings.Join(val, " ") - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, err - } - - if parsed.Path != "" { - p = p + "/" + parsed.Path - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - - var data []byte - var err error - - if len(params) > 0 { - data, err = source.VC.Write(p, params) - } else { - data, err = source.VC.Read(p) - } - if err != nil { - return nil, err - } - if len(data) == 0 { - return nil, errors.Errorf("no value found for path %s", p) - } - source.Type = "application/json" - - return data, nil -} - func readConsul(source *Source, args ...string) ([]byte, error) { if source.KV == nil { source.KV = libkv.NewConsul(source.URL) diff --git a/data/datasource_vault.go b/data/datasource_vault.go new file mode 100644 index 000000000..9033a804c --- /dev/null +++ b/data/datasource_vault.go @@ -0,0 +1,66 @@ +package data + +import ( + "net/url" + "strings" + + "github.com/pkg/errors" + + "github.com/hairyhenderson/gomplate/vault" +) + +func parseVaultParams(sourceURL *url.URL, args []string) (params map[string]interface{}, p string, err error) { + p = sourceURL.Path + params = make(map[string]interface{}) + for key, val := range sourceURL.Query() { + params[key] = strings.Join(val, " ") + } + + if len(args) == 1 { + parsed, err := url.Parse(args[0]) + if err != nil { + return nil, "", err + } + + if parsed.Path != "" { + p = p + "/" + parsed.Path + } + + for key, val := range parsed.Query() { + params[key] = strings.Join(val, " ") + } + } + return params, p, nil +} + +func readVault(source *Source, args ...string) ([]byte, error) { + if source.VC == nil { + source.VC = vault.New(source.URL) + source.VC.Login() + } + + params, p, err := parseVaultParams(source.URL, args) + if err != nil { + return nil, err + } + + var data []byte + + source.Type = "application/json" + if len(params) > 0 { + data, err = source.VC.Write(p, params) + } else if strings.HasSuffix(p, "/") { + source.Type = "application/array+json" + data, err = source.VC.List(p) + } else { + data, err = source.VC.Read(p) + } + if err != nil { + return nil, err + } + if len(data) == 0 { + return nil, errors.Errorf("no value found for path %s", p) + } + + return data, nil +} diff --git a/data/datasource_vault_test.go b/data/datasource_vault_test.go new file mode 100644 index 000000000..fc7acec15 --- /dev/null +++ b/data/datasource_vault_test.go @@ -0,0 +1,48 @@ +package data + +import ( + "net/url" + "testing" + + "github.com/hairyhenderson/gomplate/vault" + "github.com/stretchr/testify/assert" +) + +func TestReadVault(t *testing.T) { + expected := "{\"value\":\"foo\"}\n" + server, v := vault.MockServer(200, `{"data":`+expected+`}`) + defer server.Close() + + source := &Source{ + Alias: "foo", + URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, + Ext: "", + Type: "text/plain", + VC: v, + } + + r, err := readVault(source) + assert.NoError(t, err) + assert.Equal(t, []byte(expected), r) + + r, err = readVault(source, "bar") + assert.NoError(t, err) + assert.Equal(t, []byte(expected), r) + + r, err = readVault(source, "?param=value") + assert.NoError(t, err) + assert.Equal(t, []byte(expected), r) + + source.URL, _ = url.Parse("vault:///secret/foo?param1=value1¶m2=value2") + r, err = readVault(source) + assert.NoError(t, err) + assert.Equal(t, []byte(expected), r) + + expected = "[\"one\",\"two\"]\n" + server, source.VC = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) + defer server.Close() + source.URL, _ = url.Parse("vault:///secret/foo/") + r, err = readVault(source) + assert.NoError(t, err) + assert.Equal(t, []byte(expected), r) +} diff --git a/docs/content/functions/data.md b/docs/content/functions/data.md index 277daafb5..a2a187062 100644 --- a/docs/content/functions/data.md +++ b/docs/content/functions/data.md @@ -13,7 +13,7 @@ Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) Currently, `file://`, `stdin://`, `http://`, `https://`, `vault://`, and `boltdb://` URLs are supported. -Currently-supported formats are JSON, YAML, TOML, and CSV. +Currently-supported formats are JSON, YAML, TOML, and CSV. Plain-text datasources can also be specified, but can only be safely accessed with the [`include`](#include) function. ### Basic usage @@ -231,6 +231,8 @@ The `vault+http://` URL scheme can be used to indicate that request must be sent over regular unencrypted HTTP, while `vault+https://` and `vault://` are equivalent, and indicate that requests must be sent over HTTPS. +List support is also available when the URL ends with a `/` character. In order for this to work correctly, the authenticated token must have permission to use the [`list` capability](https://www.vaultproject.io/docs/concepts/policies.html#list) for the given path. + This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: | auth backend | configuration | @@ -345,8 +347,7 @@ Alias to [`datasource`](#datasource) Includes the content of a given datasource (provided by the [`--datasource/-d`](../usage/#datasource-d) argument). -This is similar to [`datasource`](#datasource), -except that the data is not parsed. +This is similar to [`datasource`](#datasource), except that the data is not parsed. There is no restriction on the type of data included, except that it should be textual. ### Usage diff --git a/test/integration/datasources_vault_test.go b/test/integration/datasources_vault_test.go index b0797e04b..ca6822285 100644 --- a/test/integration/datasources_vault_test.go +++ b/test/integration/datasources_vault_test.go @@ -37,11 +37,15 @@ func (s *VaultDatasourcesSuite) SetUpSuite(c *C) { handle(c, err) err = s.v.vc.Sys().PutPolicy("writepol", `path "*" { - policy = "write" + capabilities = ["create","update","delete"] }`) handle(c, err) err = s.v.vc.Sys().PutPolicy("readpol", `path "*" { - policy = "read" + capabilities = ["read","delete"] +}`) + handle(c, err) + err = s.v.vc.Sys().PutPolicy("listPol", `path "*" { + capabilities = ["read","list","delete"] }`) handle(c, err) } @@ -359,3 +363,33 @@ func (s *VaultDatasourcesSuite) TestDynamicAuth(c *C) { result.Assert(c, icmd.Expected{ExitCode: 0, Out: "10.1.2.3"}) } } + +func (s *VaultDatasourcesSuite) TestList(c *C) { + s.v.vc.Logical().Write("secret/dir/foo", map[string]interface{}{"value": "one"}) + s.v.vc.Logical().Write("secret/dir/bar", map[string]interface{}{"value": "two"}) + defer s.v.vc.Logical().Delete("secret/dir/foo") + defer s.v.vc.Logical().Delete("secret/dir/bar") + tok, err := s.v.tokenCreate("listpol", 5) + handle(c, err) + + result := icmd.RunCmd(icmd.Command(GomplateBin, + "-d", "vault=vault:///secret/dir/", + "-i", `{{ range (ds "vault" ) }}{{ . }}: {{ (ds "vault" .).value }} {{end}}`, + ), func(c *icmd.Cmd) { + c.Env = []string{ + "VAULT_ADDR=http://" + s.v.addr, + "VAULT_TOKEN=" + tok, + } + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar: two foo: one"}) + + result = icmd.RunCmd(icmd.Command(GomplateBin, + "-d", "vault=vault+http://"+s.v.addr+"/secret", + "-i", `{{ range (ds "vault" "dir/" ) }}{{ . }} {{end}}`, + ), func(c *icmd.Cmd) { + c.Env = []string{ + "VAULT_TOKEN=" + tok, + } + }) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar foo"}) +} diff --git a/vault/vault.go b/vault/vault.go index 57244691f..f91d48941 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -6,6 +6,8 @@ import ( "log" "net/url" + "github.com/pkg/errors" + vaultapi "github.com/hashicorp/vault/api" ) @@ -90,3 +92,26 @@ func (v *Vault) Write(path string, data map[string]interface{}) ([]byte, error) } return buf.Bytes(), nil } + +// List - +func (v *Vault) List(path string) ([]byte, error) { + secret, err := v.client.Logical().List(path) + if err != nil { + return nil, err + } + if secret == nil { + return nil, nil + } + + keys, ok := secret.Data["keys"] + if !ok { + return nil, errors.Errorf("keys param missing from vault list") + } + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + if err := enc.Encode(keys); err != nil { + return nil, err + } + return buf.Bytes(), nil +}