From 2bb7df4155acaf0a1dbbbe4f66e0f235961cfec6 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sat, 3 Feb 2018 22:12:41 -0500 Subject: [PATCH] Allow vault address to be specified in the vault:// URL Signed-off-by: Dave Henderson --- Makefile | 2 +- data/datasource.go | 4 +++- docs/content/functions/data.md | 27 +++++++++++++++++----- libkv/consul.go | 2 +- test/integration/Dockerfile | 8 +++---- test/integration/awssvc/main.go | 2 +- test/integration/datasources_consul.bats | 3 +++ test/integration/datasources_vault.bats | 17 ++++++++++++-- test/integration/helper.bash | 29 ++++++++++++++++++++++++ test/integration/metasvc/main.go | 2 +- test/integration/mirrorsvc/main.go | 6 ++--- test/integration/test.sh | 16 ------------- vault/vault.go | 18 ++++++++++++++- vault/vault_test.go | 25 ++++++++++++++++++++ 14 files changed, 124 insertions(+), 37 deletions(-) delete mode 100755 test/integration/test.sh diff --git a/Makefile b/Makefile index 105c6bcfb..e365399e4 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ test-integration-docker: build-integration-image docker run -it --rm gomplate-test test-integration: build build-mirror build-meta build-aws - @test/integration/test.sh + @bats test/integration/ gen-changelog: github_changelog_generator --no-filter-by-milestone --exclude-labels duplicate,question,invalid,wontfix,admin diff --git a/data/datasource.go b/data/datasource.go index de3474c92..bc3e076fd 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -53,6 +53,8 @@ func init() { addSourceReader("file", readFile) addSourceReader("stdin", readStdin) addSourceReader("vault", readVault) + addSourceReader("vault+http", readVault) + addSourceReader("vault+https", readVault) addSourceReader("consul", readConsul) addSourceReader("consul+http", readConsul) addSourceReader("consul+https", readConsul) @@ -372,7 +374,7 @@ func readHTTP(source *Source, args ...string) ([]byte, error) { func readVault(source *Source, args ...string) ([]byte, error) { if source.VC == nil { - source.VC = vault.New() + source.VC = vault.New(source.URL) source.VC.Login() } diff --git a/docs/content/functions/data.md b/docs/content/functions/data.md index d19088fbd..654e45af2 100644 --- a/docs/content/functions/data.md +++ b/docs/content/functions/data.md @@ -223,8 +223,13 @@ aaa ### Usage with Vault data The special `vault://` URL scheme can be used to retrieve data from [Hashicorp -Vault](https://vaultproject.io). To use this, you must put the Vault server's -URL in the `$VAULT_ADDR` environment variable. +Vault](https://vaultproject.io). To use this, you must either provide the Vault +server's hostname and port in the URL, or put the Vault server's URL in the +`$VAULT_ADDR` environment variable. + +The `vault+http://` URL scheme can be used to indicate that request must be sent +over regular unencrypted HTTP, while `vault+https://` and `vault://` are equivalent, +and indicate that requests must be sent over HTTPS. This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: @@ -246,7 +251,7 @@ any `_FILE` variable and the secret file will be ignored. To use a Vault datasource with a single secret, just use a URL of `vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left -empty. +empty in this example. ```console $ echo 'My voice is my passport. {{(datasource "vault").value}}' \ @@ -254,11 +259,12 @@ $ echo 'My voice is my passport. {{(datasource "vault").value}}' \ My voice is my passport. Verify me. ``` -You can also specify the secret path in the template by using a URL of `vault://` -(or `vault:///`, or `vault:`): +You can also specify the secret path in the template by omitting the path portion +of the URL: + ```console $ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \ - | gomplate -d vault=vault:// + | gomplate -d vault=vault:/// My voice is my passport. Verify me. ``` @@ -270,6 +276,15 @@ $ echo 'db_password={{(datasource "vault" "db/pass").value}}' \ db_password=prodsecret ``` +If you are unable to set the `VAULT_ADDR` environment variable, or need to +specify multiple Vault datasources connecting to different servers, you can set +the address as part of the URL: + +```console +$ gomplate -d v=vault://vaultserver.com/secret/foo -i '{{ (ds "v").value }}' +bar +``` + It is also possible to use dynamic secrets by using the write capability of the datasource. To use, add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then included within the JSON body of the request. diff --git a/libkv/consul.go b/libkv/consul.go index d4fb53c43..696a14b5c 100644 --- a/libkv/consul.go +++ b/libkv/consul.go @@ -25,7 +25,7 @@ func NewConsul(u *url.URL) *LibKV { if role := env.Getenv("CONSUL_VAULT_ROLE", ""); role != "" { mount := env.Getenv("CONSUL_VAULT_MOUNT", "consul") - client := vault.New() + client := vault.New(nil) client.Login() path := fmt.Sprintf("%s/creds/%s", mount, role) diff --git a/test/integration/Dockerfile b/test/integration/Dockerfile index 894a04a70..a1ef7b3cd 100644 --- a/test/integration/Dockerfile +++ b/test/integration/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:edge -ENV VAULT_VER 0.7.3 -ENV CONSUL_VER 0.9.0 +ENV VAULT_VER 0.9.3 +ENV CONSUL_VER 1.0.3 RUN apk add --no-cache \ curl \ bash \ @@ -23,9 +23,9 @@ COPY gomplate /bin/gomplate COPY mirror /bin/mirror COPY meta /bin/meta COPY aws /bin/aws -COPY *.sh /tests/ +# COPY *.sh /tests/ COPY *.bash /tests/ COPY *.bats /tests/ COPY *.db /test/integration/ -CMD ["/tests/test.sh"] +CMD ["bats", "/tests/"] diff --git a/test/integration/awssvc/main.go b/test/integration/awssvc/main.go index 0a3f8f056..7bed22884 100644 --- a/test/integration/awssvc/main.go +++ b/test/integration/awssvc/main.go @@ -19,7 +19,7 @@ func main() { flag.StringVar(&port, "p", "8082", "Port to listen to") flag.Parse() - l, err := net.Listen("tcp", ":"+port) + l, err := net.Listen("tcp", "127.0.0.1:"+port) if err != nil { log.Fatal(err) } diff --git a/test/integration/datasources_consul.bats b/test/integration/datasources_consul.bats index 8eb5da77a..c5b86805d 100644 --- a/test/integration/datasources_consul.bats +++ b/test/integration/datasources_consul.bats @@ -3,6 +3,8 @@ load helper function setup () { + export VAULT_ADDR=http://127.0.0.1:8200 + start_vault start_consul 8501 export CONSUL_HTTP_ADDR=http://127.0.0.1:8501 } @@ -12,6 +14,7 @@ function teardown () { consul kv delete foo vault unmount consul stop_consul + stop_vault } @test "Testing consul" { diff --git a/test/integration/datasources_vault.bats b/test/integration/datasources_vault.bats index f58d0090f..ae923545c 100644 --- a/test/integration/datasources_vault.bats +++ b/test/integration/datasources_vault.bats @@ -3,6 +3,9 @@ load helper function setup () { + export VAULT_ADDR=http://127.0.0.1:8200 + start_vault + unset VAULT_TOKEN cat <& /dev/null path "*" { @@ -15,13 +18,11 @@ path "*" { } EOF tmpdir=$(mktemp -d) - cp ~/.vault-token ~/.vault-token.bak start_meta_svc start_aws_svc } function teardown () { - mv ~/.vault-token.bak ~/.vault-token stop_meta_svc stop_aws_svc rm -rf $tmpdir @@ -37,6 +38,8 @@ function teardown () { vault policy-delete writepol vault policy-delete readpol vault unmount ssh + + stop_vault } @test "Testing token vault auth" { @@ -47,6 +50,16 @@ function teardown () { [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]] } +@test "Testing token vault auth with addr in URL" { + vault write secret/foo value="$BATS_TEST_DESCRIPTION" + VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) + addr=$VAULT_ADDR + unset VAULT_ADDR + VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault+${addr}/secret -i '{{(datasource "vault" "foo").value}}' + [ "$status" -eq 0 ] + [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]] +} + @test "Testing failure with non-existant secret" { VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///secret -i '{{(datasource "vault" "bar").value}}' diff --git a/test/integration/helper.bash b/test/integration/helper.bash index dd406b09d..8d3e46cf0 100644 --- a/test/integration/helper.bash +++ b/test/integration/helper.bash @@ -28,6 +28,7 @@ function __gomplate_stdin () { function start_mirror_svc () { bin/mirror & + wait_for_url http://127.0.0.1:8080/ } function stop_mirror_svc () { @@ -75,3 +76,31 @@ function stop_consul () { kill $(cat $PID_FILE) &>/dev/null rm /tmp/gomplate-test-consul.json } + +function start_vault () { + port=$1 + if [ -z $port ]; then + port=8200 + fi + PID_FILE=/tmp/gomplate-test-vault.pid + export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666 + + # back up any existing token so it doesn't get overridden + if [ -f ~/.vault-token ]; then + cp ~/.vault-token ~/.vault-token.bak + fi + + vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null & + echo $! > $PID_FILE + wait_for_url http://127.0.0.1:$port/sys/health +} + +function stop_vault () { + PID_FILE=/tmp/gomplate-test-vault.pid + kill $(cat $PID_FILE) &>/dev/null + + # restore old token if it was backed up + if [ -f ~/.vault-token.bak ]; then + mv ~/.vault-token.bak ~/.vault-token + fi +} diff --git a/test/integration/metasvc/main.go b/test/integration/metasvc/main.go index 108b9e39d..8a298c4ec 100644 --- a/test/integration/metasvc/main.go +++ b/test/integration/metasvc/main.go @@ -41,7 +41,7 @@ func main() { flag.StringVar(&port, "p", "8081", "Port to listen to") flag.Parse() - l, err := net.Listen("tcp", ":"+port) + l, err := net.Listen("tcp", "127.0.0.1:"+port) if err != nil { log.Fatal(err) } diff --git a/test/integration/mirrorsvc/main.go b/test/integration/mirrorsvc/main.go index 343ab8ca1..52b58e5de 100644 --- a/test/integration/mirrorsvc/main.go +++ b/test/integration/mirrorsvc/main.go @@ -13,13 +13,13 @@ type Req struct { Headers http.Header `json:"headers"` } -var port string +var port int func main() { - flag.StringVar(&port, "p", "8080", "Port to listen to") + flag.IntVar(&port, "p", 8080, "Port to listen to") flag.Parse() - l, err := net.Listen("tcp", ":"+port) + l, err := net.ListenTCP("tcp", &net.TCPAddr{Port: port}) if err != nil { log.Fatal(err) } diff --git a/test/integration/test.sh b/test/integration/test.sh deleted file mode 100755 index 270c1cdac..000000000 --- a/test/integration/test.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This is useful for killing vault after the script exits, but causes the CircleCI -# build to fail, so... ¯\_(ツ)_/¯ -# trap "exit" INT TERM -# trap "kill 0" EXIT - -# TODO: export these in a bats helper, as well as only launch vault in a vault helper -export VAULT_ADDR=http://127.0.0.1:8200 -export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666 - -# fire up vault in dev mode for the vault tests -vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null & - -bats $(dirname $0) diff --git a/vault/vault.go b/vault/vault.go index b07d2483f..54af08397 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "log" + "net/url" vaultapi "github.com/hashicorp/vault/api" ) @@ -17,7 +18,7 @@ type Vault struct { } // New - -func New() *Vault { +func New(u *url.URL) *Vault { vaultConfig := vaultapi.DefaultConfig() err := vaultConfig.ReadEnvironment() @@ -25,6 +26,10 @@ func New() *Vault { logFatal("Vault setup failed", err) } + if vu := vaultURL(u); vu != "" { + vaultConfig.Address = vu + } + client, err := vaultapi.NewClient(vaultConfig) if err != nil { logFatal("Vault setup failed", err) @@ -33,6 +38,17 @@ func New() *Vault { return &Vault{client} } +func vaultURL(u *url.URL) string { + if u != nil && u.Host != "" { + scheme := "https" + if u.Scheme == "vault+http" { + scheme = "http" + } + return scheme + "://" + u.Host + } + return "" +} + // Login - func (v *Vault) Login() { v.client.SetToken(v.GetToken()) diff --git a/vault/vault_test.go b/vault/vault_test.go index 9f4777a6b..0bcc3259f 100644 --- a/vault/vault_test.go +++ b/vault/vault_test.go @@ -1,11 +1,36 @@ package vault import ( + "net/url" + "os" "testing" "github.com/stretchr/testify/assert" ) +func TestNew(t *testing.T) { + v := New(nil) + assert.Equal(t, "https://127.0.0.1:8200", v.client.Address()) + + os.Setenv("VAULT_ADDR", "http://example.com:1234") + defer os.Unsetenv("VAULT_ADDR") + v = New(nil) + assert.Equal(t, "http://example.com:1234", v.client.Address()) + os.Unsetenv("VAULT_ADDR") + + u, _ := url.Parse("vault://vault.rocks:8200/secret/foo/bar") + v = New(u) + assert.Equal(t, "https://vault.rocks:8200", v.client.Address()) + + u, _ = url.Parse("vault+https://vault.rocks:8200/secret/foo/bar") + v = New(u) + assert.Equal(t, "https://vault.rocks:8200", v.client.Address()) + + u, _ = url.Parse("vault+http://vault.rocks:8200/secret/foo/bar") + v = New(u) + assert.Equal(t, "http://vault.rocks:8200", v.client.Address()) +} + func TestRead(t *testing.T) { server, v := MockServer(404, "Not Found") defer server.Close()