diff --git a/Makefile b/Makefile index 105c6bcfb..fa40f449c 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,7 @@ test-integration-docker: build-integration-image docker run -it --rm gomplate-test test-integration: build build-mirror build-meta build-aws + # @bats test/integration/ @test/integration/test.sh gen-changelog: diff --git a/data/datasource.go b/data/datasource.go index de3474c92..bc3e076fd 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -53,6 +53,8 @@ func init() { addSourceReader("file", readFile) addSourceReader("stdin", readStdin) addSourceReader("vault", readVault) + addSourceReader("vault+http", readVault) + addSourceReader("vault+https", readVault) addSourceReader("consul", readConsul) addSourceReader("consul+http", readConsul) addSourceReader("consul+https", readConsul) @@ -372,7 +374,7 @@ func readHTTP(source *Source, args ...string) ([]byte, error) { func readVault(source *Source, args ...string) ([]byte, error) { if source.VC == nil { - source.VC = vault.New() + source.VC = vault.New(source.URL) source.VC.Login() } diff --git a/docs/content/functions/data.md b/docs/content/functions/data.md index d19088fbd..654e45af2 100644 --- a/docs/content/functions/data.md +++ b/docs/content/functions/data.md @@ -223,8 +223,13 @@ aaa ### Usage with Vault data The special `vault://` URL scheme can be used to retrieve data from [Hashicorp -Vault](https://vaultproject.io). To use this, you must put the Vault server's -URL in the `$VAULT_ADDR` environment variable. +Vault](https://vaultproject.io). To use this, you must either provide the Vault +server's hostname and port in the URL, or put the Vault server's URL in the +`$VAULT_ADDR` environment variable. + +The `vault+http://` URL scheme can be used to indicate that request must be sent +over regular unencrypted HTTP, while `vault+https://` and `vault://` are equivalent, +and indicate that requests must be sent over HTTPS. This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence: @@ -246,7 +251,7 @@ any `_FILE` variable and the secret file will be ignored. To use a Vault datasource with a single secret, just use a URL of `vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left -empty. +empty in this example. ```console $ echo 'My voice is my passport. {{(datasource "vault").value}}' \ @@ -254,11 +259,12 @@ $ echo 'My voice is my passport. {{(datasource "vault").value}}' \ My voice is my passport. Verify me. ``` -You can also specify the secret path in the template by using a URL of `vault://` -(or `vault:///`, or `vault:`): +You can also specify the secret path in the template by omitting the path portion +of the URL: + ```console $ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \ - | gomplate -d vault=vault:// + | gomplate -d vault=vault:/// My voice is my passport. Verify me. ``` @@ -270,6 +276,15 @@ $ echo 'db_password={{(datasource "vault" "db/pass").value}}' \ db_password=prodsecret ``` +If you are unable to set the `VAULT_ADDR` environment variable, or need to +specify multiple Vault datasources connecting to different servers, you can set +the address as part of the URL: + +```console +$ gomplate -d v=vault://vaultserver.com/secret/foo -i '{{ (ds "v").value }}' +bar +``` + It is also possible to use dynamic secrets by using the write capability of the datasource. To use, add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then included within the JSON body of the request. diff --git a/libkv/consul.go b/libkv/consul.go index d4fb53c43..696a14b5c 100644 --- a/libkv/consul.go +++ b/libkv/consul.go @@ -25,7 +25,7 @@ func NewConsul(u *url.URL) *LibKV { if role := env.Getenv("CONSUL_VAULT_ROLE", ""); role != "" { mount := env.Getenv("CONSUL_VAULT_MOUNT", "consul") - client := vault.New() + client := vault.New(nil) client.Login() path := fmt.Sprintf("%s/creds/%s", mount, role) diff --git a/test/integration/Dockerfile b/test/integration/Dockerfile index 894a04a70..fd128ae1d 100644 --- a/test/integration/Dockerfile +++ b/test/integration/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:edge -ENV VAULT_VER 0.7.3 -ENV CONSUL_VER 0.9.0 +ENV VAULT_VER 0.9.3 +ENV CONSUL_VER 1.0.3 RUN apk add --no-cache \ curl \ bash \ @@ -23,9 +23,8 @@ COPY gomplate /bin/gomplate COPY mirror /bin/mirror COPY meta /bin/meta COPY aws /bin/aws -COPY *.sh /tests/ +# COPY *.sh /tests/ COPY *.bash /tests/ COPY *.bats /tests/ COPY *.db /test/integration/ - CMD ["/tests/test.sh"] diff --git a/test/integration/awssvc/main.go b/test/integration/awssvc/main.go index 0a3f8f056..7bed22884 100644 --- a/test/integration/awssvc/main.go +++ b/test/integration/awssvc/main.go @@ -19,7 +19,7 @@ func main() { flag.StringVar(&port, "p", "8082", "Port to listen to") flag.Parse() - l, err := net.Listen("tcp", ":"+port) + l, err := net.Listen("tcp", "127.0.0.1:"+port) if err != nil { log.Fatal(err) } diff --git a/test/integration/datasources_consul.bats b/test/integration/datasources_consul.bats index 8eb5da77a..752c5be1e 100644 --- a/test/integration/datasources_consul.bats +++ b/test/integration/datasources_consul.bats @@ -3,6 +3,7 @@ load helper function setup () { + export VAULT_ADDR=http://127.0.0.1:8200 start_consul 8501 export CONSUL_HTTP_ADDR=http://127.0.0.1:8501 } @@ -10,7 +11,7 @@ function setup () { function teardown () { export CONSUL_HTTP_ADDR=http://127.0.0.1:8501 consul kv delete foo - vault unmount consul + vault secrets disable consul stop_consul } diff --git a/test/integration/datasources_vault.bats b/test/integration/datasources_vault.bats index f58d0090f..0d45c17af 100644 --- a/test/integration/datasources_vault.bats +++ b/test/integration/datasources_vault.bats @@ -3,6 +3,9 @@ load helper function setup () { + export VAULT_ADDR=http://127.0.0.1:8200 + # start_vault + unset VAULT_TOKEN cat <& /dev/null path "*" { @@ -15,40 +18,53 @@ path "*" { } EOF tmpdir=$(mktemp -d) - cp ~/.vault-token ~/.vault-token.bak + cp ~/.vault-token ${tmpdir}/.vault-token.bak start_meta_svc start_aws_svc } function teardown () { - mv ~/.vault-token.bak ~/.vault-token + mv ${tmpdir}/.vault-token.bak ~/.vault-token stop_meta_svc stop_aws_svc rm -rf $tmpdir unset VAULT_TOKEN vault delete secret/foo - vault auth-disable userpass - vault auth-disable userpass2 - vault auth-disable approle - vault auth-disable approle2 - vault auth-disable app-id - vault auth-disable app-id2 - vault auth-disable aws - vault policy-delete writepol - vault policy-delete readpol - vault unmount ssh + vault auth disable userpass + vault auth disable userpass2 + vault auth disable approle + vault auth disable approle2 + vault auth disable app-id + vault auth disable app-id2 + vault auth disable aws + vault policy delete writepol + vault policy delete readpol + vault secrets disable ssh + + # stop_vault } @test "Testing token vault auth" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) + VAULT_TOKEN=$(vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}' [ "$status" -eq 0 ] [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]] } +@test "Testing token vault auth with addr in URL" { + vault write secret/foo value="$BATS_TEST_DESCRIPTION" + VAULT_TOKEN=$(vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) + addr=$VAULT_ADDR + unset VAULT_ADDR + VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault+${addr}/secret -i '{{(datasource "vault" "foo").value}}' + export VAULT_ADDR=$addr + [ "$status" -eq 0 ] + [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]] +} + @test "Testing failure with non-existant secret" { - VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) + VAULT_TOKEN=$(vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///secret -i '{{(datasource "vault" "bar").value}}' [ "$status" -eq 1 ] [[ "${output}" == *"No value found for [bar] from datasource 'vault'" ]] @@ -56,7 +72,7 @@ function teardown () { @test "Testing token vault auth using file" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token > $tmpdir/token + vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token > $tmpdir/token unset VAULT_TOKEN VAULT_TOKEN_FILE=$tmpdir/token gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}' [ "$status" -eq 0 ] @@ -65,7 +81,7 @@ function teardown () { @test "Testing userpass vault auth" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable userpass + vault auth enable userpass vault write auth/userpass/users/dave password=foo ttl=30s policies=readpol VAULT_AUTH_USERNAME=dave VAULT_AUTH_PASSWORD=foo gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}' [ "$status" -eq 0 ] @@ -74,7 +90,7 @@ function teardown () { @test "Testing userpass vault auth using files" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable userpass + vault auth enable userpass vault write auth/userpass/users/dave password=foo ttl=30s policies=readpol echo -n "dave" > $tmpdir/username echo -n "foo" > $tmpdir/password @@ -85,7 +101,7 @@ function teardown () { @test "Testing userpass vault auth with custom mount" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable -path=userpass2 userpass + vault auth enable -path=userpass2 userpass vault write auth/userpass2/users/dave password=foo ttl=30s policies=readpol VAULT_AUTH_USERPASS_MOUNT=userpass2 VAULT_AUTH_USERNAME=dave VAULT_AUTH_PASSWORD=foo gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}' [ "$status" -eq 0 ] @@ -94,7 +110,7 @@ function teardown () { @test "Testing approle vault auth" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable approle + vault auth enable approle vault write auth/approle/role/testrole secret_id_ttl=30s token_ttl=35s token_max_ttl=3m secret_id_num_uses=1 policies=readpol VAULT_ROLE_ID=$(vault read -field role_id auth/approle/role/testrole/role-id) VAULT_SECRET_ID=$(vault write -f -field=secret_id auth/approle/role/testrole/secret-id) @@ -105,7 +121,7 @@ function teardown () { @test "Testing approle vault auth with custom mount" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable -path=approle2 approle + vault auth enable -path=approle2 approle vault write auth/approle2/role/testrole secret_id_ttl=30s token_ttl=35s token_max_ttl=3m secret_id_num_uses=1 policies=readpol VAULT_ROLE_ID=$(vault read -field role_id auth/approle2/role/testrole/role-id) VAULT_SECRET_ID=$(vault write -f -field=secret_id auth/approle2/role/testrole/secret-id) @@ -116,7 +132,7 @@ function teardown () { @test "Testing app-id vault auth" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable app-id + vault auth enable app-id vault write auth/app-id/map/app-id/testappid value=readpol display_name=test_app_id vault write auth/app-id/map/user-id/testuserid value=testappid VAULT_APP_ID=testappid VAULT_USER_ID=testuserid gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}' @@ -126,7 +142,7 @@ function teardown () { @test "Testing app-id vault auth with custom mount" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable -path=app-id2 app-id + vault auth enable -path=app-id2 app-id vault write auth/app-id2/map/app-id/testappid value=readpol display_name=test_app_id vault write auth/app-id2/map/user-id/testuserid value=testappid @@ -138,7 +154,7 @@ function teardown () { @test "Testing ec2 vault auth" { vault write secret/foo value="$BATS_TEST_DESCRIPTION" - vault auth-enable aws + vault auth enable aws vault write auth/aws/config/client secret_key=secret access_key=access endpoint=http://127.0.0.1:8082/ec2 iam_endpoint=http://127.0.0.1:8082/iam sts_endpoint=http://127.0.0.1:8082/sts curl -o $tmpdir/certificate -s -f http://127.0.0.1:8081/certificate vault write auth/aws/config/certificate/testcert type=pkcs7 aws_public_cert=@$tmpdir/certificate @@ -153,7 +169,7 @@ function teardown () { @test "Testing vault auth with dynamic secret" { vault mount ssh vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8 - VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) + VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:/// -i '{{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").ip}}' [ "$status" -eq 0 ] [[ "${output}" == "10.1.2.3" ]] @@ -162,7 +178,7 @@ function teardown () { @test "Testing vault auth with dynamic secret using prefix" { vault mount ssh vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8 - VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) + VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///ssh/creds/test -i '{{(datasource "vault" "?ip=10.1.2.3&username=user").ip}}' [ "$status" -eq 0 ] [[ "${output}" == "10.1.2.3" ]] @@ -171,7 +187,7 @@ function teardown () { @test "Testing vault auth with dynamic secret using prefix and options in URL" { vault mount ssh vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8 - VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) + VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///ssh/creds/test?ip=10.1.2.3\&username=user -i '{{(datasource "vault").ip}}' [ "$status" -eq 0 ] [[ "${output}" == "10.1.2.3" ]] @@ -180,7 +196,7 @@ function teardown () { @test "Testing vault auth with dynamic secret using options in URL and path in template" { vault mount ssh vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8 - VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) + VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token) VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///?ip=10.1.2.3\&username=user -i '{{(datasource "vault" "ssh/creds/test").ip}}' [ "$status" -eq 0 ] [[ "${output}" == "10.1.2.3" ]] diff --git a/test/integration/helper.bash b/test/integration/helper.bash index dd406b09d..5b40ddf5c 100644 --- a/test/integration/helper.bash +++ b/test/integration/helper.bash @@ -28,6 +28,7 @@ function __gomplate_stdin () { function start_mirror_svc () { bin/mirror & + # wait_for_url http://127.0.0.1:8080/ } function stop_mirror_svc () { @@ -75,3 +76,29 @@ function stop_consul () { kill $(cat $PID_FILE) &>/dev/null rm /tmp/gomplate-test-consul.json } + +function start_vault () { + port=$1 + PID_FILE=/tmp/gomplate-test-vault.pid + export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666 + + # back up any existing token so it doesn't get overridden + if [ -f ~/.vault-token ]; then + cp ~/.vault-token ~/.vault-token.bak + fi + + vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null & + # vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} & + echo $! > $PID_FILE + wait_for_url http://127.0.0.1:$port/sys/health +} + +function stop_vault () { + PID_FILE=/tmp/gomplate-test-vault.pid + kill $(cat $PID_FILE) &>/dev/null + + # restore old token if it was backed up + if [ -f ~/.vault-token.bak ]; then + mv ~/.vault-token.bak ~/.vault-token + fi +} diff --git a/test/integration/metasvc/main.go b/test/integration/metasvc/main.go index 108b9e39d..8a298c4ec 100644 --- a/test/integration/metasvc/main.go +++ b/test/integration/metasvc/main.go @@ -41,7 +41,7 @@ func main() { flag.StringVar(&port, "p", "8081", "Port to listen to") flag.Parse() - l, err := net.Listen("tcp", ":"+port) + l, err := net.Listen("tcp", "127.0.0.1:"+port) if err != nil { log.Fatal(err) } diff --git a/test/integration/mirrorsvc/main.go b/test/integration/mirrorsvc/main.go index 343ab8ca1..52b58e5de 100644 --- a/test/integration/mirrorsvc/main.go +++ b/test/integration/mirrorsvc/main.go @@ -13,13 +13,13 @@ type Req struct { Headers http.Header `json:"headers"` } -var port string +var port int func main() { - flag.StringVar(&port, "p", "8080", "Port to listen to") + flag.IntVar(&port, "p", 8080, "Port to listen to") flag.Parse() - l, err := net.Listen("tcp", ":"+port) + l, err := net.ListenTCP("tcp", &net.TCPAddr{Port: port}) if err != nil { log.Fatal(err) } diff --git a/test/integration/test.sh b/test/integration/test.sh index 270c1cdac..18d7bc487 100755 --- a/test/integration/test.sh +++ b/test/integration/test.sh @@ -1,16 +1,16 @@ #!/bin/bash set -euo pipefail -# This is useful for killing vault after the script exits, but causes the CircleCI -# build to fail, so... ¯\_(ツ)_/¯ -# trap "exit" INT TERM -# trap "kill 0" EXIT +source $(dirname $0)/helper.bash + +function finish { + stop_vault +} +trap finish EXIT -# TODO: export these in a bats helper, as well as only launch vault in a vault helper export VAULT_ADDR=http://127.0.0.1:8200 export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666 -# fire up vault in dev mode for the vault tests -vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null & +start_vault 8200 bats $(dirname $0) diff --git a/vault/vault.go b/vault/vault.go index b07d2483f..57244691f 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "log" + "net/url" vaultapi "github.com/hashicorp/vault/api" ) @@ -17,7 +18,7 @@ type Vault struct { } // New - -func New() *Vault { +func New(u *url.URL) *Vault { vaultConfig := vaultapi.DefaultConfig() err := vaultConfig.ReadEnvironment() @@ -25,6 +26,8 @@ func New() *Vault { logFatal("Vault setup failed", err) } + setVaultURL(vaultConfig, u) + client, err := vaultapi.NewClient(vaultConfig) if err != nil { logFatal("Vault setup failed", err) @@ -33,6 +36,16 @@ func New() *Vault { return &Vault{client} } +func setVaultURL(c *vaultapi.Config, u *url.URL) { + if u != nil && u.Host != "" { + scheme := "https" + if u.Scheme == "vault+http" { + scheme = "http" + } + c.Address = scheme + "://" + u.Host + } +} + // Login - func (v *Vault) Login() { v.client.SetToken(v.GetToken()) diff --git a/vault/vault_test.go b/vault/vault_test.go index 9f4777a6b..0bcc3259f 100644 --- a/vault/vault_test.go +++ b/vault/vault_test.go @@ -1,11 +1,36 @@ package vault import ( + "net/url" + "os" "testing" "github.com/stretchr/testify/assert" ) +func TestNew(t *testing.T) { + v := New(nil) + assert.Equal(t, "https://127.0.0.1:8200", v.client.Address()) + + os.Setenv("VAULT_ADDR", "http://example.com:1234") + defer os.Unsetenv("VAULT_ADDR") + v = New(nil) + assert.Equal(t, "http://example.com:1234", v.client.Address()) + os.Unsetenv("VAULT_ADDR") + + u, _ := url.Parse("vault://vault.rocks:8200/secret/foo/bar") + v = New(u) + assert.Equal(t, "https://vault.rocks:8200", v.client.Address()) + + u, _ = url.Parse("vault+https://vault.rocks:8200/secret/foo/bar") + v = New(u) + assert.Equal(t, "https://vault.rocks:8200", v.client.Address()) + + u, _ = url.Parse("vault+http://vault.rocks:8200/secret/foo/bar") + v = New(u) + assert.Equal(t, "http://vault.rocks:8200", v.client.Address()) +} + func TestRead(t *testing.T) { server, v := MockServer(404, "Not Found") defer server.Close()