Allow vault address to be specified in the vault:// URL

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

data/datasource.go

@@ -53,6 +53,8 @@ func init() {
 	addSourceReader("file", readFile)
 	addSourceReader("stdin", readStdin)
 	addSourceReader("vault", readVault)
+	addSourceReader("vault+http", readVault)
+	addSourceReader("vault+https", readVault)
 	addSourceReader("consul", readConsul)
 	addSourceReader("consul+http", readConsul)
 	addSourceReader("consul+https", readConsul)
@@ -372,7 +374,7 @@ func readHTTP(source *Source, args ...string) ([]byte, error) {
 
 func readVault(source *Source, args ...string) ([]byte, error) {
 	if source.VC == nil {
-		source.VC = vault.New()
+		source.VC = vault.New(source.URL)
 		source.VC.Login()
 	}
 

docs/content/functions/data.md

@@ -223,8 +223,13 @@ aaa
 ### Usage with Vault data
 
 The special `vault://` URL scheme can be used to retrieve data from [Hashicorp
-Vault](https://vaultproject.io). To use this, you must put the Vault server's
-URL in the `$VAULT_ADDR` environment variable.
+Vault](https://vaultproject.io). To use this, you must either provide the Vault
+server's hostname and port in the URL, or put the Vault server's URL in the
+`$VAULT_ADDR` environment variable.
+
+The `vault+http://` URL scheme can be used to indicate that request must be sent
+over regular unencrypted HTTP, while `vault+https://` and `vault://` are equivalent,
+and indicate that requests must be sent over HTTPS.
 
 This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence:
 
@@ -246,19 +251,20 @@ any `_FILE` variable and the secret file will be ignored.
 
 To use a Vault datasource with a single secret, just use a URL of
 `vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left
-empty.
+empty in this example.
 
 ```console
 $ echo 'My voice is my passport. {{(datasource "vault").value}}' \
   | gomplate -d vault=vault:///secret/sneakers
 My voice is my passport. Verify me.
 ```
 
-You can also specify the secret path in the template by using a URL of `vault://`
-(or `vault:///`, or `vault:`):
+You can also specify the secret path in the template by omitting the path portion
+of the URL:
+
 ```console
 $ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \
-  | gomplate -d vault=vault://
+  | gomplate -d vault=vault:///
 My voice is my passport. Verify me.
 ```
 
@@ -270,6 +276,15 @@ $ echo 'db_password={{(datasource "vault" "db/pass").value}}' \
 db_password=prodsecret
 ```
 
+If you are unable to set the `VAULT_ADDR` environment variable, or need to
+specify multiple Vault datasources connecting to different servers, you can set
+the address as part of the URL:
+
+```console
+$ gomplate -d v=vault://vaultserver.com/secret/foo -i '{{ (ds "v").value }}'
+bar
+```
+
 It is also possible to use dynamic secrets by using the write capability of the datasource. To use,
 add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then
 included within the JSON body of the request.

libkv/consul.go

@@ -25,7 +25,7 @@ func NewConsul(u *url.URL) *LibKV {
 	if role := env.Getenv("CONSUL_VAULT_ROLE", ""); role != "" {
 		mount := env.Getenv("CONSUL_VAULT_MOUNT", "consul")
 
-		client := vault.New()
+		client := vault.New(nil)
 		client.Login()
 
 		path := fmt.Sprintf("%s/creds/%s", mount, role)

test/integration/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:edge
 
-ENV VAULT_VER 0.7.3
-ENV CONSUL_VER 0.9.0
+ENV VAULT_VER 0.9.3
+ENV CONSUL_VER 1.0.3
 RUN apk add --no-cache \
     curl \
     bash \

test/integration/awssvc/main.go

@@ -19,7 +19,7 @@ func main() {
 	flag.StringVar(&port, "p", "8082", "Port to listen to")
 	flag.Parse()
 
-	l, err := net.Listen("tcp", ":"+port)
+	l, err := net.Listen("tcp", "127.0.0.1:"+port)
 	if err != nil {
 		log.Fatal(err)
 	}

test/integration/datasources_consul.bats

@@ -3,14 +3,15 @@
 load helper
 
 function setup () {
+  export VAULT_ADDR=http://127.0.0.1:8200
   start_consul 8501
   export CONSUL_HTTP_ADDR=http://127.0.0.1:8501
 }
 
 function teardown () {
   export CONSUL_HTTP_ADDR=http://127.0.0.1:8501
   consul kv delete foo
-  vault unmount consul
+  vault secrets disable consul
   stop_consul
 }
 

test/integration/datasources_vault.bats

@@ -15,48 +15,59 @@ path "*" {
 }
 EOF
   tmpdir=$(mktemp -d)
-  cp ~/.vault-token ~/.vault-token.bak
+  cp ~/.vault-token ${tmpdir}/.vault-token.bak
   start_meta_svc
   start_aws_svc
 }
 
 function teardown () {
-  mv ~/.vault-token.bak ~/.vault-token
+  mv ${tmpdir}/.vault-token.bak ~/.vault-token
   stop_meta_svc
   stop_aws_svc
   rm -rf $tmpdir
   unset VAULT_TOKEN
   vault delete secret/foo
-  vault auth-disable userpass
-  vault auth-disable userpass2
-  vault auth-disable approle
-  vault auth-disable approle2
-  vault auth-disable app-id
-  vault auth-disable app-id2
-  vault auth-disable aws
-  vault policy-delete writepol
-  vault policy-delete readpol
-  vault unmount ssh
+  vault auth disable userpass
+  vault auth disable userpass2
+  vault auth disable approle
+  vault auth disable approle2
+  vault auth disable app-id
+  vault auth disable app-id2
+  vault auth disable aws
+  vault policy delete writepol
+  vault policy delete readpol
+  vault secrets disable ssh
 }
 
 @test "Testing token vault auth" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
+  VAULT_TOKEN=$(vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}'
   [ "$status" -eq 0 ]
   [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]]
 }
 
+@test "Testing token vault auth with addr in URL" {
+  vault write secret/foo value="$BATS_TEST_DESCRIPTION"
+  VAULT_TOKEN=$(vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
+  addr=$VAULT_ADDR
+  unset VAULT_ADDR
+  VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault+${addr}/secret -i '{{(datasource "vault" "foo").value}}'
+  export VAULT_ADDR=$addr
+  [ "$status" -eq 0 ]
+  [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]]
+}
+
 @test "Testing failure with non-existant secret" {
-  VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
+  VAULT_TOKEN=$(vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///secret -i '{{(datasource "vault" "bar").value}}'
   [ "$status" -eq 1 ]
   [[ "${output}" == *"No value found for [bar] from datasource 'vault'" ]]
 }
 
 @test "Testing token vault auth using file" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token > $tmpdir/token
+  vault token create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token > $tmpdir/token
   unset VAULT_TOKEN
   VAULT_TOKEN_FILE=$tmpdir/token gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}'
   [ "$status" -eq 0 ]
@@ -65,7 +76,7 @@ function teardown () {
 
 @test "Testing userpass vault auth" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable userpass
+  vault auth enable userpass
   vault write auth/userpass/users/dave password=foo ttl=30s policies=readpol
   VAULT_AUTH_USERNAME=dave VAULT_AUTH_PASSWORD=foo gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}'
   [ "$status" -eq 0 ]
@@ -74,7 +85,7 @@ function teardown () {
 
 @test "Testing userpass vault auth using files" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable userpass
+  vault auth enable userpass
   vault write auth/userpass/users/dave password=foo ttl=30s policies=readpol
   echo -n "dave" > $tmpdir/username
   echo -n "foo" > $tmpdir/password
@@ -85,7 +96,7 @@ function teardown () {
 
 @test "Testing userpass vault auth with custom mount" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable -path=userpass2 userpass
+  vault auth enable -path=userpass2 userpass
   vault write auth/userpass2/users/dave password=foo ttl=30s policies=readpol
   VAULT_AUTH_USERPASS_MOUNT=userpass2 VAULT_AUTH_USERNAME=dave VAULT_AUTH_PASSWORD=foo gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}'
   [ "$status" -eq 0 ]
@@ -94,7 +105,7 @@ function teardown () {
 
 @test "Testing approle vault auth" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable approle
+  vault auth enable approle
   vault write auth/approle/role/testrole secret_id_ttl=30s token_ttl=35s token_max_ttl=3m secret_id_num_uses=1 policies=readpol
   VAULT_ROLE_ID=$(vault read -field role_id auth/approle/role/testrole/role-id)
   VAULT_SECRET_ID=$(vault write -f -field=secret_id auth/approle/role/testrole/secret-id)
@@ -105,7 +116,7 @@ function teardown () {
 
 @test "Testing approle vault auth with custom mount" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable -path=approle2 approle
+  vault auth enable -path=approle2 approle
   vault write auth/approle2/role/testrole secret_id_ttl=30s token_ttl=35s token_max_ttl=3m secret_id_num_uses=1 policies=readpol
   VAULT_ROLE_ID=$(vault read -field role_id auth/approle2/role/testrole/role-id)
   VAULT_SECRET_ID=$(vault write -f -field=secret_id auth/approle2/role/testrole/secret-id)
@@ -116,7 +127,7 @@ function teardown () {
 
 @test "Testing app-id vault auth" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable app-id
+  vault auth enable app-id
   vault write auth/app-id/map/app-id/testappid value=readpol display_name=test_app_id
   vault write auth/app-id/map/user-id/testuserid value=testappid
   VAULT_APP_ID=testappid VAULT_USER_ID=testuserid gomplate -d vault=vault:///secret -i '{{(datasource "vault" "foo").value}}'
@@ -126,7 +137,7 @@ function teardown () {
 
 @test "Testing app-id vault auth with custom mount" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable -path=app-id2 app-id
+  vault auth enable -path=app-id2 app-id
 
   vault write auth/app-id2/map/app-id/testappid value=readpol display_name=test_app_id
   vault write auth/app-id2/map/user-id/testuserid value=testappid
@@ -138,7 +149,7 @@ function teardown () {
 
 @test "Testing ec2 vault auth" {
   vault write secret/foo value="$BATS_TEST_DESCRIPTION"
-  vault auth-enable aws
+  vault auth enable aws
   vault write auth/aws/config/client secret_key=secret access_key=access endpoint=http://127.0.0.1:8082/ec2 iam_endpoint=http://127.0.0.1:8082/iam sts_endpoint=http://127.0.0.1:8082/sts
   curl -o $tmpdir/certificate -s -f http://127.0.0.1:8081/certificate
   vault write auth/aws/config/certificate/testcert type=pkcs7 aws_public_cert=@$tmpdir/certificate
@@ -153,7 +164,7 @@ function teardown () {
 @test "Testing vault auth with dynamic secret" {
   vault mount ssh
   vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8
-  VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
+  VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:/// -i '{{(datasource "vault" "ssh/creds/test?ip=10.1.2.3&username=user").ip}}'
   [ "$status" -eq 0 ]
   [[ "${output}" == "10.1.2.3" ]]
@@ -162,7 +173,7 @@ function teardown () {
 @test "Testing vault auth with dynamic secret using prefix" {
   vault mount ssh
   vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8
-  VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
+  VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///ssh/creds/test -i '{{(datasource "vault" "?ip=10.1.2.3&username=user").ip}}'
   [ "$status" -eq 0 ]
   [[ "${output}" == "10.1.2.3" ]]
@@ -171,7 +182,7 @@ function teardown () {
 @test "Testing vault auth with dynamic secret using prefix and options in URL" {
   vault mount ssh
   vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8
-  VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
+  VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///ssh/creds/test?ip=10.1.2.3\&username=user -i '{{(datasource "vault").ip}}'
   [ "$status" -eq 0 ]
   [[ "${output}" == "10.1.2.3" ]]
@@ -180,7 +191,7 @@ function teardown () {
 @test "Testing vault auth with dynamic secret using options in URL and path in template" {
   vault mount ssh
   vault write ssh/roles/test key_type=otp default_user=user cidr_list=10.0.0.0/8
-  VAULT_TOKEN=$(vault token-create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
+  VAULT_TOKEN=$(vault token create -format=json -policy=writepol -use-limit=2 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///?ip=10.1.2.3\&username=user -i '{{(datasource "vault" "ssh/creds/test").ip}}'
   [ "$status" -eq 0 ]
   [[ "${output}" == "10.1.2.3" ]]

test/integration/helper.bash

@@ -75,3 +75,28 @@ function stop_consul () {
   kill $(cat $PID_FILE) &>/dev/null
   rm /tmp/gomplate-test-consul.json
 }
+
+function start_vault () {
+  port=$1
+  PID_FILE=/tmp/gomplate-test-vault.pid
+  export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666
+
+  # back up any existing token so it doesn't get overridden
+  if [ -f ~/.vault-token ]; then
+    cp ~/.vault-token ~/.vault-token.bak
+  fi
+
+  vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null &
+  echo $! > $PID_FILE
+  wait_for_url http://127.0.0.1:$port/sys/health
+}
+
+function stop_vault () {
+  PID_FILE=/tmp/gomplate-test-vault.pid  
+  kill $(cat $PID_FILE) &>/dev/null
+
+  # restore old token if it was backed up
+  if [ -f ~/.vault-token.bak ]; then
+    mv ~/.vault-token.bak ~/.vault-token
+  fi
+}

test/integration/metasvc/main.go

@@ -41,7 +41,7 @@ func main() {
 	flag.StringVar(&port, "p", "8081", "Port to listen to")
 	flag.Parse()
 
-	l, err := net.Listen("tcp", ":"+port)
+	l, err := net.Listen("tcp", "127.0.0.1:"+port)
 	if err != nil {
 		log.Fatal(err)
 	}

test/integration/mirrorsvc/main.go

@@ -13,13 +13,13 @@ type Req struct {
 	Headers http.Header `json:"headers"`
 }
 
-var port string
+var port int
 
 func main() {
-	flag.StringVar(&port, "p", "8080", "Port to listen to")
+	flag.IntVar(&port, "p", 8080, "Port to listen to")
 	flag.Parse()
 
-	l, err := net.Listen("tcp", ":"+port)
+	l, err := net.ListenTCP("tcp", &net.TCPAddr{Port: port})
 	if err != nil {
 		log.Fatal(err)
 	}

test/integration/test.sh

@@ -1,16 +1,16 @@
 #!/bin/bash
 set -euo pipefail
 
-# This is useful for killing vault after the script exits, but causes the CircleCI
-# build to fail, so... ¯\_(ツ)_/¯
-# trap "exit" INT TERM
-# trap "kill 0" EXIT
+source $(dirname $0)/helper.bash
+
+function finish {
+    stop_vault
+}
+trap finish EXIT
 
-# TODO: export these in a bats helper, as well as only launch vault in a vault helper
 export VAULT_ADDR=http://127.0.0.1:8200
 export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666
 
-# fire up vault in dev mode for the vault tests
-vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null &
+start_vault 8200
 
 bats $(dirname $0)