Allow vault address to be specified in the vault:// URL

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

Makefile

@@ -104,7 +104,7 @@ test-integration-docker: build-integration-image
 	docker run -it --rm gomplate-test
 
 test-integration: build build-mirror build-meta build-aws
-	@test/integration/test.sh
+	@bats test/integration/
 
 gen-changelog:
 	github_changelog_generator --no-filter-by-milestone --exclude-labels duplicate,question,invalid,wontfix,admin

data/datasource.go

@@ -53,6 +53,8 @@ func init() {
 	addSourceReader("file", readFile)
 	addSourceReader("stdin", readStdin)
 	addSourceReader("vault", readVault)
+	addSourceReader("vault+http", readVault)
+	addSourceReader("vault+https", readVault)
 	addSourceReader("consul", readConsul)
 	addSourceReader("consul+http", readConsul)
 	addSourceReader("consul+https", readConsul)
@@ -372,7 +374,7 @@ func readHTTP(source *Source, args ...string) ([]byte, error) {
 
 func readVault(source *Source, args ...string) ([]byte, error) {
 	if source.VC == nil {
-		source.VC = vault.New()
+		source.VC = vault.New(source.URL)
 		source.VC.Login()
 	}
 

docs/content/functions/data.md

@@ -223,8 +223,13 @@ aaa
 ### Usage with Vault data
 
 The special `vault://` URL scheme can be used to retrieve data from [Hashicorp
-Vault](https://vaultproject.io). To use this, you must put the Vault server's
-URL in the `$VAULT_ADDR` environment variable.
+Vault](https://vaultproject.io). To use this, you must either provide the Vault
+server's hostname and port in the URL, or put the Vault server's URL in the
+`$VAULT_ADDR` environment variable.
+
+The `vault+http://` URL scheme can be used to indicate that request must be sent
+over regular unencrypted HTTP, while `vault+https://` and `vault://` are equivalent,
+and indicate that requests must be sent over HTTPS.
 
 This table describes the currently-supported authentication mechanisms and how to use them, in order of precedence:
 
@@ -246,19 +251,20 @@ any `_FILE` variable and the secret file will be ignored.
 
 To use a Vault datasource with a single secret, just use a URL of
 `vault:///secret/mysecret`. Note the 3 `/`s - the host portion of the URL is left
-empty.
+empty in this example.
 
 ```console
 $ echo 'My voice is my passport. {{(datasource "vault").value}}' \
   | gomplate -d vault=vault:///secret/sneakers
 My voice is my passport. Verify me.
 ```
 
-You can also specify the secret path in the template by using a URL of `vault://`
-(or `vault:///`, or `vault:`):
+You can also specify the secret path in the template by omitting the path portion
+of the URL:
+
 ```console
 $ echo 'My voice is my passport. {{(datasource "vault" "secret/sneakers").value}}' \
-  | gomplate -d vault=vault://
+  | gomplate -d vault=vault:///
 My voice is my passport. Verify me.
 ```
 
@@ -270,6 +276,15 @@ $ echo 'db_password={{(datasource "vault" "db/pass").value}}' \
 db_password=prodsecret
 ```
 
+If you are unable to set the `VAULT_ADDR` environment variable, or need to
+specify multiple Vault datasources connecting to different servers, you can set
+the address as part of the URL:
+
+```console
+$ gomplate -d v=vault://vaultserver.com/secret/foo -i '{{ (ds "v").value }}'
+bar
+```
+
 It is also possible to use dynamic secrets by using the write capability of the datasource. To use,
 add a URL query to the optional path (i.e. `"key?name=value&name=value"`). These values are then
 included within the JSON body of the request.

libkv/consul.go

@@ -25,7 +25,7 @@ func NewConsul(u *url.URL) *LibKV {
 	if role := env.Getenv("CONSUL_VAULT_ROLE", ""); role != "" {
 		mount := env.Getenv("CONSUL_VAULT_MOUNT", "consul")
 
-		client := vault.New()
+		client := vault.New(nil)
 		client.Login()
 
 		path := fmt.Sprintf("%s/creds/%s", mount, role)

test/integration/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:edge
 
-ENV VAULT_VER 0.7.3
-ENV CONSUL_VER 0.9.0
+ENV VAULT_VER 0.9.3
+ENV CONSUL_VER 1.0.3
 RUN apk add --no-cache \
     curl \
     bash \
@@ -23,9 +23,9 @@ COPY gomplate /bin/gomplate
 COPY mirror /bin/mirror
 COPY meta /bin/meta
 COPY aws /bin/aws
-COPY *.sh /tests/
+# COPY *.sh /tests/
 COPY *.bash /tests/
 COPY *.bats /tests/
 COPY *.db /test/integration/
 
-CMD ["/tests/test.sh"]
+CMD ["bats", "/tests/"]

test/integration/awssvc/main.go

@@ -19,7 +19,7 @@ func main() {
 	flag.StringVar(&port, "p", "8082", "Port to listen to")
 	flag.Parse()
 
-	l, err := net.Listen("tcp", ":"+port)
+	l, err := net.Listen("tcp", "127.0.0.1:"+port)
 	if err != nil {
 		log.Fatal(err)
 	}

test/integration/datasources_consul.bats

@@ -3,6 +3,8 @@
 load helper
 
 function setup () {
+  export VAULT_ADDR=http://127.0.0.1:8200
+  start_vault
   start_consul 8501
   export CONSUL_HTTP_ADDR=http://127.0.0.1:8501
 }
@@ -12,6 +14,7 @@ function teardown () {
   consul kv delete foo
   vault unmount consul
   stop_consul
+  stop_vault
 }
 
 @test "Testing consul" {

test/integration/datasources_vault.bats

@@ -3,6 +3,9 @@
 load helper
 
 function setup () {
+  export VAULT_ADDR=http://127.0.0.1:8200
+  start_vault
+
   unset VAULT_TOKEN
   cat <<EOF | vault policy-write writepol - >& /dev/null
 path "*" {
@@ -15,13 +18,11 @@ path "*" {
 }
 EOF
   tmpdir=$(mktemp -d)
-  cp ~/.vault-token ~/.vault-token.bak
   start_meta_svc
   start_aws_svc
 }
 
 function teardown () {
-  mv ~/.vault-token.bak ~/.vault-token
   stop_meta_svc
   stop_aws_svc
   rm -rf $tmpdir
@@ -37,6 +38,8 @@ function teardown () {
   vault policy-delete writepol
   vault policy-delete readpol
   vault unmount ssh
+
+  stop_vault
 }
 
 @test "Testing token vault auth" {
@@ -47,6 +50,16 @@ function teardown () {
   [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]]
 }
 
+@test "Testing token vault auth with addr in URL" {
+  vault write secret/foo value="$BATS_TEST_DESCRIPTION"
+  VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
+  addr=$VAULT_ADDR
+  unset VAULT_ADDR
+  VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault+${addr}/secret -i '{{(datasource "vault" "foo").value}}'
+  [ "$status" -eq 0 ]
+  [[ "${output}" == "$BATS_TEST_DESCRIPTION" ]]
+}
+
 @test "Testing failure with non-existant secret" {
   VAULT_TOKEN=$(vault token-create -format=json -policy=readpol -use-limit=1 -ttl=1m | jq -j .auth.client_token)
   VAULT_TOKEN=$VAULT_TOKEN gomplate -d vault=vault:///secret -i '{{(datasource "vault" "bar").value}}'

test/integration/helper.bash

@@ -28,6 +28,7 @@ function __gomplate_stdin () {
 
 function start_mirror_svc () {
   bin/mirror &
+  wait_for_url http://127.0.0.1:8080/
 }
 
 function stop_mirror_svc () {
@@ -75,3 +76,31 @@ function stop_consul () {
   kill $(cat $PID_FILE) &>/dev/null
   rm /tmp/gomplate-test-consul.json
 }
+
+function start_vault () {
+  port=$1
+  if [ -z $port ]; then
+    port=8200
+  fi
+  PID_FILE=/tmp/gomplate-test-vault.pid
+  export VAULT_ROOT_TOKEN=00000000-1111-2222-3333-444455556666
+
+  # back up any existing token so it doesn't get overridden
+  if [ -f ~/.vault-token ]; then
+    cp ~/.vault-token ~/.vault-token.bak
+  fi
+
+  vault server -dev -dev-root-token-id=${VAULT_ROOT_TOKEN} -log-level=err >&/dev/null &
+  echo $! > $PID_FILE
+  wait_for_url http://127.0.0.1:$port/sys/health
+}
+
+function stop_vault () {
+  PID_FILE=/tmp/gomplate-test-vault.pid  
+  kill $(cat $PID_FILE) &>/dev/null
+
+  # restore old token if it was backed up
+  if [ -f ~/.vault-token.bak ]; then
+    mv ~/.vault-token.bak ~/.vault-token
+  fi
+}

test/integration/metasvc/main.go

@@ -41,7 +41,7 @@ func main() {
 	flag.StringVar(&port, "p", "8081", "Port to listen to")
 	flag.Parse()
 
-	l, err := net.Listen("tcp", ":"+port)
+	l, err := net.Listen("tcp", "127.0.0.1:"+port)
 	if err != nil {
 		log.Fatal(err)
 	}

test/integration/mirrorsvc/main.go

@@ -13,13 +13,13 @@ type Req struct {
 	Headers http.Header `json:"headers"`
 }
 
-var port string
+var port int
 
 func main() {
-	flag.StringVar(&port, "p", "8080", "Port to listen to")
+	flag.IntVar(&port, "p", 8080, "Port to listen to")
 	flag.Parse()
 
-	l, err := net.Listen("tcp", ":"+port)
+	l, err := net.ListenTCP("tcp", &net.TCPAddr{Port: port})
 	if err != nil {
 		log.Fatal(err)
 	}

vault/vault.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"log"
+	"net/url"
 
 	vaultapi "github.com/hashicorp/vault/api"
 )
@@ -17,14 +18,18 @@ type Vault struct {
 }
 
 // New -
-func New() *Vault {
+func New(u *url.URL) *Vault {
 	vaultConfig := vaultapi.DefaultConfig()
 
 	err := vaultConfig.ReadEnvironment()
 	if err != nil {
 		logFatal("Vault setup failed", err)
 	}
 
+	if vu := vaultURL(u); vu != "" {
+		vaultConfig.Address = vu
+	}
+
 	client, err := vaultapi.NewClient(vaultConfig)
 	if err != nil {
 		logFatal("Vault setup failed", err)
@@ -33,6 +38,17 @@ func New() *Vault {
 	return &Vault{client}
 }
 
+func vaultURL(u *url.URL) string {
+	if u != nil && u.Host != "" {
+		scheme := "https"
+		if u.Scheme == "vault+http" {
+			scheme = "http"
+		}
+		return scheme + "://" + u.Host
+	}
+	return ""
+}
+
 // Login -
 func (v *Vault) Login() {
 	v.client.SetToken(v.GetToken())

vault/vault_test.go

@@ -1,11 +1,36 @@
 package vault
 
 import (
+	"net/url"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
+func TestNew(t *testing.T) {
+	v := New(nil)
+	assert.Equal(t, "https://127.0.0.1:8200", v.client.Address())
+
+	os.Setenv("VAULT_ADDR", "http://example.com:1234")
+	defer os.Unsetenv("VAULT_ADDR")
+	v = New(nil)
+	assert.Equal(t, "http://example.com:1234", v.client.Address())
+	os.Unsetenv("VAULT_ADDR")
+
+	u, _ := url.Parse("vault://vault.rocks:8200/secret/foo/bar")
+	v = New(u)
+	assert.Equal(t, "https://vault.rocks:8200", v.client.Address())
+
+	u, _ = url.Parse("vault+https://vault.rocks:8200/secret/foo/bar")
+	v = New(u)
+	assert.Equal(t, "https://vault.rocks:8200", v.client.Address())
+
+	u, _ = url.Parse("vault+http://vault.rocks:8200/secret/foo/bar")
+	v = New(u)
+	assert.Equal(t, "http://vault.rocks:8200", v.client.Address())
+}
+
 func TestRead(t *testing.T) {
 	server, v := MockServer(404, "Not Found")
 	defer server.Close()