Add file annotation (#158)

Adds new annotation `vault.hashicorp.com/agent-inject-file`. This sets
the filename and path in the secrets volume where a vault secret will
be written. The container mount path for the secrets volume may be
modified with the `secret-volume-path` annotation.

This allows for filenames that aren't limited in length as k8s
annotation keys, and for arbitrary directory structures in the secrets
volume.

Co-authored-by: Samuel Beaulieu <samuel@puppetlabs.com>

agent-inject/agent/agent.go

@@ -126,7 +126,9 @@ type Agent struct {
 }
 
 type Secret struct {
-	// Name of the secret used as the filename for the rendered secret file.
+	// Name of the secret used to identify other annotation directives, and used
+	// as the filename for the rendered secret file (unless FilePathAndName is
+	// specified).
 	Name string
 
 	// Path in Vault where the secret desired can be found.
@@ -135,11 +137,14 @@ type Secret struct {
 	// Template is the optional custom template to use when rendering the secret.
 	Template string
 
-	// Mount Path
+	// Mount Path for the volume holding the rendered secret file
 	MountPath string
 
 	// Command is the optional command to run after rendering the secret.
 	Command string
+
+	// FilePathAndName is the optional file path and name for the rendered secret file.
+	FilePathAndName string
 }
 
 type Vault struct {

agent-inject/agent/annotations.go

@@ -27,11 +27,21 @@ const (
 	// path in Vault where the secret is located.
 	AnnotationAgentInjectSecret = "vault.hashicorp.com/agent-inject-secret"
 
+	// AnnotationAgentInjectFile is the key of the annotation that contains the
+	// name (and optional path) of the file to create on disk. The name of the
+	// secret is the string after "vault.hashicorp.com/agent-inject-file-", and
+	// should map to the same unique value provided in
+	// "vault.hashicorp.com/agent-inject-secret-". The value is the filename and
+	// path in the secrets volume where the vault secret will be written. The
+	// container mount path of the secrets volume may be modified with the
+	// secret-volume-path annotation.
+	AnnotationAgentInjectFile = "vault.hashicorp.com/agent-inject-file"
+
 	// AnnotationAgentInjectTemplate is the key annotation that configures Vault
 	// Agent what template to use for rendering the secrets.  The name
 	// of the template is any unique string after "vault.hashicorp.com/agent-inject-template-",
 	// such as "vault.hashicorp.com/agent-inject-template-foobar".  This should map
-	// to the same unique value provided in ""vault.hashicorp.com/agent-inject-secret-".
+	// to the same unique value provided in "vault.hashicorp.com/agent-inject-secret-".
 	// If not provided, a default generic template is used.
 	AnnotationAgentInjectTemplate = "vault.hashicorp.com/agent-inject-template"
 
@@ -42,7 +52,7 @@ const (
 	// AnnotationAgentInjectCommand is the key annotation that configures Vault Agent
 	// to run a command after the secret is rendered. The name of the template is any
 	// unique string after "vault.hashicorp.com/agent-inject-command-". This should map
-	// to the same unique value provided in ""vault.hashicorp.com/agent-inject-secret-".
+	// to the same unique value provided in "vault.hashicorp.com/agent-inject-secret-".
 	// If not provided (the default), no command is executed.
 	AnnotationAgentInjectCommand = "vault.hashicorp.com/agent-inject-command"
 
@@ -318,11 +328,13 @@ func Init(pod *corev1.Pod, cfg AgentConfig) error {
 }
 
 // secrets parses annotations with the pattern "vault.hashicorp.com/agent-inject-secret-".
-// Everything following the final dash becomes the name of the secret,
-// and the value is the path in Vault.
+// Everything following the final dash becomes the name of the secret, and the
+// value is the path in Vault. This method also matches and returns the
+// Template, Command, and FilePathAndName settings from annotations associated
+// with a secret name.
 //
 // For example: "vault.hashicorp.com/agent-inject-secret-foobar: db/creds/foobar"
-// name: foobar, value: db/creds/foobar
+// Name: foobar, Path: db/creds/foobar
 func (a *Agent) secrets() []*Secret {
 	var secrets []*Secret
 
@@ -345,28 +357,30 @@ func (a *Agent) secrets() []*Secret {
 				continue
 			}
 
-			var template string
-			templateName := fmt.Sprintf("%s-%s", AnnotationAgentInjectTemplate, raw)
+			s := &Secret{Name: name, Path: path}
 
+			templateName := fmt.Sprintf("%s-%s", AnnotationAgentInjectTemplate, raw)
 			if val, ok := a.Annotations[templateName]; ok {
-				template = val
+				s.Template = val
 			}
 
-			mountPath := a.Annotations[AnnotationVaultSecretVolumePath]
+			s.MountPath = a.Annotations[AnnotationVaultSecretVolumePath]
 			mountPathAnnotationName := fmt.Sprintf("%s-%s", AnnotationVaultSecretVolumePath, raw)
-
 			if val, ok := a.Annotations[mountPathAnnotationName]; ok {
-				mountPath = val
+				s.MountPath = val
 			}
 
-			var command string
 			commandName := fmt.Sprintf("%s-%s", AnnotationAgentInjectCommand, raw)
-
 			if val, ok := a.Annotations[commandName]; ok {
-				command = val
+				s.Command = val
+			}
+
+			file := fmt.Sprintf("%s-%s", AnnotationAgentInjectFile, raw)
+			if val, ok := a.Annotations[file]; ok {
+				s.FilePathAndName = val
 			}
 
-			secrets = append(secrets, &Secret{Name: name, Path: path, Template: template, Command: command, MountPath: mountPath})
+			secrets = append(secrets, s)
 		}
 	}
 	return secrets

agent-inject/agent/annotations_test.go

@@ -179,7 +179,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOff(t *testing.T) {
 
 			}
 		} else if len(agent.Secrets) > 0 {
-			t.Error("Secrets length was greater than zero, it shouldn't have been")
+			t.Errorf("Secrets length was greater than zero, it shouldn't have been: %s", tt.key)
 		}
 	}
 }
@@ -235,6 +235,98 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOn(t *testing.T) {
 	}
 }
 
+func TestSecretLocationFileAnnotations(t *testing.T) {
+	tests := []struct {
+		name             string
+		annotations      map[string]string
+		expectedName     string
+		expectedFilename string
+		expectedLocation string
+	}{
+		{
+			"simple name",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foobar": "vault/test1",
+				"vault.hashicorp.com/agent-inject-file-foobar":   "foobar_simple_name",
+			},
+			"foobar",
+			"foobar_simple_name",
+			"vault/test1",
+		},
+		{
+			"absolute file path",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foobar": "vault/test1",
+				"vault.hashicorp.com/agent-inject-file-foobar":   "/some/path/foobar_simple_name",
+			},
+			"foobar",
+			"/some/path/foobar_simple_name",
+			"vault/test1",
+		},
+		{
+			"long file name",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foobar": "vault/test2",
+				"vault.hashicorp.com/agent-inject-file-foobar":   "this_is_very_long_and/would_fail_in_kubernetes/if_in_annotation",
+			},
+			"foobar",
+			"this_is_very_long_and/would_fail_in_kubernetes/if_in_annotation",
+			"vault/test2",
+		},
+		{
+			"file doesn't match secret annotation",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foobar":         "vault/test2",
+				"vault.hashicorp.com/agent-inject-file-notcorresponding": "this_is_very_long_and/would_fail_in_kubernetes/if_in_annotation",
+			},
+			"foobar",
+			"",
+			"vault/test2",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pod := testPod(tt.annotations)
+			var patches []*jsonpatch.JsonPatchOperation
+
+			agentConfig := AgentConfig{
+				"", "http://foobar:8200", "test", "test", true, "100", "1000",
+				DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext,
+			}
+			err := Init(pod, agentConfig)
+			if err != nil {
+				t.Errorf("got error, shouldn't have: %s", err)
+			}
+
+			agent, err := New(pod, patches)
+			if err != nil {
+				t.Errorf("got error, shouldn't have: %s", err)
+			}
+
+			if tt.expectedName != "" {
+				if len(agent.Secrets) == 0 {
+					t.Error("Secrets length was zero, it shouldn't have been")
+				}
+
+				if agent.Secrets[0].Name != tt.expectedName {
+					t.Errorf("expected name %s, got %s", tt.expectedName, agent.Secrets[0].Name)
+				}
+
+				if agent.Secrets[0].FilePathAndName != tt.expectedFilename {
+					t.Errorf("expected file %s, got %s", tt.expectedFilename, agent.Secrets[0].Name)
+				}
+
+				if agent.Secrets[0].Path != tt.expectedLocation {
+					t.Errorf("expected path %s, got %s", tt.expectedLocation, agent.Secrets[0].Path)
+				}
+			} else if len(agent.Secrets) > 0 {
+				t.Errorf("Secrets length was greater than zero, it shouldn't have been")
+			}
+		})
+	}
+}
+
 func TestSecretTemplateAnnotations(t *testing.T) {
 	tests := []struct {
 		annotations      map[string]string
@@ -275,7 +367,7 @@ func TestSecretTemplateAnnotations(t *testing.T) {
 			map[string]string{
 				"vault.hashicorp.com/agent-inject-secret-foobar2":  "test1",
 				"vault.hashicorp.com/agent-inject-TEMPLATE-foobar": "foobarTemplate",
-			}, "foobar2", "foobarTemplate",
+			}, "foobar2", "",
 		},
 	}
 
@@ -305,7 +397,7 @@ func TestSecretTemplateAnnotations(t *testing.T) {
 			t.Errorf("expected name %s, got %s", tt.expectedKey, agent.Secrets[0].Name)
 		}
 
-		if agent.Secrets[0].Name != tt.expectedKey {
+		if agent.Secrets[0].Template != tt.expectedTemplate {
 			t.Errorf("expected template %s, got %s", tt.expectedTemplate, agent.Secrets[0].Template)
 		}
 	}

agent-inject/agent/config.go

@@ -3,6 +3,7 @@ package agent
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 	"time"
 )
 
@@ -95,9 +96,14 @@ func (a *Agent) newTemplateConfigs() []*Template {
 			template = fmt.Sprintf(DefaultTemplate, secret.Path)
 		}
 
+		filePathAndName := fmt.Sprintf("%s/%s", secret.MountPath, secret.Name)
+		if secret.FilePathAndName != "" {
+			filePathAndName = filepath.Join(secret.MountPath, secret.FilePathAndName)
+		}
+
 		tmpl := &Template{
 			Contents:    template,
-			Destination: fmt.Sprintf("%s/%s", secret.MountPath, secret.Name),
+			Destination: filePathAndName,
 			LeftDelim:   "{{",
 			RightDelim:  "}}",
 			Command:     secret.Command,

agent-inject/agent/config_test.go

@@ -142,6 +142,96 @@ func TestNewConfig(t *testing.T) {
 	}
 }
 
+func TestFilePathAndName(t *testing.T) {
+
+	tests := []struct {
+		name        string
+		annotations map[string]string
+		destination string
+	}{
+		{
+			"just secret",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foo": "db/creds/foo",
+			},
+			secretVolumePath + "/foo",
+		},
+		{
+			"with relative file path",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foo": "db/creds/foo",
+				"vault.hashicorp.com/agent-inject-file-foo":   "nested/foofile",
+			},
+			secretVolumePath + "/nested/foofile",
+		},
+		{
+			"with absolute file path",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foo": "db/creds/foo",
+				"vault.hashicorp.com/agent-inject-file-foo":   "/special/volume/foofile",
+			},
+			secretVolumePath + "/special/volume/foofile",
+		},
+		{
+			"with global volume mount set, long file name",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foo": "db/creds/foo",
+				"vault.hashicorp.com/agent-inject-file-foo":   "foofile_name_is_very_very_very_long",
+				"vault.hashicorp.com/secret-volume-path":      "/new/mount/path",
+			},
+			"/new/mount/path/foofile_name_is_very_very_very_long",
+		},
+		{
+			"with global volume mount set, absolute file path",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foo": "db/creds/foo",
+				"vault.hashicorp.com/agent-inject-file-foo":   "/special/foofile",
+				"vault.hashicorp.com/secret-volume-path":      "/new/mount/path",
+			},
+			"/new/mount/path/special/foofile",
+		},
+		{
+			"with secret volume mount set, relative file path",
+			map[string]string{
+				"vault.hashicorp.com/agent-inject-secret-foo": "db/creds/foo",
+				"vault.hashicorp.com/agent-inject-file-foo":   "nested/foofile",
+				"vault.hashicorp.com/secret-volume-path-foo":  "/new/mount/path",
+			},
+			"/new/mount/path/nested/foofile",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pod := testPod(tt.annotations)
+			var patches []*jsonpatch.JsonPatchOperation
+
+			agentConfig := AgentConfig{
+				"foobar-image", "http://foobar:8200", "test", "test", true, "100", "1000",
+				DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext,
+			}
+			err := Init(pod, agentConfig)
+			if err != nil {
+				t.Errorf("got error initialising pod, shouldn't have: %s", err)
+			}
+
+			agent, err := New(pod, patches)
+			cfg, err := agent.newConfig(true)
+			if err != nil {
+				t.Errorf("got error creating Vault config, shouldn't have: %s", err)
+			}
+
+			config := &Config{}
+			if err := json.Unmarshal(cfg, config); err != nil {
+				t.Errorf("got error unmarshalling Vault config, shouldn't have: %s", err)
+			}
+			if config.Templates[0].Destination != tt.destination {
+				t.Errorf("wrong destination: %s != %s", config.Templates[0].Destination, tt.destination)
+			}
+		})
+	}
+}
+
 func TestConfigVaultAgentCacheNotEnabledByDefault(t *testing.T) {
 	annotations := map[string]string{}
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/hashicorp/serf v0.8.3 // indirect
 	github.com/hashicorp/vault/sdk v0.1.14-0.20191205220236-47cffd09f972
 	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/kr/text v0.2.0 // indirect
 	github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/miekg/dns v1.1.15 // indirect

go.sum

@@ -40,6 +40,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coredns/coredns v1.1.2/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -212,6 +213,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v0.0.0-20180523175426-90697d60dd84/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lyft/protoc-gen-validate v0.0.0-20180911180927-64fcb82c878e/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8=