Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for multiple replicas with auto-tls #198

Merged
merged 16 commits into from
Dec 17, 2020
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions agent-inject/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
DefaultAgentCacheEnable = "false"
DefaultAgentCacheUseAutoAuthToken = "true"
DefaultAgentCacheListenerPort = "8200"
DefaultAgentUseLeaderElector = false
)

// Agent is the top level structure holding all the
Expand Down Expand Up @@ -123,8 +124,8 @@ type Agent struct {
// SetSecurityContext controls whether the injected containers have a
// SecurityContext set.
SetSecurityContext bool
// ExtraSecret is the Kubernetes secret to mount as a volume in the Vault agent container

// ExtraSecret is the Kubernetes secret to mount as a volume in the Vault agent container
// which can be referenced by the Agent config for secrets. Mounted at /vault/custom/
ExtraSecret string
}
Expand Down
36 changes: 35 additions & 1 deletion deploy/injector-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ metadata:
app.kubernetes.io/name: vault-injector
app.kubernetes.io/instance: vault
spec:
replicas: 1
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: vault-injector
Expand All @@ -20,6 +20,38 @@ spec:
spec:
serviceAccountName: "vault-injector"
containers:
- name: leader-elector
image: gcr.io/google_containers/leader-elector:0.4
args:
- --election=injector-leader
- --election-namespace=$(NAMESPACE)
- --http=0.0.0.0:4040
- --ttl=60s
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
livenessProbe:
httpGet:
path: /
port: 4040
scheme: HTTP
failureThreshold: 2
initialDelaySeconds: 1
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /
port: 4040
scheme: HTTP
failureThreshold: 2
initialDelaySeconds: 2
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 5
- name: sidecar-injector
image: "hashicorp/vault-k8s:0.6.0"
imagePullPolicy: IfNotPresent
Expand All @@ -42,6 +74,8 @@ spec:
value: vault-agent-injector-cfg
- name: AGENT_INJECT_TLS_AUTO_HOSTS
value: "vault-agent-injector-svc,vault-agent-injector-svc.$(NAMESPACE),vault-agent-injector-svc.$(NAMESPACE).svc"
- name: AGENT_INJECT_USE_LEADER_ELECTOR
value: "true"
args:
- agent-inject
- 2>&1
Expand Down
13 changes: 13 additions & 0 deletions deploy/injector-leader-extras.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# These are created here so they can be cleaned up easily. The endpoints
# especially, since if they're left around the leader won't expire for about a
# minute.
---
apiVersion: v1
kind: Endpoints
metadata:
name: vault-agent-injector-leader
---
apiVersion: v1
kind: Secret
metadata:
name: vault-injector-leader
33 changes: 33 additions & 0 deletions deploy/injector-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,36 @@ subjects:
- kind: ServiceAccount
name: vault-injector
namespace: vault
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: vault-injector-role
labels:
app.kubernetes.io/name: vault-injector
app.kubernetes.io/instance: vault
rules:
- apiGroups: [""]
resources: ["endpoints", "secrets"]
verbs:
- "create"
tomhjp marked this conversation as resolved.
Show resolved Hide resolved
- "get"
- "watch"
- "list"
- "update"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: vault-injector-rolebinding
labels:
app.kubernetes.io/name: vault-injector
app.kubernetes.io/instance: vault
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: vault-injector-role
subjects:
- kind: ServiceAccount
name: vault-injector
namespace: vault
1 change: 1 addition & 0 deletions deploy/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ resources:
- injector-mutating-webhook.yaml
- injector-rbac.yaml
- injector-service.yaml
- injector-leader-extras.yaml
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ go 1.12

require (
github.com/armon/go-radix v1.0.0 // indirect
github.com/evanphx/json-patch v4.9.0+incompatible // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/google/btree v1.0.0 // indirect
github.com/hashicorp/consul v1.5.0
github.com/hashicorp/go-hclog v0.9.2
Expand Down Expand Up @@ -31,6 +33,7 @@ require (
k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
k8s.io/klog v1.0.0 // indirect
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 // indirect
k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d // indirect
sigs.k8s.io/yaml v1.1.0 // indirect
)
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,13 @@ github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74 h1:2MIh
github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74/go.mod h1:UqXY1lYT/ERa4OEAywUqdok1T4RCRdArkhic1Opuavo=
github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
github.com/envoyproxy/go-control-plane v0.0.0-20180919002855-2137d9196328/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
Expand All @@ -93,6 +96,8 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
Expand Down Expand Up @@ -222,6 +227,7 @@ github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35n
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
Expand Down Expand Up @@ -297,8 +303,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk=
github.com/oklog/run v0.0.0-20180308005104-6934b124db28/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
Expand Down Expand Up @@ -479,6 +487,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
Expand All @@ -487,6 +496,7 @@ gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3M
gopkg.in/ory-am/dockertest.v3 v3.3.4/go.mod h1:s9mmoLkaGeAh97qygnNj4xWkiN7e1SKekYC6CovU+ek=
gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4=
gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down Expand Up @@ -515,6 +525,8 @@ k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7v
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d h1:1P0iBJsBzxRmR+dIFnM+Iu4aLxnoa7lBqozW/0uHbT8=
k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
Expand Down
72 changes: 72 additions & 0 deletions helper/cert/source_gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,19 @@ import (
"strings"
"sync"
"time"

"github.com/hashicorp/vault-k8s/leader"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
informerv1 "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
)

// Name of the k8s Secret used to share the caBundle between leader and
// followers
const certSecretName = "vault-injector-certs"
tomhjp marked this conversation as resolved.
Show resolved Hide resolved

// GenSource generates a self-signed CA and certificate pair.
//
// This generator is stateful. On the first run (last == nil to Certificate),
Expand All @@ -43,6 +54,11 @@ type GenSource struct {
caCert []byte
caCertTemplate *x509.Certificate
caSigner crypto.Signer

K8sClient kubernetes.Interface
Namespace string
SecretsCache informerv1.SecretInformer
LeaderElector *leader.LeaderElector
}

// Certificate implements source
Expand All @@ -51,6 +67,19 @@ func (s *GenSource) Certificate(ctx context.Context, last *Bundle) (Bundle, erro
defer s.mu.Unlock()
var result Bundle

if s.LeaderElector != nil {
leaderCheck, err := s.LeaderElector.IsLeader()
if err != nil {
return result, err
}
// For followers, run different function here that reads bundle from Secret,
// and returns that in the result. That will flow through the existing
// notify channel structure, testing if it's the same cert as last, etc.
if !leaderCheck {
Copy link
Contributor

@jasonodonnell jasonodonnell Dec 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we get a log saying whether this pod is the leader or follower? Also wondering if we should apply a label so we can query for the leader (not blocking, future enhancement probably).

Currently I can't tell which is which:

[~] kubectl logs vault-injector-57485f744-24jwd -c sidecar-injector
Using leader elector logic
2020-12-16T15:22:43.623Z [INFO]  handler: Starting handler..
Listening on ":8080"...
Updated certificate bundle received. Updating certs...

[~] kubectl logs vault-injector-57485f744-lts8w -c sidecar-injector
Using leader elector logic
2020-12-16T15:22:44.109Z [INFO]  handler: Starting handler..
Listening on ":8080"...
Updated certificate bundle received. Updating certs...

Copy link
Member Author

@tvoran tvoran Dec 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, good point, added more logging in ac4b4f4. I set the leader and follower log lines to debug since they are logged pretty frequently. Looks something like this:

❯ kubectl logs -n vault vault-agent-injector-c6d5c7689-6q4w7 -c sidecar-injector 
Using leader elector logic
2020-12-17T02:36:22.710Z [INFO]  handler: Starting handler..
Listening on ":8080"...
2020-12-17T02:36:22.719Z [INFO]  handler.auto-tls: Currently the leader
2020-12-17T02:36:22.726Z [INFO]  handler.auto-tls: Generated CA
2020-12-17T02:36:22.734Z [INFO]  handler.certwatcher: Updated certificate bundle received. Updating certs...
2020-12-17T02:36:27.711Z [INFO]  handler.auto-tls: Currently the leader

❯ kubectl logs -n vault vault-agent-injector-c6d5c7689-lmrjb -c sidecar-injector 
Using leader elector logic
2020-12-17T02:36:26.307Z [INFO]  handler: Starting handler..
Listening on ":8080"...
2020-12-17T02:36:26.314Z [DEBUG] handler.auto-tls: Currently a follower
2020-12-17T02:36:26.314Z [INFO]  handler.certwatcher: Updated certificate bundle received. Updating certs...
2020-12-17T02:36:31.308Z [DEBUG] handler.auto-tls: Currently a follower
...

return s.getBundleFromSecret()
}
}

// If we have no CA, generate it for the first time.
if len(s.caCert) == 0 {
if err := s.generateCA(); err != nil {
Expand Down Expand Up @@ -92,11 +121,54 @@ func (s *GenSource) Certificate(ctx context.Context, last *Bundle) (Bundle, erro
if err == nil {
result.Cert = []byte(cert)
result.Key = []byte(key)

if s.LeaderElector != nil {
if err := s.updateSecret(result); err != nil {
return result, fmt.Errorf("failed to update Secret: %s", err)
}
}
}

return result, err
}

func (s *GenSource) updateSecret(bundle Bundle) error {
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: certSecretName,
},
Data: map[string][]byte{
"ca": bundle.CACert,
"cert": bundle.Cert,
"key": bundle.Key,
},
}
// Attempt updating the Secret first, and if it doesn't exist, fallback to
// create
_, err := s.K8sClient.CoreV1().Secrets(s.Namespace).Update(secret)
if errors.IsNotFound(err) {
_, err = s.K8sClient.CoreV1().Secrets(s.Namespace).Create(secret)
}
if err != nil {
return err
}
return nil
}

func (s *GenSource) getBundleFromSecret() (Bundle, error) {
var bundle Bundle

secret, err := s.SecretsCache.Lister().Secrets(s.Namespace).Get(certSecretName)
if err != nil {
return bundle, fmt.Errorf("failed to get secret: %s", err)
}
bundle.CACert = secret.Data["ca"]
bundle.Cert = secret.Data["cert"]
bundle.Key = secret.Data["key"]

return bundle, nil
}

func (s *GenSource) expiry() time.Duration {
if s.Expiry > 0 {
return s.Expiry
Expand Down
Loading