feat: Support secret updates (#1)

* feat: use only the AWS credentials from the environment and not the CR

* feat: remove AWS credentials from the CRD and regenerate files

* feat: default logger to hunman readable timestamps

* feat: support updates for secrets

* feat correctly handle credstash secret version

* fix: silence generation commands in the makefile

.gitignore

@@ -2,6 +2,7 @@ node_modules/
 # Temporary Build Files
 build/_output
 build/_test
+bin/
 # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode
 ### Emacs ###
 # -*- mode: gitignore; -*-

Makefile

@@ -1,15 +1,29 @@
 # Get current directory
 DIR := ${CURDIR}
 
+.PHONY: setup
+setup:
+	@which ./bin/openapi-gen > /dev/null || go build -o ./bin/openapi-gen k8s.io/kube-openapi/cmd/openapi-gen
+
+.PHONY: generate
+generate: setup
+	@operator-sdk generate k8s
+	@operator-sdk generate crds
+	@./bin/openapi-gen --logtostderr=true \
+	    -o "" -i ./pkg/apis/credstash/v1alpha1 \
+	    -O zz_generated.openapi \
+	    -p ./pkg/apis/credstash/v1alpha1 \
+	    -h ./hack/boilerplate.go.txt -r "-"
+
 .PHONY: semantic-release
 semantic-release:
-	npm ci
-	npx semantic-release
+	@npm ci
+	@npx semantic-release
 
 .PHONY: semantic-release-dry-run
 semantic-release-dry-run:
-	npm ci
-	npx semantic-release -d
+	@npm ci
+	@npx semantic-release -d
 
 package-lock.json: package.json
-	npm install
+	@npm install

cmd/manager/main.go

@@ -70,9 +70,13 @@ func main() {
 	// implementing the logr.Logger interface. This logger will
 	// be propagated through the whole operator, generating
 	// uniform and structured logs.
-	logf.SetLogger(zap.Logger())
 
-	log.Info(fmt.Sprintf("hahja: %s", flags.SelectorLabelValue))
+	//Set the default logging time encoding to iso8601 unless otherwise specified
+	timeEncodingFlagValue := zap.FlagSet().Lookup("zap-time-encoding").Value.String()
+	if timeEncodingFlagValue == "" {
+		_ = zap.FlagSet().Set("zap-time-encoding", "iso8601")
+	}
+	logf.SetLogger(zap.Logger())
 
 	printVersion()
 

deploy/crds/credstash.ouzi.tech_credstashsecrets_crd.yaml

@@ -31,45 +31,6 @@ spec:
         spec:
           description: CredstashSecretSpec defines the desired state of CredstashSecret
           properties:
-            awsCredentials:
-              properties:
-                awsAccessKeyId:
-                  description: The secret in the controller namespace that contains
-                    the AWSAccessKeyId for authentication.
-                  properties:
-                    key:
-                      description: The key of the secret to select from.  Must be
-                        a valid secret key.
-                      type: string
-                    name:
-                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                        TODO: Add other useful fields. apiVersion, kind, uid?'
-                      type: string
-                    optional:
-                      description: Specify whether the Secret or its key must be defined
-                      type: boolean
-                  required:
-                  - key
-                  type: object
-                awsSecretAccessKey:
-                  description: The secret in the controller monitor namespace that
-                    contains the AWSSecretAccessKey for authentication.
-                  properties:
-                    key:
-                      description: The key of the secret to select from.  Must be
-                        a valid secret key.
-                      type: string
-                    name:
-                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                        TODO: Add other useful fields. apiVersion, kind, uid?'
-                      type: string
-                    optional:
-                      description: Specify whether the Secret or its key must be defined
-                      type: boolean
-                  required:
-                  - key
-                  type: object
-              type: object
             secrets:
               items:
                 properties:

deploy/crds/credstash.ouzi.tech_v1alpha1_credstashsecret_cr.yaml

@@ -4,21 +4,6 @@ metadata:
   name: example-credstashsecret
   namespace: georgi
 spec:
-  awsConfig:
-    region: us-west-2
-    credentials:
-      awsAccessKeyId:
-        name: aws-creds
-        key: access-key-id
-      awsSecretAccessKey:
-        name: aws-creds
-        key: secret-access-key
   secrets:
-    - key: status-dashboard-db-user
-      table: credential-store
-    - key: mocade-events-producer-pass-test
-      table: credential-store
-    - key: status-dashboard-admin-user
-      table: credential-store
-    - key: fourleaf-bot-github-username
-      table: credential-store
+    - key: key1
+      table: credential-store

pkg/apis/credstash/v1alpha1/credstashsecret_types.go

@@ -1,27 +1,9 @@
 package v1alpha1
 
 import (
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
-type AWSCredentials struct {
-	// The secret in the controller namespace that contains the AWSAccessKeyId
-	// for authentication.
-	AWSAccessKeyId v1.SecretKeySelector `json:"awsAccessKeyId,omitempty"`
-	// The secret in the controller monitor namespace that contains the AWSSecretAccessKey
-	// for authentication.
-	AWSSecretAccessKey v1.SecretKeySelector `json:"awsSecretAccessKey,omitempty"`
-}
-
-type AWSConfig struct {
-	Region string `json:"region,omitempty"`
-	Credentials AWSCredentials `json:"credentials,omitempty"`
-}
-
 type CredstashSecretDef struct {
 	Key string `json:"key,omitempty"`
 	Table string `json:"table,omitempty"`
@@ -31,7 +13,6 @@ type CredstashSecretDef struct {
 // CredstashSecretSpec defines the desired state of CredstashSecret
 type CredstashSecretSpec struct {
 	Secrets []CredstashSecretDef `json:"secrets,omitempty"`
-	AWSConfig AWSConfig `json:"awsConfig,omitempty"`
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
 	// Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html

pkg/aws/credentials.go

@@ -39,4 +39,17 @@ func GetAwsSession(region string, awsAccessKeyId string, awsSecretAccessKey stri
 
 	return sess, nil
 
+}
+
+// Gets the aws session to use for looking up credstash secrets falling back to the environment config
+func GetAwsSessionFromEnv() (*session.Session, error){
+	sess, err := session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return sess, nil
 }

pkg/controller/credstashsecret/credstashsecret_controller.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ouzi-dev/credstash-operator/pkg/aws"
 	"github.com/ouzi-dev/credstash-operator/pkg/credstash"
 	"github.com/ouzi-dev/credstash-operator/pkg/flags"
+	"reflect"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
@@ -40,7 +41,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
-const LabelNameForSelector = "controllerInstance"
+const LabelNameForSelector = "operatorInstance"
 
 var log = logf.Log.WithName("controller_credstashsecret")
 
@@ -123,8 +124,11 @@ func (r *ReconcileCredstashSecret) Reconcile(request reconcile.Request) (reconci
 	}
 
 	// Define a new Secret object
-	// TODO handle error
-	secret, _ := r.newSecretForCR(instance)
+	secret, err := r.secretForCR(instance)
+	if err != nil {
+		reqLogger.Error(err, "Failed fetching secret from credstash")
+		return reconcile.Result{}, err
+	}
 
 	// Set CredstashSecret instance as the owner and controller
 	if err := controllerutil.SetControllerReference(instance, secret, r.scheme); err != nil {
@@ -134,62 +138,62 @@ func (r *ReconcileCredstashSecret) Reconcile(request reconcile.Request) (reconci
 	// Check if this Secret already exists
 	found := &corev1.Secret{}
 	err = r.client.Get(context.TODO(), types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}, found)
+
+	// Create new secret if it doesn't exist
 	if err != nil && errors.IsNotFound(err) {
 		reqLogger.Info("Creating a new Secret", "Secret.Namespace", secret.Namespace, "Secret.Name", secret.Name)
 		err = r.client.Create(context.TODO(), secret)
 		if err != nil {
 			return reconcile.Result{}, err
 		}
 
-		// Pod created successfully - don't requeue
+		// Secret created successfully - don't requeue
 		return reconcile.Result{}, nil
 	} else if err != nil {
 		return reconcile.Result{}, err
 	}
 
-	// Pod already exists - don't requeue
-	reqLogger.Info("Skip reconcile: Secret already exists", "Secret.Namespace", found.Namespace, "Secret.Name", found.Name)
-	return reconcile.Result{}, nil
-}
-
-// newSecretForCR returns a busybox pod with the same name/namespace as the cr
-func (r *ReconcileCredstashSecret) newSecretForCR(cr *credstashv1alpha1.CredstashSecret) (*corev1.Secret, error) {
-	//TODO extract all this secret logic to its own method
-	//TODO allow support for default controller level credentials as a catch-all
-	awsAccessKeyIdSecret := &corev1.Secret{}
-	err := r.client.Get(context.TODO(), types.NamespacedName{Name: cr.Spec.AWSConfig.Credentials.AWSAccessKeyId.Name, Namespace: cr.Namespace}, awsAccessKeyIdSecret)
-	if err != nil {
-		return nil, err
+	// Secret is out of date with credstash data
+	if !reflect.DeepEqual(secret.Data, found.Data) {
+		reqLogger.Info("Updating Secret because contents have changed", "Secret.Namespace", secret.Namespace, "Secret.Name", secret.Name)
+		err = r.client.Update(context.TODO(), secret)
+		if err != nil {
+			return reconcile.Result{}, err
+		} else {
+			return reconcile.Result{}, nil
+		}
 	}
 
-	awsAccessKey := string(awsAccessKeyIdSecret.Data[cr.Spec.AWSConfig.Credentials.AWSAccessKeyId.Key])
+	// Secret already exists - don't requeue
+	reqLogger.Info("Skip reconcile: Secret already exists and is up to date", "Secret.Namespace", found.Namespace, "Secret.Name", found.Name)
+	return reconcile.Result{}, nil
+}
 
-	awsSecretAccessKeySecret := &corev1.Secret{}
-	err = r.client.Get(context.TODO(), types.NamespacedName{Name: cr.Spec.AWSConfig.Credentials.AWSSecretAccessKey.Name, Namespace: cr.Namespace}, awsSecretAccessKeySecret)
+// secretForCR returns a secret the same name/namespace as the cr
+func (r *ReconcileCredstashSecret) secretForCR(cr *credstashv1alpha1.CredstashSecret) (*corev1.Secret, error) {
+	awsSession, err := aws.GetAwsSessionFromEnv()
 	if err != nil {
 		return nil, err
 	}
 
-	awsSecretAccessKey := string(awsAccessKeyIdSecret.Data[cr.Spec.AWSConfig.Credentials.AWSSecretAccessKey.Key])
-
-	awsSession, err := aws.GetAwsSession(cr.Spec.AWSConfig.Region, awsAccessKey, awsSecretAccessKey)
 	credstashSecretGetter := credstash.NewHelper(awsSession)
-
-	credstashSecretsValueMap := credstashSecretGetter.GetCredstashSecretsForCredstashSecretDefs(cr.Spec.Secrets)
-
+	credstashSecretsValueMap, err := credstashSecretGetter.GetCredstashSecretsForCredstashSecretDefs(cr.Spec.Secrets)
 	if err != nil {
 		return nil, err
 	}
 
-	return &corev1.Secret{
+	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      cr.Name,
 			Namespace: cr.Namespace,
 		},
-		StringData: credstashSecretsValueMap,
-	}, nil
+		Data: credstashSecretsValueMap,
+	}
+
+	return secret, nil
 }
 
+// setupPredicateFuncs makes sure that we only watch resources that match the correct label selector that we want
 func setupPredicateFuncs() predicate.Funcs {
 	return predicate.Funcs{
 		CreateFunc: func(e event.CreateEvent) bool {