feat(kubernetes): subset-diff (grafana#11)

So far, diffing was offloaded to kubectl. While this produces very nice results,
it is only possible for kubernetes version 1.13+.

Nevertheless, there are older cluster versions around, so these need to be
supported as well. subset-diff addresses those cases in the hopefully best way
possible.

To reduce field bloat, it only diffes those fields, that are present in the
local config. Kubernetes adds dynamic fields on the fly which we cannot know
about, so this is required.

Note: You WILL NOT see removed fields in the diff output. Upgrade your cluster
version to 1.13+ and use native diffing.

Author: tombrk

cmd/tk/workflow.go

@@ -6,6 +6,8 @@ import (
 	"os"
 
 	"github.com/alecthomas/chroma/quick"
+	"github.com/posener/complete"
+	"github.com/sh0rez/tanka/pkg/cmp"
 	"github.com/spf13/cobra"
 	"golang.org/x/crypto/ssh/terminal"
 )
@@ -38,12 +40,16 @@ func applyCmd() *cobra.Command {
 }
 
 func diffCmd() *cobra.Command {
+	// completion
+	cmp.Handlers.Add("diffStrategy", complete.PredictSet("native", "subset"))
+
 	cmd := &cobra.Command{
 		Use:   "diff [directory]",
 		Short: "differences between the configuration and the cluster",
 		Args:  cobra.ExactArgs(1),
 		Annotations: map[string]string{
-			"args": "baseDir",
+			"args":                "baseDir",
+			"flags/diff-strategy": "diffStrategy",
 		},
 	}
 	cmd.Run = func(cmd *cobra.Command, args []string) {
@@ -52,6 +58,10 @@ func diffCmd() *cobra.Command {
 			log.Fatalln("Evaluating jsonnet:", err)
 		}
 
+		if kube.Spec.DiffStrategy == "" {
+			kube.Spec.DiffStrategy = cmd.Flag("diff-strategy").Value.String()
+		}
+
 		desired, err := kube.Reconcile(raw)
 		if err != nil {
 			log.Fatalln("Reconciling:", err)
@@ -70,6 +80,7 @@ func diffCmd() *cobra.Command {
 			fmt.Println(changes)
 		}
 	}
+	cmd.Flags().String("diff-strategy", "", "force the diff-strategy to use. Automatically chosen if not set.")
 	return cmd
 }
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/spf13/pflag v1.0.3
 	github.com/spf13/viper v1.3.2
 	github.com/stretchr/objx v0.2.0
+	github.com/stretchr/testify v1.3.0
 	github.com/thoas/go-funk v0.4.0
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
 	golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f // indirect

pkg/config/v1alpha1/config.go

@@ -16,6 +16,7 @@ type Metadata struct {
 
 // Spec defines Kubernetes properties
 type Spec struct {
-	APIServer string `json:"apiServer"`
-	Namespace string `json:"namespace"`
+	APIServer    string `json:"apiServer"`
+	Namespace    string `json:"namespace"`
+	DiffStrategy string `json:"diffStrategy"`
 }

pkg/kubernetes/errors.go

@@ -0,0 +1,11 @@
+package kubernetes
+
+import "fmt"
+
+type ErrorNotFound struct {
+	resource string
+}
+
+func (e ErrorNotFound) Error() string {
+	return fmt.Sprintf(`error from server (NotFound): secrets "%s" not found`, e.resource)
+}

pkg/kubernetes/kubectl.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/Masterminds/semver"
 	"github.com/fatih/color"
@@ -29,6 +30,9 @@ type Kubectl struct {
 // Version returns the version of kubectl and the Kubernetes api server
 func (k Kubectl) Version() (client, server semver.Version, err error) {
 	zero := *semver.MustParse("0.0.0")
+	if err := k.setupContext(); err != nil {
+		return zero, zero, err
+	}
 	cmd := exec.Command("kubectl", "version",
 		"-o", "json",
 		"--context", k.context.Get("name").MustStr(),
@@ -46,6 +50,10 @@ func (k Kubectl) Version() (client, server semver.Version, err error) {
 
 // setupContext uses `kubectl config view` to obtain the KUBECONFIG and extracts the correct context from it
 func (k *Kubectl) setupContext() error {
+	if k.context != nil {
+		return nil
+	}
+
 	cmd := exec.Command("kubectl", "config", "view", "-o", "json")
 	cfgJSON := bytes.Buffer{}
 	cmd.Stdout = &cfgJSON
@@ -102,14 +110,18 @@ func (k Kubectl) Get(namespace, kind, name string) (map[string]interface{}, erro
 		kind, name,
 	}
 	cmd := exec.Command("kubectl", argv...)
-	raw := bytes.Buffer{}
-	cmd.Stdout = &raw
-	cmd.Stderr = os.Stderr
+	var sout, serr bytes.Buffer
+	cmd.Stdout = &sout
+	cmd.Stderr = &serr
 	if err := cmd.Run(); err != nil {
+		if strings.HasPrefix(serr.String(), "Error from server (NotFound)") {
+			return nil, ErrorNotFound{name}
+		}
+		fmt.Println(serr.String())
 		return nil, err
 	}
 	var obj map[string]interface{}
-	if err := json.Unmarshal(raw.Bytes(), &obj); err != nil {
+	if err := json.Unmarshal(sout.Bytes(), &obj); err != nil {
 		return nil, err
 	}
 	return obj, nil
@@ -164,14 +176,6 @@ func (k Kubectl) Diff(yaml string) (string, error) {
 		return "", err
 	}
 
-	client, server, err := k.Version()
-	if !client.GreaterThan(semver.MustParse("1.13.0")) || !server.GreaterThan(semver.MustParse("1.13.0")) {
-		return "", fmt.Errorf("The kubernetes diff feature requires at least version 1.13 on both, kubectl (is `%s`) and server (is `%s`)", client.String(), server.String())
-	}
-	if err != nil {
-		return "", err
-	}
-
 	argv := []string{"diff",
 		"--context", k.context.Get("name").MustStr(),
 		"-f", "-",

pkg/kubernetes/kubernetes.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"bytes"
 
+	"github.com/Masterminds/semver"
 	"github.com/pkg/errors"
 	"github.com/stretchr/objx"
 	yaml "gopkg.in/yaml.v2"
@@ -13,13 +14,24 @@ import (
 // Kubernetes bridges tanka to the Kubernetse orchestrator.
 type Kubernetes struct {
 	client Kubectl
-	spec   v1alpha1.Spec
+	Spec   v1alpha1.Spec
+
+	// Diffing
+	differs map[string]Differ // List of diff strategies
 }
 
+type Differ func(yaml string) (string, error)
+
 // New creates a new Kubernetes
 func New(s v1alpha1.Spec) *Kubernetes {
-	k := Kubernetes{spec: s}
-	k.client.APIServer = k.spec.APIServer
+	k := Kubernetes{
+		Spec: s,
+	}
+	k.client.APIServer = k.Spec.APIServer
+	k.differs = map[string]Differ{
+		"native": k.client.Diff,
+		"subset": k.client.SubsetDiff,
+	}
 	return &k
 }
 
@@ -36,7 +48,7 @@ func (k *Kubernetes) Reconcile(raw map[string]interface{}) (state []Manifest, er
 	}
 	for _, d := range docs {
 		m := objx.New(d)
-		m.Set("metadata.namespace", k.spec.Namespace)
+		m.Set("metadata.namespace", k.Spec.Namespace)
 		out = append(out, Manifest(m))
 	}
 	return out, nil
@@ -71,5 +83,15 @@ func (k *Kubernetes) Diff(state []Manifest) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return k.client.Diff(yaml)
+
+	if k.Spec.DiffStrategy == "" {
+		k.Spec.DiffStrategy = "native"
+		if _, server, err := k.client.Version(); err == nil {
+			if !server.GreaterThan(semver.MustParse("0.13.0")) {
+				k.Spec.DiffStrategy = "subset"
+			}
+		}
+	}
+
+	return k.differs[k.Spec.DiffStrategy](yaml)
 }

pkg/kubernetes/subsetdiff.go

@@ -0,0 +1,130 @@
+package kubernetes
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/objx"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/sh0rez/tanka/pkg/util"
+)
+
+type difference struct {
+	live, merged string
+}
+
+func (k Kubectl) SubsetDiff(y string) (string, error) {
+	docs := map[string]difference{}
+	d := yaml.NewDecoder(strings.NewReader(y))
+	for {
+
+		// jsonnet output -> desired state
+		var rawShould map[interface{}]interface{}
+		err := d.Decode(&rawShould)
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return "", errors.Wrap(err, "decoding yaml")
+		}
+
+		// filename
+		m := objx.New(util.CleanupInterfaceMap(rawShould))
+		name := strings.Replace(fmt.Sprintf("%s.%s.%s.%s",
+			m.Get("apiVersion").MustStr(),
+			m.Get("kind").MustStr(),
+			m.Get("metadata.namespace").MustStr(),
+			m.Get("metadata.name").MustStr(),
+		), "/", "-", -1)
+
+		// kubectl output -> current state
+		rawIs, err := k.Get(
+			m.Get("metadata.namespace").MustStr(),
+			m.Get("kind").MustStr(),
+			m.Get("metadata.name").MustStr(),
+		)
+		if err != nil {
+			if _, ok := err.(ErrorNotFound); ok {
+				rawIs = map[string]interface{}{}
+			} else {
+				return "", errors.Wrap(err, "getting state from cluster")
+			}
+		}
+
+		should, err := yaml.Marshal(rawShould)
+		if err != nil {
+			return "", err
+		}
+
+		is, err := yaml.Marshal(subset(m, rawIs))
+		if err != nil {
+			return "", err
+		}
+		if string(is) == "{}\n" {
+			is = []byte("")
+		}
+		docs[name] = difference{string(is), string(should)}
+	}
+
+	s := ""
+	for k, v := range docs {
+		d, err := diff(k, v.live, v.merged)
+		if err != nil {
+			return "", errors.Wrap(err, "invoking diff")
+		}
+		if d != "" {
+			d += "\n"
+		}
+		s += d
+	}
+
+	return s, nil
+}
+
+// subset removes all keys from is, that are not present in should.
+// It makes is a subset of should.
+// Kubernetes returns more keys than we can know about.
+// This means, we need to remove all keys from the kubectl output, that are not present locally.
+func subset(should, is map[string]interface{}) map[string]interface{} {
+	if should["namespace"] != nil {
+		is["namespace"] = should["namespace"]
+	}
+	for k, v := range is {
+		if should[k] == nil {
+			delete(is, k)
+			continue
+		}
+
+		switch b := v.(type) {
+		case map[string]interface{}:
+			if a, ok := should[k].(map[string]interface{}); ok {
+				is[k] = subset(a, b)
+			}
+		case []map[string]interface{}:
+			for i := range b {
+				if a, ok := should[k].([]map[string]interface{}); ok {
+					b[i] = subset(a[i], b[i])
+				}
+			}
+		case []interface{}:
+			for i := range b {
+				if a, ok := should[k].([]interface{}); ok {
+					aa, ok := a[i].(map[string]interface{})
+					if !ok {
+						continue
+					}
+					bb, ok := b[i].(map[string]interface{})
+					if !ok {
+						continue
+					}
+					b[i] = subset(aa, bb)
+				}
+			}
+		}
+	}
+	return is
+}