Skip to content

Commit

Permalink
feat(kubernetes): subset-diff (grafana#11)
Browse files Browse the repository at this point in the history
So far, diffing was offloaded to kubectl. While this produces very nice results,
it is only possible for kubernetes version 1.13+.

Nevertheless, there are older cluster versions around, so these need to be
supported as well. subset-diff addresses those cases in the hopefully best way
possible.

To reduce field bloat, it only diffes those fields, that are present in the
local config. Kubernetes adds dynamic fields on the fly which we cannot know
about, so this is required.

Note: You WILL NOT see removed fields in the diff output. Upgrade your cluster
version to 1.13+ and use native diffing.
  • Loading branch information
sh0rez authored Aug 7, 2019
1 parent aea3bdf commit 13f6fdd
Show file tree
Hide file tree
Showing 10 changed files with 416 additions and 20 deletions.
13 changes: 12 additions & 1 deletion cmd/tk/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"os"

"github.com/alecthomas/chroma/quick"
"github.com/posener/complete"
"github.com/sh0rez/tanka/pkg/cmp"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh/terminal"
)
Expand Down Expand Up @@ -38,12 +40,16 @@ func applyCmd() *cobra.Command {
}

func diffCmd() *cobra.Command {
// completion
cmp.Handlers.Add("diffStrategy", complete.PredictSet("native", "subset"))

cmd := &cobra.Command{
Use: "diff [directory]",
Short: "differences between the configuration and the cluster",
Args: cobra.ExactArgs(1),
Annotations: map[string]string{
"args": "baseDir",
"args": "baseDir",
"flags/diff-strategy": "diffStrategy",
},
}
cmd.Run = func(cmd *cobra.Command, args []string) {
Expand All @@ -52,6 +58,10 @@ func diffCmd() *cobra.Command {
log.Fatalln("Evaluating jsonnet:", err)
}

if kube.Spec.DiffStrategy == "" {
kube.Spec.DiffStrategy = cmd.Flag("diff-strategy").Value.String()
}

desired, err := kube.Reconcile(raw)
if err != nil {
log.Fatalln("Reconciling:", err)
Expand All @@ -70,6 +80,7 @@ func diffCmd() *cobra.Command {
fmt.Println(changes)
}
}
cmd.Flags().String("diff-strategy", "", "force the diff-strategy to use. Automatically chosen if not set.")
return cmd
}

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.3.2
github.com/stretchr/objx v0.2.0
github.com/stretchr/testify v1.3.0
github.com/thoas/go-funk v0.4.0
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f // indirect
Expand Down
5 changes: 3 additions & 2 deletions pkg/config/v1alpha1/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type Metadata struct {

// Spec defines Kubernetes properties
type Spec struct {
APIServer string `json:"apiServer"`
Namespace string `json:"namespace"`
APIServer string `json:"apiServer"`
Namespace string `json:"namespace"`
DiffStrategy string `json:"diffStrategy"`
}
11 changes: 11 additions & 0 deletions pkg/kubernetes/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package kubernetes

import "fmt"

type ErrorNotFound struct {
resource string
}

func (e ErrorNotFound) Error() string {
return fmt.Sprintf(`error from server (NotFound): secrets "%s" not found`, e.resource)
}
28 changes: 16 additions & 12 deletions pkg/kubernetes/kubectl.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"os"
"os/exec"
"strings"

"github.com/Masterminds/semver"
"github.com/fatih/color"
Expand All @@ -29,6 +30,9 @@ type Kubectl struct {
// Version returns the version of kubectl and the Kubernetes api server
func (k Kubectl) Version() (client, server semver.Version, err error) {
zero := *semver.MustParse("0.0.0")
if err := k.setupContext(); err != nil {
return zero, zero, err
}
cmd := exec.Command("kubectl", "version",
"-o", "json",
"--context", k.context.Get("name").MustStr(),
Expand All @@ -46,6 +50,10 @@ func (k Kubectl) Version() (client, server semver.Version, err error) {

// setupContext uses `kubectl config view` to obtain the KUBECONFIG and extracts the correct context from it
func (k *Kubectl) setupContext() error {
if k.context != nil {
return nil
}

cmd := exec.Command("kubectl", "config", "view", "-o", "json")
cfgJSON := bytes.Buffer{}
cmd.Stdout = &cfgJSON
Expand Down Expand Up @@ -102,14 +110,18 @@ func (k Kubectl) Get(namespace, kind, name string) (map[string]interface{}, erro
kind, name,
}
cmd := exec.Command("kubectl", argv...)
raw := bytes.Buffer{}
cmd.Stdout = &raw
cmd.Stderr = os.Stderr
var sout, serr bytes.Buffer
cmd.Stdout = &sout
cmd.Stderr = &serr
if err := cmd.Run(); err != nil {
if strings.HasPrefix(serr.String(), "Error from server (NotFound)") {
return nil, ErrorNotFound{name}
}
fmt.Println(serr.String())
return nil, err
}
var obj map[string]interface{}
if err := json.Unmarshal(raw.Bytes(), &obj); err != nil {
if err := json.Unmarshal(sout.Bytes(), &obj); err != nil {
return nil, err
}
return obj, nil
Expand Down Expand Up @@ -164,14 +176,6 @@ func (k Kubectl) Diff(yaml string) (string, error) {
return "", err
}

client, server, err := k.Version()
if !client.GreaterThan(semver.MustParse("1.13.0")) || !server.GreaterThan(semver.MustParse("1.13.0")) {
return "", fmt.Errorf("The kubernetes diff feature requires at least version 1.13 on both, kubectl (is `%s`) and server (is `%s`)", client.String(), server.String())
}
if err != nil {
return "", err
}

argv := []string{"diff",
"--context", k.context.Get("name").MustStr(),
"-f", "-",
Expand Down
32 changes: 27 additions & 5 deletions pkg/kubernetes/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package kubernetes
import (
"bytes"

"github.com/Masterminds/semver"
"github.com/pkg/errors"
"github.com/stretchr/objx"
yaml "gopkg.in/yaml.v2"
Expand All @@ -13,13 +14,24 @@ import (
// Kubernetes bridges tanka to the Kubernetse orchestrator.
type Kubernetes struct {
client Kubectl
spec v1alpha1.Spec
Spec v1alpha1.Spec

// Diffing
differs map[string]Differ // List of diff strategies
}

type Differ func(yaml string) (string, error)

// New creates a new Kubernetes
func New(s v1alpha1.Spec) *Kubernetes {
k := Kubernetes{spec: s}
k.client.APIServer = k.spec.APIServer
k := Kubernetes{
Spec: s,
}
k.client.APIServer = k.Spec.APIServer
k.differs = map[string]Differ{
"native": k.client.Diff,
"subset": k.client.SubsetDiff,
}
return &k
}

Expand All @@ -36,7 +48,7 @@ func (k *Kubernetes) Reconcile(raw map[string]interface{}) (state []Manifest, er
}
for _, d := range docs {
m := objx.New(d)
m.Set("metadata.namespace", k.spec.Namespace)
m.Set("metadata.namespace", k.Spec.Namespace)
out = append(out, Manifest(m))
}
return out, nil
Expand Down Expand Up @@ -71,5 +83,15 @@ func (k *Kubernetes) Diff(state []Manifest) (string, error) {
if err != nil {
return "", err
}
return k.client.Diff(yaml)

if k.Spec.DiffStrategy == "" {
k.Spec.DiffStrategy = "native"
if _, server, err := k.client.Version(); err == nil {
if !server.GreaterThan(semver.MustParse("0.13.0")) {
k.Spec.DiffStrategy = "subset"
}
}
}

return k.differs[k.Spec.DiffStrategy](yaml)
}
130 changes: 130 additions & 0 deletions pkg/kubernetes/subsetdiff.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package kubernetes

import (
"fmt"
"io"
"strings"

"github.com/pkg/errors"
"github.com/stretchr/objx"
yaml "gopkg.in/yaml.v2"

"github.com/sh0rez/tanka/pkg/util"
)

type difference struct {
live, merged string
}

func (k Kubectl) SubsetDiff(y string) (string, error) {
docs := map[string]difference{}
d := yaml.NewDecoder(strings.NewReader(y))
for {

// jsonnet output -> desired state
var rawShould map[interface{}]interface{}
err := d.Decode(&rawShould)
if err == io.EOF {
break
}

if err != nil {
return "", errors.Wrap(err, "decoding yaml")
}

// filename
m := objx.New(util.CleanupInterfaceMap(rawShould))
name := strings.Replace(fmt.Sprintf("%s.%s.%s.%s",
m.Get("apiVersion").MustStr(),
m.Get("kind").MustStr(),
m.Get("metadata.namespace").MustStr(),
m.Get("metadata.name").MustStr(),
), "/", "-", -1)

// kubectl output -> current state
rawIs, err := k.Get(
m.Get("metadata.namespace").MustStr(),
m.Get("kind").MustStr(),
m.Get("metadata.name").MustStr(),
)
if err != nil {
if _, ok := err.(ErrorNotFound); ok {
rawIs = map[string]interface{}{}
} else {
return "", errors.Wrap(err, "getting state from cluster")
}
}

should, err := yaml.Marshal(rawShould)
if err != nil {
return "", err
}

is, err := yaml.Marshal(subset(m, rawIs))
if err != nil {
return "", err
}
if string(is) == "{}\n" {
is = []byte("")
}
docs[name] = difference{string(is), string(should)}
}

s := ""
for k, v := range docs {
d, err := diff(k, v.live, v.merged)
if err != nil {
return "", errors.Wrap(err, "invoking diff")
}
if d != "" {
d += "\n"
}
s += d
}

return s, nil
}

// subset removes all keys from is, that are not present in should.
// It makes is a subset of should.
// Kubernetes returns more keys than we can know about.
// This means, we need to remove all keys from the kubectl output, that are not present locally.
func subset(should, is map[string]interface{}) map[string]interface{} {
if should["namespace"] != nil {
is["namespace"] = should["namespace"]
}
for k, v := range is {
if should[k] == nil {
delete(is, k)
continue
}

switch b := v.(type) {
case map[string]interface{}:
if a, ok := should[k].(map[string]interface{}); ok {
is[k] = subset(a, b)
}
case []map[string]interface{}:
for i := range b {
if a, ok := should[k].([]map[string]interface{}); ok {
b[i] = subset(a[i], b[i])
}
}
case []interface{}:
for i := range b {
if a, ok := should[k].([]interface{}); ok {
aa, ok := a[i].(map[string]interface{})
if !ok {
continue
}
bb, ok := b[i].(map[string]interface{})
if !ok {
continue
}
b[i] = subset(aa, bb)
}
}
}
}
return is
}
Loading

0 comments on commit 13f6fdd

Please sign in to comment.