diff --git a/.gitignore b/.gitignore index a5546f775..1521c8b76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1 @@ dist -tk - diff --git a/cmd/tk/tool.go b/cmd/tk/tool.go index 72f77e51b..11d5fb3de 100644 --- a/cmd/tk/tool.go +++ b/cmd/tk/tool.go @@ -20,8 +20,11 @@ func toolCmd() *cli.Command { Short: "handy utilities for working with jsonnet", Use: "tool [command]", } - cmd.AddCommand(jpathCmd()) - cmd.AddCommand(importsCmd()) + cmd.AddCommand( + jpathCmd(), + importsCmd(), + chartsCmd(), + ) return cmd } diff --git a/cmd/tk/toolCharts.go b/cmd/tk/toolCharts.go new file mode 100644 index 000000000..217234c86 --- /dev/null +++ b/cmd/tk/toolCharts.go @@ -0,0 +1,126 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/go-clix/cli" + "github.com/grafana/tanka/pkg/helm" + "gopkg.in/yaml.v2" +) + +func chartsCmd() *cli.Command { + cmd := &cli.Command{ + Use: "charts", + Short: "Declarative vendoring of Helm Charts", + } + + cmd.AddCommand( + chartsInitCmd(), + chartsAddCmd(), + chartsVendorCmd(), + chartsConfigCmd(), + ) + + return cmd +} + +func chartsVendorCmd() *cli.Command { + cmd := &cli.Command{ + Use: "vendor", + Short: "Download Charts to a local folder", + } + + cmd.Run = func(cmd *cli.Command, args []string) error { + c, err := loadChartfile() + if err != nil { + return err + } + + return c.Vendor() + } + + return cmd +} + +func chartsAddCmd() *cli.Command { + cmd := &cli.Command{ + Use: "add [chart@version] [...]", + Short: "Adds Charts to the chartfile", + } + + cmd.Run = func(cmd *cli.Command, args []string) error { + c, err := loadChartfile() + if err != nil { + return err + } + + return c.Add(args) + } + + return cmd +} + +func chartsConfigCmd() *cli.Command { + cmd := &cli.Command{ + Use: "config", + Short: "Displays the current manifest", + } + + cmd.Run = func(cmd *cli.Command, args []string) error { + c, err := loadChartfile() + if err != nil { + return err + } + + data, err := yaml.Marshal(c.Manifest) + if err != nil { + return err + } + + fmt.Print(string(data)) + + return nil + } + + return cmd +} + +func chartsInitCmd() *cli.Command { + cmd := &cli.Command{ + Use: "init", + Short: "Create a new Chartfile", + } + + cmd.Run = func(cmd *cli.Command, args []string) error { + wd, err := os.Getwd() + if err != nil { + return err + } + + path := filepath.Join(wd, helm.Filename) + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("Chartfile at '%s' already exists. Aborting", path) + } + + if _, err := helm.InitChartfile(path); err != nil { + return err + } + + log.Printf("Success! New Chartfile created at '%s'", path) + return nil + } + + return cmd +} + +func loadChartfile() (*helm.Charts, error) { + wd, err := os.Getwd() + if err != nil { + return nil, err + } + + return helm.LoadChartfile(wd) +} diff --git a/go.mod b/go.mod index 9db35c4c5..8a06dbdc0 100644 --- a/go.mod +++ b/go.mod @@ -21,4 +21,5 @@ require ( gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652 k8s.io/apimachinery v0.18.3 + sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index ae1d2b21e..538a0c44b 100644 --- a/go.sum +++ b/go.sum @@ -151,4 +151,5 @@ sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/helm/charts.go b/pkg/helm/charts.go new file mode 100644 index 000000000..7a66c1e53 --- /dev/null +++ b/pkg/helm/charts.go @@ -0,0 +1,201 @@ +package helm + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/Masterminds/semver" + "sigs.k8s.io/yaml" +) + +// LoadChartfile opens a Chartfile tree +func LoadChartfile(projectRoot string) (*Charts, error) { + // make sure project root is valid + abs, err := filepath.Abs(projectRoot) + if err != nil { + return nil, err + } + + // open chartfile + chartfile := filepath.Join(abs, Filename) + data, err := ioutil.ReadFile(chartfile) + if err != nil { + return nil, err + } + + // parse it + c := Chartfile{ + Version: Version, + Directory: DefaultDir, + } + if err := yaml.UnmarshalStrict(data, &c); err != nil { + return nil, err + } + + for i, r := range c.Requires { + if r.Chart == "" { + return nil, fmt.Errorf("requirements[%v]: 'chart' must be set", i) + } + } + + // return Charts handle + charts := &Charts{ + Manifest: c, + projectRoot: abs, + + // default to ExecHelm, but allow injecting from the outside + Helm: ExecHelm{}, + } + return charts, nil +} + +// Charts exposes the central Chartfile management functions +type Charts struct { + // Manifest are the chartfile.yaml contents. It holds data about the developers intentions + Manifest Chartfile + + // projectRoot is the enclosing directory of chartfile.yaml + projectRoot string + + // Helm is the helm implementation underneath. ExecHelm is the default, but + // any implementation of the Helm interface may be used + Helm Helm +} + +// ChartDir returns the directory pulled charts are saved in +func (c Charts) ChartDir() string { + return filepath.Join(c.projectRoot, c.Manifest.Directory) +} + +// ManifestFile returns the full path to the chartfile.yaml +func (c Charts) ManifestFile() string { + return filepath.Join(c.projectRoot, Filename) +} + +// Vendor pulls all Charts specified in the manifest into the local charts +// directory. It fetches the repository index before doing so. +func (c Charts) Vendor() error { + dir := c.ChartDir() + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + log.Println("Syncing Repositories ...") + if err := c.Helm.RepoUpdate(Opts{Repositories: c.Manifest.Repositories}); err != nil { + return err + } + + log.Println("Pulling Charts ...") + for _, r := range c.Manifest.Requires { + err := c.Helm.Pull(r.Chart, r.Version.String(), PullOpts{ + Destination: dir, + Opts: Opts{Repositories: c.Manifest.Repositories}, + }) + if err != nil { + return err + } + + log.Printf(" %s@%s", r.Chart, r.Version.String()) + } + + return nil +} + +// Add adds every Chart in reqs to the Manifest after validation, and runs +// Vendor afterwards +func (c Charts) Add(reqs []string) error { + log.Printf("Adding %v Charts ...", len(reqs)) + + skip := func(s string, err error) { + log.Printf(" Skipping %s: %s.", s, err) + } + + // parse new charts, append in memory + added := 0 + for _, s := range reqs { + r, err := parseReq(s) + if err != nil { + skip(s, err) + continue + } + + if c.Manifest.Requires.Has(*r) { + skip(s, fmt.Errorf("already exists")) + continue + } + + c.Manifest.Requires = append(c.Manifest.Requires, *r) + added++ + log.Println(" OK:", s) + } + + // write out + if err := write(c.Manifest, c.ManifestFile()); err != nil { + return err + } + + // skipped some? fail then + if added != len(reqs) { + return fmt.Errorf("%v Charts were skipped. Please check above logs for details", len(reqs)-added) + } + + // worked fine? vendor it + log.Printf("Added %v Charts to helmfile.yaml. Vendoring ...", added) + return c.Vendor() +} + +func InitChartfile(path string) (*Charts, error) { + c := Chartfile{ + Version: Version, + Repositories: []Repo{{ + Name: "stable", + URL: "https://kubernetes-charts.storage.googleapis.com", + }}, + Requires: make(Requirements, 0), + } + + if err := write(c, path); err != nil { + return nil, err + } + + return LoadChartfile(filepath.Dir(path)) +} + +// write saves a Chartfile to dest +func write(c Chartfile, dest string) error { + data, err := yaml.Marshal(c) + if err != nil { + return err + } + + return ioutil.WriteFile(dest, data, 0644) +} + +var chartExp = regexp.MustCompile(`\w+\/\w+@.+`) + +// parseReq parses a requirement from a string of the format `repo/name@version` +func parseReq(s string) (*Requirement, error) { + if !chartExp.MatchString(s) { + return nil, fmt.Errorf("not of form 'repo/chart@version'") + } + + elems := strings.Split(s, "@") + chart := elems[0] + ver, err := semver.NewVersion(elems[1]) + if errors.Is(err, semver.ErrInvalidSemVer) { + return nil, fmt.Errorf("version is invalid") + } else if err != nil { + return nil, fmt.Errorf("version is invalid: %s", err) + } + + return &Requirement{ + Chart: chart, + Version: *ver, + }, nil +} diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go new file mode 100644 index 000000000..10354456c --- /dev/null +++ b/pkg/helm/helm.go @@ -0,0 +1,114 @@ +package helm + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" +) + +// Helm provides high level access to some Helm operations +type Helm interface { + // Pull downloads a Helm Chart from a remote + Pull(chart, version string, opts PullOpts) error + + // RepoUpdate fetches the latest remote index + RepoUpdate(opts Opts) error +} + +// PullOpts are additional, non-required options for Helm.Pull +type PullOpts struct { + Opts + + // Directory to put the resulting .tgz into + Destination string +} + +// Opts are additional, non-required options that all Helm operations accept +type Opts struct { + Repositories []Repo +} + +// ExecHelm is a Helm implementation powered by the `helm` command line utility +type ExecHelm struct{} + +// Pull implements Helm.Pull +func (e ExecHelm) Pull(chart, version string, opts PullOpts) error { + repoFile, err := writeRepoTmpFile(opts.Repositories) + if err != nil { + return err + } + defer os.Remove(repoFile) + + cmd := e.cmd("pull", chart, + "--version", version, + "--destination", opts.Destination, + "--repository-config", repoFile, + ) + + return cmd.Run() +} + +// RepoUpdate implements Helm.RepoUpdate +func (e ExecHelm) RepoUpdate(opts Opts) error { + repoFile, err := writeRepoTmpFile(opts.Repositories) + if err != nil { + return err + } + defer os.Remove(repoFile) + + cmd := e.cmd("repo", "update", + "--repository-config", repoFile, + ) + var errBuf bytes.Buffer + cmd.Stderr = &errBuf + + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s\n%s", errBuf.String(), err) + } + + return nil +} + +// cmd returns a prepared exec.Cmd to use the `helm` binary +func (e ExecHelm) cmd(action string, args ...string) *exec.Cmd { + argv := []string{action} + argv = append(argv, args...) + + cmd := helmCmd(argv...) + cmd.Stderr = os.Stderr + + return cmd +} + +// helmCmd returns a bare exec.Cmd pointed at the local helm binary +func helmCmd(args ...string) *exec.Cmd { + bin := "helm" + if env := os.Getenv("TANKA_HELM_PATH"); env != "" { + bin = env + } + + return exec.Command(bin, args...) +} + +// writeRepoTmpFile creates a temporary repositories.yaml from the passed Repo +// slice to be used by the helm binary +func writeRepoTmpFile(r []Repo) (string, error) { + m := map[string]interface{}{ + "repositories": r, + } + + f, err := ioutil.TempFile("", "charts-repos") + if err != nil { + return "", err + } + + enc := json.NewEncoder(f) + if err := enc.Encode(m); err != nil { + return "", err + } + + return f.Name(), nil +} diff --git a/pkg/helm/spec.go b/pkg/helm/spec.go new file mode 100644 index 000000000..23b799234 --- /dev/null +++ b/pkg/helm/spec.go @@ -0,0 +1,65 @@ +package helm + +import ( + "github.com/Masterminds/semver" +) + +const ( + // Version of the current Chartfile implementation + Version = 1 + + // Filename of the Chartfile + Filename = "chartfile.yaml" + + // DefaultDir is the directory used for storing Charts if not specified + // otherwise + DefaultDir = "charts" +) + +// Chartfile is the schema used to declaratively define locally required Helm +// Charts +type Chartfile struct { + // Version of the Chartfile schema (for future use) + Version uint `json:"version"` + + // Repositories to source from + Repositories []Repo `json:"repositories"` + + // Requires lists Charts expected to be present in the charts folder + Requires Requirements `json:"requires"` + + // Folder to use for storing Charts. Defaults to 'charts' + Directory string `json:"directory,omitempty"` +} + +// Repo describes a single Helm repository +type Repo struct { + Name string `json:"name,omitempty"` + URL string `json:"url,omitempty"` + CAFile string `json:"caFile,omitempty"` + CertFile string `json:"certFile,omitempty"` + KeyFile string `json:"keyFile,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` +} + +// Requirement describes a single required Helm Chart. +// Both, Chart and Version are required +type Requirement struct { + Chart string `json:"chart"` + Version semver.Version `json:"version"` +} + +// Requirements is an aggregate of all required Charts +type Requirements []Requirement + +// Has reports whether 'req' is already part of the requirements +func (r Requirements) Has(req Requirement) bool { + for _, x := range r { + if x == req { + return true + } + } + + return false +} diff --git a/pkg/helmraiser/helm.go b/pkg/helm/template.go similarity index 87% rename from pkg/helmraiser/helm.go rename to pkg/helm/template.go index 488b771c9..2f8171947 100644 --- a/pkg/helmraiser/helm.go +++ b/pkg/helm/template.go @@ -1,4 +1,4 @@ -package helmraiser +package helm import ( "bytes" @@ -7,7 +7,6 @@ import ( "io" "io/ioutil" "os" - "os/exec" "strings" jsonnet "github.com/google/go-jsonnet" @@ -17,24 +16,6 @@ import ( yaml "gopkg.in/yaml.v3" ) -// Helm provides actions on Helm charts. -type Helm struct{} - -func (h Helm) cmd(action string, args ...string) *exec.Cmd { - argv := []string{action} - argv = append(argv, args...) - - return helmCmd(argv...) -} - -func helmCmd(args ...string) *exec.Cmd { - binary := "helm" - if env := os.Getenv("TANKA_HELM_PATH"); env != "" { - binary = env - } - return exec.Command(binary, args...) -} - // TemplateOpts defines additional parameters that can be passed to the // Helm.Template action type TemplateOpts struct { @@ -74,7 +55,7 @@ func confToArgs(conf TemplateOpts) ([]string, []string, error) { // Template expands a Helm Chart into a regular manifest.List using the `helm // template` command -func (h Helm) Template(name, chart string, opts TemplateOpts) (manifest.List, error) { +func (h ExecHelm) Template(name, chart string, opts TemplateOpts) (manifest.List, error) { confArgs, tmpFiles, err := confToArgs(opts) if err != nil { return nil, err @@ -139,7 +120,8 @@ func NativeFunc() *jsonnet.NativeFunction { return "", err } - var h Helm + // TODO: Define Template on the Helm interface instead + var h ExecHelm list, err := h.Template(name, chart, conf) if err != nil { return nil, err diff --git a/pkg/helmraiser/helm_test.go b/pkg/helm/template_test.go similarity index 98% rename from pkg/helmraiser/helm_test.go rename to pkg/helm/template_test.go index f316fc4b0..056e69dbd 100644 --- a/pkg/helmraiser/helm_test.go +++ b/pkg/helm/template_test.go @@ -1,4 +1,4 @@ -package helmraiser +package helm import ( "fmt" diff --git a/pkg/jsonnet/native/funcs.go b/pkg/jsonnet/native/funcs.go index 8dbdb463a..e5b250d83 100644 --- a/pkg/jsonnet/native/funcs.go +++ b/pkg/jsonnet/native/funcs.go @@ -9,7 +9,7 @@ import ( jsonnet "github.com/google/go-jsonnet" "github.com/google/go-jsonnet/ast" - "github.com/grafana/tanka/pkg/helmraiser" + "github.com/grafana/tanka/pkg/helm" "github.com/pkg/errors" yaml "gopkg.in/yaml.v3" ) @@ -31,7 +31,7 @@ func Funcs() []*jsonnet.NativeFunction { regexMatch(), regexSubst(), - helmraiser.NativeFunc(), + helm.NativeFunc(), } }