From f918033ac28867e6f3cbfff6bf92d07fe5d273ed Mon Sep 17 00:00:00 2001 From: Michael Burt Date: Fri, 6 Sep 2024 12:01:04 -0600 Subject: [PATCH] Helm3 Support (#307) --- .github/workflows/ci.yaml | 14 +- .gitignore | 1 + .goreleaser.yaml | 5 +- .../components/apache/datasource.libsonnet | 14 + .../helm3/components/apache/index.jsonnet | 1 + .../victoria-metrics/datasource.libsonnet | 7 + .../components/victoria-metrics/index.jsonnet | 1 + examples/helm3/environments/base.libsonnet | 5 + examples/helm3/environments/default.libsonnet | 7 + examples/helm3/params.libsonnet | 9 + examples/helm3/qbec.yaml | 22 ++ go.mod | 1 + go.sum | 1 + internal/commands/eval_test.go | 8 +- site/content/userguide/usage/authoring.md | 8 +- vm/internal/ds/exec/exec.go | 2 +- vm/internal/ds/factory/datasource.go | 4 + vm/internal/ds/helm3/helm3.go | 361 ++++++++++++++++++ vm/internal/ds/helm3/helm3_test.go | 210 ++++++++++ vm/internal/ds/helm3/testdata/apache.json | 358 +++++++++++++++++ 20 files changed, 1021 insertions(+), 18 deletions(-) create mode 100644 examples/helm3/components/apache/datasource.libsonnet create mode 100644 examples/helm3/components/apache/index.jsonnet create mode 100644 examples/helm3/components/victoria-metrics/datasource.libsonnet create mode 100644 examples/helm3/components/victoria-metrics/index.jsonnet create mode 100644 examples/helm3/environments/base.libsonnet create mode 100644 examples/helm3/environments/default.libsonnet create mode 100644 examples/helm3/params.libsonnet create mode 100644 examples/helm3/qbec.yaml create mode 100644 vm/internal/ds/helm3/helm3.go create mode 100644 vm/internal/ds/helm3/helm3_test.go create mode 100644 vm/internal/ds/helm3/testdata/apache.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 87cae96b..1d356190 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,10 +13,10 @@ jobs: with: go-version: 1.17 id: go - - uses: azure/setup-helm@v1 + - uses: azure/setup-helm@v4.2.0 with: - version: "v3.7.0" # default is latest stable - id: helm + version: 'v3.7.0' + id: install - uses: azure/setup-kubectl@v1 id: install-kubectl - uses: actions/checkout@v2 @@ -63,7 +63,7 @@ jobs: - name: goreleaser uses: goreleaser/goreleaser-action@v2 with: - args: release --snapshot --skip-publish --rm-dist --release-notes .release-notes.md + args: release --snapshot --skip=publish --clean --release-notes .release-notes.md env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-windows: @@ -74,10 +74,10 @@ jobs: with: go-version: 1.17 id: go - - uses: azure/setup-helm@v1 + - uses: azure/setup-helm@v4.2.0 with: - version: "v3.3.1" # default is latest stable - id: helm + version: 'v3.7.0' + id: install - uses: actions/checkout@v2 - uses: actions/cache@v2 with: diff --git a/.gitignore b/.gitignore index 4de96360..5a65c867 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ bin/ .tools/ .vscode/ *.generated +.gitconfig diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b62a577b..80053fdd 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,3 +1,4 @@ +version: 2 before: hooks: - make release-notes @@ -43,7 +44,7 @@ archives: brews: - name: qbec - tap: + repository: owner: splunk name: homebrew-tap url_template: https://github.com/splunk/qbec/releases/download/{{.Tag}}/{{.ArtifactName}} @@ -59,7 +60,7 @@ brews: checksum: name_template: "sha256-checksums.txt" snapshot: - name_template: "{{.Version}}-next" + version_template: "{{.Version}}-next" changelog: sort: asc filters: diff --git a/examples/helm3/components/apache/datasource.libsonnet b/examples/helm3/components/apache/datasource.libsonnet new file mode 100644 index 00000000..7925ea12 --- /dev/null +++ b/examples/helm3/components/apache/datasource.libsonnet @@ -0,0 +1,14 @@ +{ + objects: import 'data://helm/apache?config-from=apache-config', + config: { + name: 'mock-release', + options: { + repo: 'https://charts.bitnami.com/bitnami', + version: '11.2.17', + namespace: 'foobar', + }, + values: { + key: 'value', + }, + }, +} diff --git a/examples/helm3/components/apache/index.jsonnet b/examples/helm3/components/apache/index.jsonnet new file mode 100644 index 00000000..7a6a63bf --- /dev/null +++ b/examples/helm3/components/apache/index.jsonnet @@ -0,0 +1 @@ +(import 'datasource.libsonnet').objects diff --git a/examples/helm3/components/victoria-metrics/datasource.libsonnet b/examples/helm3/components/victoria-metrics/datasource.libsonnet new file mode 100644 index 00000000..ccd4468e --- /dev/null +++ b/examples/helm3/components/victoria-metrics/datasource.libsonnet @@ -0,0 +1,7 @@ +{ + objects: import 'data://helm/github.com/VictoriaMetrics/helm-charts/raw/347d4558d9c25cd341718bf5a2ee167da042c080/packages/victoria-metrics-cluster-0.9.6.tgz?config-from=victoria-config', + config: { + options: {}, + values: {}, + }, +} diff --git a/examples/helm3/components/victoria-metrics/index.jsonnet b/examples/helm3/components/victoria-metrics/index.jsonnet new file mode 100644 index 00000000..7a6a63bf --- /dev/null +++ b/examples/helm3/components/victoria-metrics/index.jsonnet @@ -0,0 +1 @@ +(import 'datasource.libsonnet').objects diff --git a/examples/helm3/environments/base.libsonnet b/examples/helm3/environments/base.libsonnet new file mode 100644 index 00000000..3e56f436 --- /dev/null +++ b/examples/helm3/environments/base.libsonnet @@ -0,0 +1,5 @@ +// this file has the baseline default parameters +{ + components: { + }, +} diff --git a/examples/helm3/environments/default.libsonnet b/examples/helm3/environments/default.libsonnet new file mode 100644 index 00000000..50a2b9e7 --- /dev/null +++ b/examples/helm3/environments/default.libsonnet @@ -0,0 +1,7 @@ +// this file has the param overrides for the default environment +local base = import './base.libsonnet'; + +base { + components+: { + }, +} diff --git a/examples/helm3/params.libsonnet b/examples/helm3/params.libsonnet new file mode 100644 index 00000000..7f4e0d1d --- /dev/null +++ b/examples/helm3/params.libsonnet @@ -0,0 +1,9 @@ +// this file returns the params for the current qbec environment +local env = std.extVar('qbec.io/env'); +local paramsMap = import 'glob-import:environments/*.libsonnet'; +local baseFile = if env == '_' then 'base' else env; +local key = 'environments/%s.libsonnet' % baseFile; + +if std.objectHas(paramsMap, key) +then paramsMap[key] +else error 'no param file %s found for environment %s' % [key, env] diff --git a/examples/helm3/qbec.yaml b/examples/helm3/qbec.yaml new file mode 100644 index 00000000..1ef4e79a --- /dev/null +++ b/examples/helm3/qbec.yaml @@ -0,0 +1,22 @@ +apiVersion: qbec.io/v1alpha1 +kind: App +metadata: + name: helm3 +spec: + environments: + default: + defaultNamespace: charts + context: kind + vars: + computed: + - name: helmSetup + code: | + {} + - name: victoria-config + code: | + (import 'components/victoria-metrics/datasource.libsonnet').config + - name: apache-config + code: | + (import 'components/apache/datasource.libsonnet').config + dataSources: + - helm3://helm?configVar=helmSetup diff --git a/go.mod b/go.mod index cdf8b998..99b4993a 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/google/go-jsonnet v0.18.0 github.com/googleapis/gnostic v0.5.5 + github.com/iancoleman/strcase v0.2.0 github.com/jonboulle/clockwork v0.2.2 github.com/mattn/go-isatty v0.0.14 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index ed0cf55d..3b90c81b 100644 --- a/go.sum +++ b/go.sum @@ -456,6 +456,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= diff --git a/internal/commands/eval_test.go b/internal/commands/eval_test.go index d3f68216..1301ec9f 100644 --- a/internal/commands/eval_test.go +++ b/internal/commands/eval_test.go @@ -17,7 +17,6 @@ package commands import ( - "runtime" "testing" "github.com/stretchr/testify/assert" @@ -47,13 +46,8 @@ func TestEvalWithDataSources(t *testing.T) { var data map[string]interface{} err = s.jsonOutput(&data) require.NoError(t, err) - a := assert.New(t) - if runtime.GOOS == "windows" { - a.Equal("-n bar\r\n", data["foo"]) - } else { - a.Equal("bar", data["foo"]) - } + a.Equal("bar", data["foo"]) } func TestEvalVars(t *testing.T) { diff --git a/site/content/userguide/usage/authoring.md b/site/content/userguide/usage/authoring.md index 9130b649..6f6e92a6 100644 --- a/site/content/userguide/usage/authoring.md +++ b/site/content/userguide/usage/authoring.md @@ -26,7 +26,7 @@ A component is: It is valid for a component to return an empty set of objects if runtime parameters determine that nothing should be installed for a specific target environment. -## Using helm charts and external data sources +## Using external data sources qbec provides integration to run external commands and consume their output in jsonnet code. See the [Jsonnet data importer](../../../reference/jsonnet-external-data) for more information on how this works. @@ -34,6 +34,12 @@ See the [Jsonnet data importer](../../../reference/jsonnet-external-data) for mo Note that the [expandHelmTemplate](../../../reference/jsonnet-native-funcs/#expandhelmtemplate) native function is now deprecated in favor of the data importer mechanism. +## Native Helm integration + +qbec provides native support for Helm3, allowing you to render Helm `values` during runtime via jsonnet. To use this +feature, configure a Helm datasource. See [examples/helm3](https://github.com/splunk/qbec/tree/main/examples/helm3/) for +an example component. + ## Using other jsonnet libraries [k8s-yaml-patch](https://github.com/splunk/k8s-yaml-patch), diff --git a/vm/internal/ds/exec/exec.go b/vm/internal/ds/exec/exec.go index cc421a0f..77d809b4 100644 --- a/vm/internal/ds/exec/exec.go +++ b/vm/internal/ds/exec/exec.go @@ -121,11 +121,11 @@ func (d *execSource) Init(p datasource.ConfigProvider) (fErr error) { if err != nil { return err } + c.initDefaults() err = c.assertValid() if err != nil { return err } - c.initDefaults() d.runner = newRunner(&c) return nil } diff --git a/vm/internal/ds/factory/datasource.go b/vm/internal/ds/factory/datasource.go index 230005fa..921161eb 100644 --- a/vm/internal/ds/factory/datasource.go +++ b/vm/internal/ds/factory/datasource.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/splunk/qbec/vm/internal/ds" "github.com/splunk/qbec/vm/internal/ds/exec" + "github.com/splunk/qbec/vm/internal/ds/helm3" ) // Create creates a new data source from the supplied URL. @@ -35,6 +36,7 @@ func Create(u string) (ds.DataSourceWithLifecycle, error) { scheme := parsed.Scheme switch scheme { case exec.Scheme: + case helm3.Scheme: default: return nil, fmt.Errorf("data source URL '%s', unsupported scheme '%s'", u, scheme) } @@ -52,6 +54,8 @@ func Create(u string) (ds.DataSourceWithLifecycle, error) { switch scheme { case exec.Scheme: return makeLazy(exec.New(name, varName)), nil + case helm3.Scheme: + return makeLazy(helm3.New(name, varName)), nil default: return nil, fmt.Errorf("internal error: unable to create a data source for %s", u) } diff --git a/vm/internal/ds/helm3/helm3.go b/vm/internal/ds/helm3/helm3.go new file mode 100644 index 00000000..88c8385e --- /dev/null +++ b/vm/internal/ds/helm3/helm3.go @@ -0,0 +1,361 @@ +/* + Copyright 2021 Splunk Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package helm3 provides a data source implementation that can extract k8s objects out of helm3 charts +package helm3 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/iancoleman/strcase" + "github.com/pkg/errors" + "github.com/splunk/qbec/internal/sio" + "github.com/splunk/qbec/vm/datasource" + "github.com/splunk/qbec/vm/internal/ds" + "github.com/splunk/qbec/vm/internal/natives" +) + +// Scheme is the scheme supported by this data source +const ( + Scheme = "helm3" + configVarParam = "config-from" +) + +var defaultNamespaceVar = "qbec.io/defaultNs" // TODO: make this blank and have qbec set it + +// SetDefaultNamespaceVar sets the name of the external variable that is always set to the default namespace to use. +func SetDefaultNamespaceVar(s string) { + defaultNamespaceVar = s +} + +// Config is the configuration of the data source. TODO: add version check, SHA check options etc. +type Config struct { + Command string `json:"command"` // the executable that is run, default is "helm" + Timeout string `json:"timeout,omitempty"` // command timeout as a duration string + timeout time.Duration // internal representation +} + +// TemplateOptions are a subset of command line arguments that can be passed to `helm template` +type TemplateOptions struct { + Namespace string `json:"namespace,omitempty"` + CreateNamespace bool `json:"createNamespace,omitempty"` + APIVersions []string `json:"apiVersions,omitempty"` + DependencyUpdate bool `json:"dependencyUpdate,omitempty"` + Description string `json:"description"` + Devel bool `json:"devel"` + DisableOpenAPIValidation bool `json:"disableOpenapiValidation,omitempty"` + GenerateName bool `json:"generateName,omitempty"` + IncludeCRDs bool `json:"includeCrds,omitempty"` + IsUpgrade bool `json:"isUpgrade,omitempty"` + SkipCRDs bool `json:"skipCrds,omitempty"` + SkipTests bool `json:"skipTests,omitempty"` + InsecureSkipTLSVerify bool `json:"insecureSkipTlsVerify,omitempty"` + KubeVersion string `json:"kubeVersion,omitempty"` + NameTemplate string `json:"nameTemplate,omitempty"` + NoHooks bool `json:"noHooks,omitempty"` + PassCredentials bool `json:"passCredentials,omitempty"` + Password string `json:"password,omitempty" fld:"secure"` + RenderSubchartNotes bool `json:"renderSubchartNotes,omitempty"` + Replace bool `json:"replace,omitempty"` + Repo string `json:"repo,omitempty"` + ShowOnly []string `json:"showOnly,omitempty"` + Username string `json:"username,omitempty"` + Validate bool `json:"validate,omitempty"` + Verify bool `json:"verify,omitempty"` + Version string `json:"version,omitempty"` + /* + // first pass: omit all options that refer to files since we need to figure out what the exact + // semantics of relative paths are and its corresponding implications of qbec root etc. + --ca-file string verify certificates of HTTPS-enabled servers using this CA bundle + --cert-file string identify HTTPS client using this SSL certificate file + --key-file string identify HTTPS client using this SSL key file + --keyring string location of public keys used for verification (defaults to the public keyring unless your environment is otherwise configured.) + --output-dir string writes the executed templates to files in output-dir instead of stdout + --post-renderer postrenderer the path to an executable to be used for post rendering. If it exists in $PATH, the binary will be used, otherwise it will try to look for the executable at the given path (default exec) + --release-name use release name in the output-dir path. + // ignore values related stuff since we send JSON object via stdin + --set stringArray set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + --set-file stringArray set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) + --set-string stringArray set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + -f, --values strings specify values in a YAML file or a URL (can specify multiple) + // values not relevant for template expansion + --timeout duration time to wait for any individual Kubernetes operation (like Jobs for hooks) (default 5m0s) + --wait if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout + --wait-for-jobs if set and --wait enabled, will wait until all Jobs have been completed before marking the release as successful. It will wait for as long as --timeout + Global Flags: + // we always set debug for better error messages, don't expose + --debug enable verbose output + // we _could_ set all the kube options based on what we know but this requires a dance between qbec knowledge and data source knowledge + --kube-apiserver string the address and the port for the Kubernetes API server + --kube-as-group stringArray group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --kube-as-user string username to impersonate for the operation + --kube-ca-file string the certificate authority file for the Kubernetes API server connection + --kube-context string name of the kubeconfig context to use + --kube-token string bearer token used for authentication + --kubeconfig string path to the kubeconfig file + // more files to ignore for now + --registry-config string path to the registry config file (default "") + --repository-cache string path to the file containing cached repository + --repository-config string path to the file containing repository names and URLs + */ +} + +func (o TemplateOptions) toInternalCommandLine(display bool) []string { + t := reflect.TypeOf(o) + v := reflect.ValueOf(o) + var ret []string + for i := 0; i < t.NumField(); i++ { + fld := t.Field(i) + val := v.FieldByName(fld.Name) + tag := strings.Split(fld.Tag.Get("json"), ",")[0] + secure := strings.Contains(fld.Tag.Get("fld"), "secure") + option := fmt.Sprintf("--%s", strcase.ToKebab(tag)) + kv := func(v interface{}) { + if secure && display { + v = "" + } + ret = append(ret, fmt.Sprintf("%s=%s", option, v)) + } + switch { + case fld.Type.Name() == "string": + out := val.String() + if out != "" { + kv(out) + } + case fld.Type.Name() == "bool": + out := val.Bool() + if out { + ret = append(ret, option) + } + case fld.Type.Kind() == reflect.Slice && fld.Type.Elem().Name() == "string": + for j := 0; j < val.Len(); j++ { + kv(val.Index(j).String()) + } + default: + panic("unsupported field type for field " + fld.Name) + } + } + return ret +} + +func (o TemplateOptions) toCommandLine() []string { + return o.toInternalCommandLine(false) +} + +func (o TemplateOptions) toDisplay() string { + parts := o.toInternalCommandLine(true) + return strings.Join(parts, " ") // TODO: improve me for quoting +} + +// TemplateConfig is the configuration for the template command that includes options and values. +// TODO: make a schema for this and validate inputs before JSON unmarshal +type TemplateConfig struct { + Name string `json:"name,omitempty"` + Options TemplateOptions `json:"options,omitempty"` + Values map[string]interface{} `json:"values,omitempty"` +} + +func findExecutable(cmd string) (string, error) { + if !filepath.IsAbs(cmd) { + p, err := filepath.Abs(cmd) + if err == nil { + stat, err := os.Stat(cmd) + if err == nil { + if m := stat.Mode(); !m.IsDir() && m&0111 != 0 { + return p, nil + } + } + } + } + return exec.LookPath(cmd) +} + +func (c *Config) assertValid() error { + if c.Command == "" { + return fmt.Errorf("command not specified") + } + if c.Timeout != "" { + t, err := time.ParseDuration(c.Timeout) + if err != nil { + return fmt.Errorf("invalid timeout '%s': %v", c.Timeout, err) + } + c.timeout = t + } + exe, err := findExecutable(c.Command) + if err != nil { + return fmt.Errorf("invalid command '%s': %v", c.Command, err) + } + c.Command = exe + // TODO: support version/ SHA checking etc. + return nil +} + +func (c *Config) initDefaults() { + if c.Command == "" { + c.Command = "helm" + } + if c.timeout == 0 { + c.timeout = time.Minute + } +} + +type helm3Source struct { + name string + configVar string + cp datasource.ConfigProvider + config Config +} + +// New creates a new helm3 data source +func New(name string, configVar string) ds.DataSourceWithLifecycle { + return &helm3Source{ + name: name, + configVar: configVar, + } +} + +// Name implements the interface method +func (d *helm3Source) Name() string { + return d.name +} + +// Init implements the interface method. +func (d *helm3Source) Init(p datasource.ConfigProvider) (fErr error) { + defer func() { + fErr = errors.Wrapf(fErr, "init data source %s", d.name) // nil wraps as nil + }() + cfgJSON, err := p(d.configVar) + if err != nil { + return err + } + var c Config + err = json.Unmarshal([]byte(cfgJSON), &c) + if err != nil { + return err + } + c.initDefaults() + err = c.assertValid() + if err != nil { + return err + } + d.cp = p + d.config = c + return nil +} + +// Resolve implements the interface method. +func (d *helm3Source) Resolve(path string) (_ string, finalErr error) { + u, err := url.Parse(path) + if err != nil { + return "", errors.Wrapf(err, "parse path %q", path) + } + configVar := u.Query().Get(configVarParam) + if configVar == "" { + return "", fmt.Errorf("%s query param not set in data source path %q", configVarParam, path) + } + u.Query().Del(configVarParam) + str, err := d.cp(configVar) + if err != nil { + return "", errors.Wrapf(err, "get ext code variable %s", configVar) + } + var tc TemplateConfig + err = json.Unmarshal([]byte(str), &tc) + if err != nil { + return "", errors.Wrapf(err, "json unmarshal of %s value", configVar) + } + if tc.Options.Namespace == "" { + if defaultNamespaceVar == "" { + return "", fmt.Errorf("namespace option not specified and no default value exists") + } + ns, err := d.cp(defaultNamespaceVar) + if err != nil { + return "", errors.Wrapf(err, "get default namespace from %s", defaultNamespaceVar) + } + tc.Options.Namespace = ns + } + out, err := d.runTemplate(u, tc) + if err != nil { + return "", err + } + b, err := json.Marshal(out) + if err != nil { + return "", errors.Wrap(err, "marshal output") + } + return string(b), nil +} + +func (d *helm3Source) runTemplate(u *url.URL, tc TemplateConfig) (interface{}, error) { + path := strings.TrimPrefix(u.Path, "/") + chart := path + if tc.Options.Repo == "" { // then assume path is a URL with https scheme + parts := strings.SplitN(path, "/", 2) // first component of part is actually the host + if len(parts) == 1 { + return nil, fmt.Errorf("unable to extract host and path from %s", path) + } + u.Host = parts[0] + u.Path = "/" + parts[1] + u.Scheme = "https" + chart = u.String() + } + b, err := json.Marshal(tc.Values) + if err != nil { + return "", errors.Wrap(err, "marshal values") + } + args := append([]string{"template", "--debug"}, tc.Options.toCommandLine()...) + if tc.Name != "" { // TODO: figure out if omitting name is the correct strategy + args = append(args, tc.Name) + } + args = append(args, chart) + args = append(args, "--values", "-") + + ctx, cancel := context.WithTimeout(context.Background(), d.config.timeout) + defer cancel() + + displayCommand := fmt.Sprintf("%s template %s %s %s", d.config.Command, tc.Options.toDisplay(), tc.Name, chart) + sio.Debugln(displayCommand) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, d.config.Command, args...) + cmd.Stdin = bytes.NewBuffer(b) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { + sio.Warnf("%s\n%s\n", "debug output from helm", stdout.String()) + return "", fmt.Errorf("%s\n%s", err.Error(), stderr.String()) + } + docs, err := natives.ParseYAMLDocuments(bytes.NewReader(stdout.Bytes())) + if err != nil { + return "", err + } + return docs, nil +} + +// Close implements the interface method. +func (d *helm3Source) Close() error { + return nil +} diff --git a/vm/internal/ds/helm3/helm3_test.go b/vm/internal/ds/helm3/helm3_test.go new file mode 100644 index 00000000..35540ac6 --- /dev/null +++ b/vm/internal/ds/helm3/helm3_test.go @@ -0,0 +1,210 @@ +/* + Copyright 2021 Splunk Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package helm3 + +import ( + "encoding/json" + "io/ioutil" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTemplateOptionsEmpty(t *testing.T) { + tc := TemplateOptions{} + parts := tc.toCommandLine() + require.Equal(t, 0, len(parts)) +} + +func TestTemplateOptionsFull(t *testing.T) { + tc := TemplateOptions{ + Namespace: "default", + CreateNamespace: true, + APIVersions: []string{"1.2", "2.1"}, + DependencyUpdate: true, + Description: "foo bar", + Devel: true, + DisableOpenAPIValidation: true, + GenerateName: true, + IncludeCRDs: true, + IsUpgrade: true, + SkipCRDs: true, + SkipTests: true, + InsecureSkipTLSVerify: true, + KubeVersion: "1.16", + NameTemplate: "gentemp", + NoHooks: true, + PassCredentials: true, + Password: "foobar", + RenderSubchartNotes: true, + Replace: true, + Repo: "r1", + ShowOnly: []string{"foo", "bar"}, + Username: "me", + Validate: true, + Verify: true, + Version: "1.2.3", + } + ret := tc.toCommandLine() + a := assert.New(t) + a.Contains(ret, "--namespace=default") + a.Contains(ret, "--api-versions=1.2") + a.Contains(ret, "--api-versions=2.1") + a.Contains(ret, "--kube-version=1.16") + a.Contains(ret, "--name-template=gentemp") + a.Contains(ret, "--password=foobar") + a.Contains(ret, "--repo=r1") + a.Contains(ret, "--show-only=foo") + a.Contains(ret, "--show-only=bar") + a.Contains(ret, "--username=me") + a.Contains(ret, "--version=1.2.3") + for _, s := range []string{ + "create-namespace", + "dependency-update", + "devel", + "disable-openapi-validation", + "generate-name", + "include-crds", + "is-upgrade", + "skip-crds", + "skip-tests", + "insecure-skip-tls-verify", + "no-hooks", + "pass-credentials", + "render-subchart-notes", + "replace", + "validate", + "verify", + } { + a.Contains(ret, "--"+s) + } + + str := tc.toDisplay() + a.Contains(str, "--namespace=default") + a.Contains(str, "--api-versions=1.2") + a.Contains(str, "--api-versions=2.1") + a.Contains(str, "--kube-version=1.16") + a.Contains(str, "--name-template=gentemp") + a.Contains(str, "--password=") + a.Contains(str, "--repo=r1") + a.Contains(str, "--show-only=foo") + a.Contains(str, "--show-only=bar") + a.Contains(str, "--username=me") + a.Contains(str, "--version=1.2.3") + for _, s := range []string{ + "create-namespace", + "dependency-update", + "devel", + "disable-openapi-validation", + "generate-name", + "include-crds", + "is-upgrade", + "skip-crds", + "skip-tests", + "insecure-skip-tls-verify", + "no-hooks", + "pass-credentials", + "render-subchart-notes", + "replace", + "validate", + "verify", + } { + a.Contains(str, "--"+s) + } +} + +func TestRunTemplate(t *testing.T) { + + mockConfigProvider := func(varName string) (string, error) { + if varName == "config-var-name" { + return `{"command": "helm", "timeout": "30s"}`, nil + } + return "", nil + } + + mockTemplateConfig := TemplateConfig{ + Name: "mock-release", + Options: TemplateOptions{ + Repo: "https://charts.bitnami.com/bitnami", + Namespace: "foobar", + Version: "11.2.17", + }, + Values: map[string]interface{}{ + "key": "value", + }, + } + + helm3Src := &helm3Source{configVar: "config-var-name"} + err := helm3Src.Init(mockConfigProvider) + require.NoError(t, err) + mockURL := &url.URL{ + Scheme: "http", + Host: "", + Path: "apache", + } + templated, err := helm3Src.runTemplate(mockURL, mockTemplateConfig) + b, err := json.MarshalIndent(templated, "", " ") + require.NoError(t, err) + filePath := filepath.Join("testdata", "apache.json") + fileContent, err := ioutil.ReadFile(filePath) + templatedStr := string(b) + templatedUnixStr := strings.ReplaceAll(templatedStr, "\r\n", "\n") + fileContentStr := string(fileContent) + fileContentUnixStr := strings.ReplaceAll(fileContentStr, "\r\n", "\n") + require.NoError(t, err) + require.Equal(t, templatedUnixStr, fileContentUnixStr) +} + +func TestInitDefaults(t *testing.T) { + cfg := Config{} + cfg.initDefaults() + require.Equal(t, cfg.Command, "helm") + require.Equal(t, cfg.Timeout, "") + require.Equal(t, cfg.timeout, time.Minute) +} + +func TestFindExecutable(t *testing.T) { + cmd, err := findExecutable("helm") + require.NoError(t, err) + require.FileExists(t, cmd) +} + +func TestClose(t *testing.T) { + helm3Src := &helm3Source{configVar: "config-var-name"} + err := helm3Src.Close() + require.NoError(t, err) +} + +func TestName(t *testing.T) { + mockConfigProvider := func(varName string) (string, error) { + if varName == "config-var-name" { + return `{"command": "helm", "timeout": "5s"}`, nil + } + return "", nil + } + + helm3Src := &helm3Source{name: "baz", configVar: "config-var-name"} + err := helm3Src.Init(mockConfigProvider) + require.NoError(t, err) + name := helm3Src.Name() + require.Equal(t, name, "baz") +} diff --git a/vm/internal/ds/helm3/testdata/apache.json b/vm/internal/ds/helm3/testdata/apache.json new file mode 100644 index 00000000..8c610b5a --- /dev/null +++ b/vm/internal/ds/helm3/testdata/apache.json @@ -0,0 +1,358 @@ +[ + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "app.kubernetes.io/version": "2.4.62", + "helm.sh/chart": "apache-11.2.17" + }, + "name": "mock-release-apache", + "namespace": "foobar" + }, + "spec": { + "egress": [ + {} + ], + "ingress": [ + { + "ports": [ + { + "port": 8080 + }, + { + "port": 8443 + } + ] + } + ], + "podSelector": { + "matchLabels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + } + }, + "policyTypes": [ + "Ingress", + "Egress" + ] + } + }, + { + "apiVersion": "policy/v1beta1", + "kind": "PodDisruptionBudget", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "app.kubernetes.io/version": "2.4.62", + "helm.sh/chart": "apache-11.2.17" + }, + "name": "mock-release-apache", + "namespace": "foobar" + }, + "spec": { + "maxUnavailable": 1, + "selector": { + "matchLabels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + } + } + } + }, + { + "apiVersion": "v1", + "automountServiceAccountToken": false, + "kind": "ServiceAccount", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "app.kubernetes.io/version": "2.4.62", + "helm.sh/chart": "apache-11.2.17" + }, + "name": "mock-release-apache", + "namespace": "foobar" + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "app.kubernetes.io/version": "2.4.62", + "helm.sh/chart": "apache-11.2.17" + }, + "name": "mock-release-apache", + "namespace": "foobar" + }, + "spec": { + "externalTrafficPolicy": "Cluster", + "loadBalancerSourceRanges": [], + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": "http" + }, + { + "name": "https", + "port": 443, + "targetPort": "https" + } + ], + "selector": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + }, + "sessionAffinity": "None", + "type": "LoadBalancer" + } + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "app.kubernetes.io/version": "2.4.62", + "helm.sh/chart": "apache-11.2.17" + }, + "name": "mock-release-apache", + "namespace": "foobar" + }, + "spec": { + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + } + }, + "strategy": { + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "app.kubernetes.io/version": "2.4.62", + "helm.sh/chart": "apache-11.2.17" + } + }, + "spec": { + "affinity": { + "nodeAffinity": null, + "podAffinity": null, + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + } + }, + "topologyKey": "kubernetes.io/hostname" + }, + "weight": 1 + } + ] + } + }, + "automountServiceAccountToken": false, + "containers": [ + { + "env": [ + { + "name": "BITNAMI_DEBUG", + "value": "false" + }, + { + "name": "APACHE_HTTP_PORT_NUMBER", + "value": "8080" + }, + { + "name": "APACHE_HTTPS_PORT_NUMBER", + "value": "8443" + } + ], + "envFrom": null, + "image": "docker.io/bitnami/apache:2.4.62-debian-12-r8", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 6, + "initialDelaySeconds": 180, + "periodSeconds": 20, + "successThreshold": 1, + "tcpSocket": { + "port": "http" + }, + "timeoutSeconds": 5 + }, + "name": "apache", + "ports": [ + { + "containerPort": 8080, + "name": "http" + }, + { + "containerPort": 8443, + "name": "https" + } + ], + "readinessProbe": { + "failureThreshold": 6, + "httpGet": { + "path": "/", + "port": "http" + }, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "cpu": "150m", + "ephemeral-storage": "2Gi", + "memory": "192Mi" + }, + "requests": { + "cpu": "100m", + "ephemeral-storage": "50Mi", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "privileged": false, + "readOnlyRootFilesystem": true, + "runAsGroup": 1001, + "runAsNonRoot": true, + "runAsUser": 1001, + "seLinuxOptions": {}, + "seccompProfile": { + "type": "RuntimeDefault" + } + }, + "volumeMounts": [ + { + "mountPath": "/opt/bitnami/apache/conf", + "name": "empty-dir", + "subPath": "app-conf-dir" + }, + { + "mountPath": "/opt/bitnami/apache/logs", + "name": "empty-dir", + "subPath": "app-logs-dir" + }, + { + "mountPath": "/tmp", + "name": "empty-dir", + "subPath": "tmp-dir" + }, + { + "mountPath": "/opt/bitnami/apache/var/run", + "name": "empty-dir", + "subPath": "app-tmp-dir" + } + ] + } + ], + "hostAliases": [ + { + "hostnames": [ + "status.localhost" + ], + "ip": "127.0.0.1" + } + ], + "initContainers": [ + { + "args": [ + "-ec", + "#!/bin/bash\n\n. /opt/bitnami/scripts/libfs.sh\n# We copy the logs folder because it has symlinks to stdout and stderr\nif ! is_dir_empty /opt/bitnami/apache/logs; then\n cp -r /opt/bitnami/apache/logs /emptydir/app-logs-dir\nfi\n" + ], + "command": [ + "/bin/bash" + ], + "image": "docker.io/bitnami/apache:2.4.62-debian-12-r8", + "imagePullPolicy": "IfNotPresent", + "name": "preserve-logs-symlinks", + "resources": { + "limits": { + "cpu": "150m", + "ephemeral-storage": "2Gi", + "memory": "192Mi" + }, + "requests": { + "cpu": "100m", + "ephemeral-storage": "50Mi", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "privileged": false, + "readOnlyRootFilesystem": true, + "runAsGroup": 1001, + "runAsNonRoot": true, + "runAsUser": 1001, + "seLinuxOptions": {}, + "seccompProfile": { + "type": "RuntimeDefault" + } + }, + "volumeMounts": [ + { + "mountPath": "/emptydir", + "name": "empty-dir" + } + ] + } + ], + "priorityClassName": "", + "securityContext": { + "fsGroup": 1001, + "fsGroupChangePolicy": "Always", + "supplementalGroups": [], + "sysctls": [] + }, + "serviceAccountName": "mock-release-apache", + "volumes": [ + { + "emptyDir": {}, + "name": "empty-dir" + } + ] + } + } + } + } +] \ No newline at end of file