From f2a6d281a76c4c436caa9a95ee65dfdd6a236c2d Mon Sep 17 00:00:00 2001 From: Cole Snodgrass Date: Thu, 25 Jul 2024 13:24:51 -0700 Subject: [PATCH] feat: add support for --secret (#60) --- internal/cmd/local/check.go | 10 ++-- internal/cmd/local/helm/helm.go | 2 +- internal/cmd/local/k8s/client.go | 19 ++---- internal/cmd/local/local/cmd.go | 83 ++++++++++++++++++++------- internal/cmd/local/local/cmd_test.go | 10 ++-- internal/cmd/local/local_install.go | 41 +++++++------ internal/cmd/local/local_status.go | 6 +- internal/cmd/local/local_uninstall.go | 2 +- 8 files changed, 104 insertions(+), 69 deletions(-) diff --git a/internal/cmd/local/check.go b/internal/cmd/local/check.go index bd85a59..91f188c 100644 --- a/internal/cmd/local/check.go +++ b/internal/cmd/local/check.go @@ -25,14 +25,14 @@ func dockerInstalled(ctx context.Context) (docker.Version, error) { var err error if dockerClient == nil { if dockerClient, err = docker.New(ctx); err != nil { - pterm.Error.Println("Could not create Docker client") - return docker.Version{}, fmt.Errorf("%w: could not create client: %w", localerr.ErrDocker, err) + pterm.Error.Println("Unable to create Docker client") + return docker.Version{}, fmt.Errorf("%w: unable to create client: %w", localerr.ErrDocker, err) } } version, err := dockerClient.Version(ctx) if err != nil { - pterm.Error.Println("Could not communicate with the Docker daemon") + pterm.Error.Println("Unable to communicate with the Docker daemon") return docker.Version{}, fmt.Errorf("%w: %w", localerr.ErrDocker, err) } pterm.Success.Println(fmt.Sprintf("Found Docker installation: version %s", version.Version)) @@ -72,13 +72,13 @@ func portAvailable(ctx context.Context, port int) error { req, errInner := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:%d", port), nil) if errInner != nil { pterm.Error.Printfln("Port %d request could not be created", port) - return fmt.Errorf("%w: could not create request: %w", localerr.ErrPort, err) + return fmt.Errorf("%w: unable to create request: %w", localerr.ErrPort, err) } res, errInner := httpClient.Do(req) if errInner != nil { pterm.Error.Printfln("Port %d appears to already be in use", port) - return fmt.Errorf("%w: could not send request: %w", localerr.ErrPort, err) + return fmt.Errorf("%w: unable to send request: %w", localerr.ErrPort, err) } if res.StatusCode == http.StatusUnauthorized && strings.Contains(res.Header.Get("WWW-Authenticate"), "abctl") { diff --git a/internal/cmd/local/helm/helm.go b/internal/cmd/local/helm/helm.go index 7522584..1be90ee 100644 --- a/internal/cmd/local/helm/helm.go +++ b/internal/cmd/local/helm/helm.go @@ -32,7 +32,7 @@ func New(kubecfg, kubectx, namespace string) (Client, error) { restCfg, err := k8sCfg.ClientConfig() if err != nil { - return nil, fmt.Errorf("%w: could not create rest config: %w", localerr.ErrKubernetes, err) + return nil, fmt.Errorf("%w: unable to create rest config: %w", localerr.ErrKubernetes, err) } logger := helmLogger{} diff --git a/internal/cmd/local/k8s/client.go b/internal/cmd/local/k8s/client.go index 558f64a..1bb766e 100644 --- a/internal/cmd/local/k8s/client.go +++ b/internal/cmd/local/k8s/client.go @@ -50,7 +50,7 @@ type Client interface { PersistentVolumeClaimDelete(ctx context.Context, namespace, name, volumeName string) error // SecretCreateOrUpdate will update or create the secret name with the payload of data in the specified namespace - SecretCreateOrUpdate(ctx context.Context, secretType corev1.SecretType, namespace, name string, data map[string][]byte) error + SecretCreateOrUpdate(ctx context.Context, secret corev1.Secret) error // ServiceGet returns a the service for the given namespace and name ServiceGet(ctx context.Context, namespace, name string) (*corev1.Service, error) @@ -175,20 +175,13 @@ func (d *DefaultK8sClient) PersistentVolumeClaimDelete(ctx context.Context, name return d.ClientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, name, metav1.DeleteOptions{}) } -func (d *DefaultK8sClient) SecretCreateOrUpdate(ctx context.Context, secretType corev1.SecretType, namespace, name string, data map[string][]byte) error { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Data: data, - Type: secretType, - } +func (d *DefaultK8sClient) SecretCreateOrUpdate(ctx context.Context, secret corev1.Secret) error { + namespace := secret.ObjectMeta.Namespace + name := secret.ObjectMeta.Name _, err := d.ClientSet.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) if err == nil { // update - if _, err := d.ClientSet.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + if _, err := d.ClientSet.CoreV1().Secrets(namespace).Update(ctx, &secret, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("unable to update the secret %s: %w", name, err) } @@ -196,7 +189,7 @@ func (d *DefaultK8sClient) SecretCreateOrUpdate(ctx context.Context, secretType } if k8serrors.IsNotFound(err) { - if _, err := d.ClientSet.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err := d.ClientSet.CoreV1().Secrets(namespace).Create(ctx, &secret, metav1.CreateOptions{}); err != nil { return fmt.Errorf("unable to create the secret %s: %w", name, err) } diff --git a/internal/cmd/local/local/cmd.go b/internal/cmd/local/local/cmd.go index f9153e2..1e79a5c 100644 --- a/internal/cmd/local/local/cmd.go +++ b/internal/cmd/local/local/cmd.go @@ -8,6 +8,7 @@ import ( "github.com/airbytehq/abctl/internal/cmd/local/migrate" "github.com/airbytehq/abctl/internal/cmd/local/paths" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/yaml" "net/http" "os" "path/filepath" @@ -186,7 +187,7 @@ func New(provider k8s.Provider, opts ...Option) (*Command, error) { { k8sVersion, err := c.k8s.ServerVersionGet() if err != nil { - return nil, fmt.Errorf("%w: could not fetch kubernetes server version: %w", localerr.ErrKubernetes, err) + return nil, fmt.Errorf("%w: unable to fetch kubernetes server version: %w", localerr.ErrKubernetes, err) } c.tel.Attr("k8s_version", k8sVersion) } @@ -203,6 +204,7 @@ type InstallOpts struct { HelmChartVersion string ValuesFile string + Secrets []string Migrate bool Host string @@ -251,12 +253,12 @@ func (c *Command) persistentVolume(ctx context.Context, namespace, name string) pterm.Debug.Println(fmt.Sprintf("Creating directory '%s'", path)) if err := os.MkdirAll(path, 0766); err != nil { - pterm.Error.Println(fmt.Sprintf("Could not create directory '%s'", name)) + pterm.Error.Println(fmt.Sprintf("Unable to create directory '%s'", name)) return fmt.Errorf("unable to create persistent volume '%s': %w", name, err) } if err := c.k8s.PersistentVolumeCreate(ctx, namespace, name); err != nil { - pterm.Error.Println(fmt.Sprintf("Could not create persistent volume '%s'", name)) + pterm.Error.Println(fmt.Sprintf("Unable to create persistent volume '%s'", name)) return fmt.Errorf("unable to create persistent volume '%s': %w", name, err) } @@ -273,7 +275,7 @@ func (c *Command) persistentVolume(ctx context.Context, namespace, name string) // access to the persisted volume directory. pterm.Debug.Println(fmt.Sprintf("Updating permissions for '%s'", path)) if err := os.Chmod(path, 0777); err != nil { - pterm.Error.Println(fmt.Sprintf("Could not set permissions for '%s'", path)) + pterm.Error.Println(fmt.Sprintf("Unable to set permissions for '%s'", path)) return fmt.Errorf("unable to set permissions for '%s': %w", path, err) } @@ -289,7 +291,7 @@ func (c *Command) persistentVolumeClaim(ctx context.Context, namespace, name, vo if !c.k8s.PersistentVolumeClaimExists(ctx, namespace, name, volumeName) { c.spinner.UpdateText(fmt.Sprintf("Creating persistent volume claim '%s'", name)) if err := c.k8s.PersistentVolumeClaimCreate(ctx, namespace, name, volumeName); err != nil { - pterm.Error.Println(fmt.Sprintf("Could not create persistent volume claim '%s'", name)) + pterm.Error.Println(fmt.Sprintf("Unable to create persistent volume claim '%s'", name)) return fmt.Errorf("unable to create persistent volume claim '%s': %w", name, err) } pterm.Info.Println(fmt.Sprintf("Persistent volume claim '%s' created", name)) @@ -302,13 +304,13 @@ func (c *Command) persistentVolumeClaim(ctx context.Context, namespace, name, vo // Install handles the installation of Airbyte func (c *Command) Install(ctx context.Context, opts InstallOpts) error { - var values string + var vals string if opts.ValuesFile != "" { raw, err := os.ReadFile(opts.ValuesFile) if err != nil { return fmt.Errorf("unable to read values file '%s': %w", opts.ValuesFile, err) } - values = string(raw) + vals = string(raw) } go c.watchEvents(ctx) @@ -316,7 +318,7 @@ func (c *Command) Install(ctx context.Context, opts InstallOpts) error { if !c.k8s.NamespaceExists(ctx, airbyteNamespace) { c.spinner.UpdateText(fmt.Sprintf("Creating namespace '%s'", airbyteNamespace)) if err := c.k8s.NamespaceCreate(ctx, airbyteNamespace); err != nil { - pterm.Error.Println(fmt.Sprintf("Could not create namespace '%s'", airbyteNamespace)) + pterm.Error.Println(fmt.Sprintf("Unable to create namespace '%s'", airbyteNamespace)) return fmt.Errorf("unable to create airbyte namespace: %w", err) } pterm.Info.Println(fmt.Sprintf("Namespace '%s' created", airbyteNamespace)) @@ -362,13 +364,36 @@ func (c *Command) Install(ctx context.Context, opts InstallOpts) error { if opts.dockerAuth() { pterm.Debug.Println(fmt.Sprintf("Creating '%s' secret", dockerAuthSecretName)) if err := c.handleDockerSecret(ctx, opts.DockerServer, opts.DockerUser, opts.DockerPass, opts.DockerEmail); err != nil { - pterm.Debug.Println(fmt.Sprintf("Could not create '%s' secret", dockerAuthSecretName)) + pterm.Debug.Println(fmt.Sprintf("Unable to create '%s' secret", dockerAuthSecretName)) return fmt.Errorf("unable to create '%s' secret: %w", dockerAuthSecretName, err) } pterm.Debug.Println(fmt.Sprintf("Created '%s' secret", dockerAuthSecretName)) airbyteValues = append(airbyteValues, fmt.Sprintf("global.imagePullSecrets[0].name=%s", dockerAuthSecretName)) } + for _, secretFile := range opts.Secrets { + c.spinner.UpdateText(fmt.Sprintf("Creating secret from '%s'", secretFile)) + raw, err := os.ReadFile(secretFile) + if err != nil { + pterm.Error.Println(fmt.Sprintf("Unable to read secret file '%s': %s", secretFile, err)) + return fmt.Errorf("unable to read secret file '%s': %w", secretFile, err) + } + + var secret corev1.Secret + if err := yaml.Unmarshal(raw, &secret); err != nil { + pterm.Error.Println(fmt.Sprintf("Unable to unmarshal secret file '%s': %s", secretFile, err)) + return fmt.Errorf("unable to unmarshal secret file '%s': %w", secretFile, err) + } + secret.ObjectMeta.Namespace = airbyteNamespace + + if err := c.k8s.SecretCreateOrUpdate(ctx, secret); err != nil { + pterm.Error.Println(fmt.Sprintf("Unable to create secret from file '%s'", secretFile)) + return fmt.Errorf("unable to create secret from file '%s': %w", secretFile, err) + } + + pterm.Success.Println(fmt.Sprintf("Secret from '%s' created or updated", secretFile)) + } + if err := c.handleChart(ctx, chartRequest{ name: "airbyte", repoName: airbyteRepoName, @@ -378,7 +403,7 @@ func (c *Command) Install(ctx context.Context, opts InstallOpts) error { chartVersion: opts.HelmChartVersion, namespace: airbyteNamespace, values: airbyteValues, - valuesYAML: values, + valuesYAML: vals, }); err != nil { return fmt.Errorf("unable to install airbyte chart: %w", err) } @@ -545,9 +570,19 @@ func (c *Command) handleBasicAuthSecret(ctx context.Context, user, pass string) return fmt.Errorf("unable to hash basic auth password: %w", err) } - data := map[string][]byte{"auth": []byte(fmt.Sprintf("%s:%s", user, hashedPass))} - if err := c.k8s.SecretCreateOrUpdate(ctx, corev1.SecretTypeOpaque, airbyteNamespace, "basic-auth", data); err != nil { - pterm.Error.Println("Could not create Basic-Auth secret") + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: airbyteNamespace, + Name: "basic-auth", + }, + Data: map[string][]byte{"auth": []byte(fmt.Sprintf("%s:%s", user, hashedPass))}, + StringData: nil, + Type: corev1.SecretTypeOpaque, + } + + if err := c.k8s.SecretCreateOrUpdate(ctx, secret); err != nil { + pterm.Error.Println("Unable to create Basic-Auth secret") return fmt.Errorf("unable to create Basic-Auth secret: %w", err) } pterm.Success.Println("Basic-Auth secret created") @@ -555,16 +590,24 @@ func (c *Command) handleBasicAuthSecret(ctx context.Context, user, pass string) } func (c *Command) handleDockerSecret(ctx context.Context, server, user, pass, email string) error { - data := map[string][]byte{} - secret, err := docker.Secret(server, user, pass, email) + secretBody, err := docker.Secret(server, user, pass, email) if err != nil { pterm.Error.Println("Unable to create docker secret") return fmt.Errorf("unable to create docker secret: %w", err) } - data[corev1.DockerConfigJsonKey] = secret - if err := c.k8s.SecretCreateOrUpdate(ctx, corev1.SecretTypeDockerConfigJson, airbyteNamespace, dockerAuthSecretName, data); err != nil { - pterm.Error.Println("Could not create Docker-auth secret") + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: airbyteNamespace, + Name: dockerAuthSecretName, + }, + Data: map[string][]byte{corev1.DockerConfigJsonKey: secretBody}, + Type: corev1.SecretTypeDockerConfigJson, + } + + if err := c.k8s.SecretCreateOrUpdate(ctx, secret); err != nil { + pterm.Error.Println("Unable to create Docker-auth secret") return fmt.Errorf("unable to create docker-auth secret: %w", err) } pterm.Success.Println("Docker-Auth secret created") @@ -598,8 +641,8 @@ func (c *Command) Status(_ context.Context) error { rel, err := c.helm.GetRelease(name) if err != nil { - pterm.Warning.Println("Could not get airbyte release") - pterm.Debug.Printfln("unable to get airbyte release: %s", err) + pterm.Warning.Println("Unable to fetch airbyte release") + pterm.Debug.Printfln("unable to fetch airbyte release: %s", err) continue } diff --git a/internal/cmd/local/local/cmd_test.go b/internal/cmd/local/local/cmd_test.go index f83b8a7..38b3572 100644 --- a/internal/cmd/local/local/cmd_test.go +++ b/internal/cmd/local/local/cmd_test.go @@ -124,7 +124,7 @@ func TestCommand_Install(t *testing.T) { serverVersionGet: func() (string, error) { return "test", nil }, - secretCreateOrUpdate: func(ctx context.Context, secretType coreV1.SecretType, namespace, name string, data map[string][]byte) error { + secretCreateOrUpdate: func(ctx context.Context, secret coreV1.Secret) error { return nil }, ingressExists: func(ctx context.Context, namespace string, ingress string) bool { @@ -265,7 +265,7 @@ func TestCommand_Install_ValuesFile(t *testing.T) { serverVersionGet: func() (string, error) { return "test", nil }, - secretCreateOrUpdate: func(ctx context.Context, secretType coreV1.SecretType, namespace, name string, data map[string][]byte) error { + secretCreateOrUpdate: func(ctx context.Context, secret coreV1.Secret) error { return nil }, ingressExists: func(ctx context.Context, namespace string, ingress string) bool { @@ -384,7 +384,7 @@ type mockK8sClient struct { persistentVolumeClaimCreate func(ctx context.Context, namespace, name, volumeName string) error persistentVolumeClaimExists func(ctx context.Context, namespace, name, volumeName string) bool persistentVolumeClaimDelete func(ctx context.Context, namespace, name, volumeName string) error - secretCreateOrUpdate func(ctx context.Context, secretType coreV1.SecretType, namespace, name string, data map[string][]byte) error + secretCreateOrUpdate func(ctx context.Context, secret coreV1.Secret) error serviceGet func(ctx context.Context, namespace, name string) (*coreV1.Service, error) serverVersionGet func() (string, error) eventsWatch func(ctx context.Context, namespace string) (watch.Interface, error) @@ -471,9 +471,9 @@ func (m *mockK8sClient) PersistentVolumeClaimDelete(ctx context.Context, namespa return nil } -func (m *mockK8sClient) SecretCreateOrUpdate(ctx context.Context, secretType coreV1.SecretType, namespace, name string, data map[string][]byte) error { +func (m *mockK8sClient) SecretCreateOrUpdate(ctx context.Context, secret coreV1.Secret) error { if m.secretCreateOrUpdate != nil { - return m.secretCreateOrUpdate(ctx, secretType, namespace, name, data) + return m.secretCreateOrUpdate(ctx, secret) } return nil diff --git a/internal/cmd/local/local_install.go b/internal/cmd/local/local_install.go index a159d28..1271726 100644 --- a/internal/cmd/local/local_install.go +++ b/internal/cmd/local/local_install.go @@ -35,6 +35,7 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { flagBasicAuthPass string flagChartValuesFile string + flagChartSecrets []string flagChartVersion string flagMigrate bool flagPort int @@ -75,7 +76,7 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { cluster, err := provider.Cluster() if err != nil { - pterm.Error.Printfln("Could not determine status of any existing '%s' cluster", provider.ClusterName) + pterm.Error.Printfln("Unable to determine status of any existing '%s' cluster", provider.ClusterName) return err } @@ -89,7 +90,7 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { if dockerClient == nil { dockerClient, err = docker.New(cmd.Context()) if err != nil { - pterm.Error.Printfln("Could not connect to Docker daemon") + pterm.Error.Printfln("Unable to connect to Docker daemon") return fmt.Errorf("unable to connect to docker: %w", err) } } @@ -135,6 +136,7 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { BasicAuthPass: flagBasicAuthPass, HelmChartVersion: flagChartVersion, ValuesFile: flagChartValuesFile, + Secrets: flagChartSecrets, Migrate: flagMigrate, Docker: dockerClient, Host: flagHost, @@ -149,25 +151,12 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { opts.HelmChartVersion = "" } - if env := os.Getenv(envBasicAuthUser); env != "" { - opts.BasicAuthUser = env - } - if env := os.Getenv(envBasicAuthPass); env != "" { - opts.BasicAuthPass = env - } - - if env := os.Getenv(envDockerServer); env != "" { - opts.DockerServer = env - } - if env := os.Getenv(envDockerUser); env != "" { - opts.DockerUser = env - } - if env := os.Getenv(envDockerPass); env != "" { - opts.DockerPass = env - } - if env := os.Getenv(envDockerEmail); env != "" { - opts.DockerEmail = env - } + envOverride(&opts.BasicAuthUser, envBasicAuthUser) + envOverride(&opts.BasicAuthPass, envBasicAuthPass) + envOverride(&opts.DockerServer, envDockerServer) + envOverride(&opts.DockerUser, envDockerUser) + envOverride(&opts.DockerPass, envDockerPass) + envOverride(&opts.DockerEmail, envDockerEmail) if err := lc.Install(cmd.Context(), opts); err != nil { spinner.Fail("Unable to install Airbyte locally") @@ -189,6 +178,7 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { cmd.Flags().StringVar(&flagChartVersion, "chart-version", "latest", "specify the Airbyte helm chart version to install") cmd.Flags().StringVar(&flagChartValuesFile, "values", "", "the Airbyte helm chart values file to load") + cmd.Flags().StringSliceVar(&flagChartSecrets, "secret", []string{}, "an Airbyte helm chart secret file") cmd.Flags().BoolVar(&flagMigrate, "migrate", false, "migrate data from docker compose installation") cmd.Flags().StringVar(&flagDockerServer, "docker-server", "https://index.docker.io/v1/", "docker registry, can also be specified via "+envDockerServer) @@ -200,3 +190,12 @@ func NewCmdInstall(provider k8s.Provider) *cobra.Command { return cmd } + +// envOverride checks if the env exists and is not empty, if that is true +// update the original value to be the value returned from the env environment variable. +// Otherwise, leave the original value alone. +func envOverride(original *string, env string) { + if v := os.Getenv(env); v != "" { + *original = v + } +} diff --git a/internal/cmd/local/local_status.go b/internal/cmd/local/local_status.go index 7bd3d81..eec5455 100644 --- a/internal/cmd/local/local_status.go +++ b/internal/cmd/local/local_status.go @@ -38,7 +38,7 @@ func NewCmdStatus(provider k8s.Provider) *cobra.Command { cluster, err := provider.Cluster() if err != nil { - pterm.Error.Printfln("Could not determine status of any existing '%s' cluster", provider.ClusterName) + pterm.Error.Printfln("Unable to determine status of any existing '%s' cluster", provider.ClusterName) return err } @@ -56,14 +56,14 @@ func NewCmdStatus(provider k8s.Provider) *cobra.Command { if dockerClient == nil { dockerClient, err = docker.New(cmd.Context()) if err != nil { - pterm.Error.Printfln("Could not connect to Docker daemon") + pterm.Error.Printfln("Unable to connect to Docker daemon") return fmt.Errorf("unable to connect to docker: %w", err) } } port, err = dockerClient.Port(cmd.Context(), fmt.Sprintf("%s-control-plane", provider.ClusterName)) if err != nil { - pterm.Warning.Printfln("Could not determine docker port for cluster '%s'", provider.ClusterName) + pterm.Warning.Printfln("Unable to determine docker port for cluster '%s'", provider.ClusterName) return nil } } diff --git a/internal/cmd/local/local_uninstall.go b/internal/cmd/local/local_uninstall.go index 71df964..2476c6e 100644 --- a/internal/cmd/local/local_uninstall.go +++ b/internal/cmd/local/local_uninstall.go @@ -39,7 +39,7 @@ func NewCmdUninstall(provider k8s.Provider) *cobra.Command { cluster, err := provider.Cluster() if err != nil { - pterm.Error.Printfln("Could not determine if the cluster '%s' exists", provider.ClusterName) + pterm.Error.Printfln("Unable to determine if the cluster '%s' exists", provider.ClusterName) return err }