deploy app using kots run (first pass)

cmd/kots/cli/run.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/mholt/archiver"
 	"github.com/replicatedhq/kots/pkg/apiserver"
 	"github.com/replicatedhq/kots/pkg/cluster"
 	"github.com/replicatedhq/kots/pkg/filestore"
@@ -63,24 +64,30 @@ func RunCmd() *cobra.Command {
 			// this is here to ensure that the store is initialized before we spawn kots and kubernetes at the same time, which
 			// might both try to initialize the store.
 			_ = persistence.MustGetDBSession()
+			persistence.SQLiteURI = fmt.Sprintf("%s/kots.db", v.GetString("data-dir")) // initialize here as well for the Authnz server to be able to use the store before kots comes up
 
-			// stat the kots api (aka, kotsadm in a former world)
-			if err := startKotsadm(ctx, v.GetString("data-dir"), v.GetString("shared-password")); err != nil {
-				return err
+			// ensure data dir exist
+			if _, err := os.Stat(v.GetString("data-dir")); os.IsNotExist(err) {
+				if err := os.MkdirAll(v.GetString("data-dir"), 0755); err != nil {
+					return err
+				}
 			}
 
 			if err := startK8sAuthnz(ctx, v.GetString("data-dir")); err != nil {
 				return err
 			}
 
-			// ensure data dir exist
-			if _, err := os.Stat(v.GetString("data-dir")); os.IsNotExist(err) {
-				if err := os.MkdirAll(v.GetString("data-dir"), 0755); err != nil {
-					return err
-				}
+			kubeconfigPath, err := cluster.Start(ctx, slug, v.GetString("data-dir"))
+			if err != nil {
+				return err
 			}
 
-			if err := cluster.Start(ctx, slug, v.GetString("data-dir")); err != nil {
+			if err := ensureBinaries(v.GetString("data-dir")); err != nil {
+				return err
+			}
+
+			// start the kots api (aka, kotsadm in a former world)
+			if err := startKotsadm(ctx, v.GetString("data-dir"), v.GetString("shared-password"), kubeconfigPath); err != nil {
 				return err
 			}
 
@@ -108,7 +115,7 @@ func RunCmd() *cobra.Command {
 	return cmd
 }
 
-func startKotsadm(ctx context.Context, dataDir string, sharedPassword string) error {
+func startKotsadm(ctx context.Context, dataDir string, sharedPassword string, kubeconfigPath string) error {
 	filestore.ArchivesDir = filepath.Join(dataDir, "archives")
 
 	// TODO @divolgin: something is odd about this pattern. these variables are set in two places to two different values, yet they are global.
@@ -121,6 +128,8 @@ func startKotsadm(ctx context.Context, dataDir string, sharedPassword string) er
 		AutocreateClusterToken: "TODO", // this needs to be static for an install, but different per installation
 		EnableIdentity:         false,
 		SharedPassword:         sharedPassword,
+		KubeconfigPath:         kubeconfigPath,
+		KotsDataDir:            dataDir,
 	}
 
 	go apiserver.Start(&params)
@@ -133,3 +142,65 @@ func startK8sAuthnz(ctx context.Context, dataDir string) error {
 
 	return nil
 }
+
+func ensureBinaries(dataDir string) error {
+	binariesRoot := filepath.Join(dataDir, "binaries")
+	if _, err := os.Stat(binariesRoot); os.IsNotExist(err) {
+		if err := os.MkdirAll(binariesRoot, 0755); err != nil {
+			return err
+		}
+	}
+
+	if err := ensureKubectlBinary(binariesRoot); err != nil {
+		return err
+	}
+
+	if err := ensureKustomizeBinary(binariesRoot); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ensureKubectlBinary(rootDir string) error {
+	kubectlFilePath := filepath.Join(rootDir, "kubectl")
+	if err := downloadFileFromURL(kubectlFilePath, "https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl"); err != nil {
+		return err
+	}
+
+	if err := os.Chmod(kubectlFilePath, 0755); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ensureKustomizeBinary(rootDir string) error {
+	kustomizeArchive := filepath.Join(rootDir, "kustomize.tar.gz")
+	if err := downloadFileFromURL(kustomizeArchive, "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.5.4/kustomize_v3.5.4_linux_amd64.tar.gz"); err != nil {
+		return err
+	}
+	defer os.RemoveAll(kustomizeArchive)
+
+	unarchived, err := ioutil.TempDir("", "kustomize")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(unarchived)
+
+	tarGz := archiver.TarGz{
+		Tar: &archiver.Tar{
+			ImplicitTopLevelFolder: false,
+		},
+	}
+	if err := tarGz.Unarchive(kustomizeArchive, unarchived); err != nil {
+		return err
+	}
+
+	err = os.Rename(filepath.Join(unarchived, "kustomize"), filepath.Join(rootDir, "kustomize3.5.4"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/kots/cli/util.go

@@ -1,6 +1,8 @@
 package cli
 
 import (
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -31,3 +33,28 @@ func homeDir() string {
 	}
 	return os.Getenv("USERPROFILE")
 }
+
+func downloadFileFromURL(destination string, url string) error {
+	out, err := os.Create(destination)
+	if err != nil {
+		return errors.Wrap(err, "failed to create file")
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return errors.Wrap(err, "failed to http get")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "failed to copy to file")
+	}
+
+	return nil
+}

pkg/apiserver/server.go

@@ -36,11 +36,22 @@ type APIServerParams struct {
 	AutocreateClusterToken string
 	EnableIdentity         bool
 	SharedPassword         string
+	KubeconfigPath         string
+	KotsDataDir            string
 }
 
 func Start(params *APIServerParams) {
 	log.Printf("kotsadm version %s\n", params.Version)
 
+	if params.KubeconfigPath != "" {
+		// it's only possible to set this in the kots run workflow
+		os.Setenv("KUBECONFIG", params.KubeconfigPath)
+	}
+	if params.KotsDataDir != "" {
+		// it's only possible to set this in the kots run workflow
+		os.Setenv("KOTS_DATA_DIR", params.KotsDataDir)
+	}
+
 	// set some persistence variables
 	persistence.PostgresURI = params.PostgresURI
 	persistence.SQLiteURI = params.SQLiteURI
@@ -74,7 +85,6 @@ func Start(params *APIServerParams) {
 		if err != nil {
 			panic(err)
 		}
-
 		os.Setenv("SHARED_PASSWORD_BCRYPT", string(bcryptPassword))
 	}
 

pkg/cluster/antrea.go

@@ -52,23 +52,23 @@ func installCNI(kubeconfigPath string) error {
 		obj := &unstructured.Unstructured{}
 		_, gvk, err := decUnstructured.Decode(buf, nil, obj)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "decode doc")
 		}
 
 		dc, err := discovery.NewDiscoveryClientForConfig(config)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "new discovery client for config")
 		}
 		mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc))
 
 		dyn, err := dynamic.NewForConfig(config)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "new dynamic config")
 		}
 
 		mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "rest mapping")
 		}
 
 		var dr dynamic.ResourceInterface
@@ -82,19 +82,15 @@ func installCNI(kubeconfigPath string) error {
 
 		data, err := json.Marshal(obj)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "json marshal")
 		}
 
 		_, err = dr.Patch(context.Background(), obj.GetName(), types.ApplyPatchType, data, metav1.PatchOptions{
 			FieldManager: "kots",
 		})
-
 		if err != nil {
-			return err
+			return errors.Wrap(err, "patch")
 		}
-
-		return nil
-
 	}
 }
 

pkg/cluster/cluster.go

@@ -10,52 +10,53 @@ import (
 
 // Start will start the embedded cluster.
 // This function blocks until the cluster control plane has started
-func Start(ctx context.Context, slug string, dataDir string) error {
+func Start(ctx context.Context, slug string, dataDir string) (string, error) {
 	log := ctx.Value("log").(*logger.CLILogger)
 	log.ActionWithoutSpinner("Starting cluster")
 
 	// init tls and misc
 	// this function is synchronous and blocks until ready
 	if err := clusterInit(ctx, dataDir, slug, "1.21.3"); err != nil {
-		return errors.Wrap(err, "init cluster")
+		return "", errors.Wrap(err, "init cluster")
 	}
 
 	// start the api server
 	if err := runAPIServer(ctx, dataDir, slug); err != nil {
-		return errors.Wrap(err, "start api server")
+		return "", errors.Wrap(err, "start api server")
 	}
 
 	// start the scheduler on port 11251 (this is a non-standard port)
 	if err := runScheduler(ctx, dataDir); err != nil {
-		return errors.Wrap(err, "start scheduler")
+		return "", errors.Wrap(err, "start scheduler")
 	}
 
 	// start the controller manager on port 11252 (non standard)
 	// TODO the controller should start
 	if err := runController(ctx, dataDir); err != nil {
-		return errors.Wrap(err, "start controller")
+		return "", errors.Wrap(err, "start controller")
 	}
 
 	// because these are all synchoronous, the api is ready and we
 	// can install our addons
 	kubeconfigPath, err := kubeconfigFilePath(dataDir)
 	if err != nil {
-		return errors.Wrap(err, "get kubeconfig path")
+		return "", errors.Wrap(err, "get kubeconfig path")
 	}
 
+	fmt.Println("installing cni")
 	if err := installCNI(kubeconfigPath); err != nil {
-		return errors.Wrap(err, "install antrea")
+		return "", errors.Wrap(err, "install antrea")
 	}
 
 	fmt.Println("starting cri")
 	if err := startCRI(dataDir); err != nil {
-		return errors.Wrap(err, "install cri")
+		return "", errors.Wrap(err, "install cri")
 	}
 
 	fmt.Println("starting kubelet")
 	if err := startKubelet(ctx, dataDir); err != nil {
-		return errors.Wrap(err, "install kubelet")
+		return "", errors.Wrap(err, "install kubelet")
 	}
 
-	return nil
+	return kubeconfigPath, nil
 }

pkg/cluster/containerd.go

@@ -334,33 +334,3 @@ func writeContainerdConfig(installDir string) error {
 
 	return nil
 }
-
-// generateDefaultConfig will run containerd and output the config.toml for a default installation
-// this is not currently used
-func generateDefaultConfig(dataDir string, installDir string) error {
-	cmd := exec.Command(filepath.Join(installDir, "bin", "containerd"), "config", "default")
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return errors.Wrap(err, "exec containerd config default")
-	}
-
-	configFile := filepath.Join(installDir, "config.toml")
-	if _, err := os.Stat(configFile); err == nil {
-		if err := os.RemoveAll(configFile); err != nil {
-			return errors.Wrap(err, "remove containerd config")
-		}
-	}
-
-	d := filepath.Dir(configFile)
-	if _, err := os.Stat(d); os.IsNotExist(err) {
-		if err := os.MkdirAll(d, 0755); err != nil {
-			return errors.Wrap(err, "mkdir")
-		}
-	}
-
-	if err := ioutil.WriteFile(configFile, out, 0644); err != nil {
-		return errors.Wrap(err, "write containerd config")
-	}
-
-	return nil
-}

pkg/cluster/controller.go

@@ -50,7 +50,7 @@ func runController(ctx context.Context, dataDir string) error {
 		"--use-service-account-credentials=true",
 		"--v=2",
 		fmt.Sprintf("--flex-volume-plugin-dir=%s", dataDir),
-		"--controllers=-service,-route,-cloud-node-lifecycle", // This is equivalent to using ExternalLoops in the ControllerLoopMode type, which excludes cloud specific controlers
+		"--controllers=*,-service,-route,-cloud-node-lifecycle", // This is equivalent to using ExternalLoops in the ControllerLoopMode type, which excludes cloud specific controlers
 	}
 
 	command := app.NewControllerManagerCommand()

pkg/cluster/init.go

@@ -63,7 +63,7 @@ func clusterInit(ctx context.Context, dataDir string, slug string, version strin
 
 	serviceAccountKeyFile, err := ensureServiceAccountKeyFile(dataDir)
 	if err != nil {
-		return errors.Wrap(err, "ensuire service account key file")
+		return errors.Wrap(err, "ensure service account key file")
 	}
 	logger.Info("verified service account key file",
 		zap.String("serviceAccountKeyFile", serviceAccountKeyFile))
@@ -175,8 +175,8 @@ func ensureKubeconfigFile(slug string, dataDir string) (string, error) {
 		if err := ioutil.WriteFile(kubeconfigFile, kubeconfigBytes, 0600); err != nil {
 			return "", errors.Wrap(err, "write file")
 		}
-
 	}
+
 	return kubeconfigFile, nil
 }
 

pkg/gitops/gitops.go

@@ -25,6 +25,7 @@ import (
 	"github.com/replicatedhq/kots/pkg/k8sutil"
 	"github.com/replicatedhq/kots/pkg/kotsadm/types"
 	"github.com/replicatedhq/kots/pkg/kotsutil"
+	"github.com/replicatedhq/kots/pkg/kustomize"
 	"github.com/replicatedhq/kots/pkg/util"
 	"golang.org/x/crypto/ssh"
 	v1 "k8s.io/api/core/v1"
@@ -625,7 +626,7 @@ func CreateGitOpsCommit(gitOpsConfig *GitOpsConfig, appSlug string, appName stri
 	}
 
 	// we use the kustomize binary here...
-	cmd := exec.Command(fmt.Sprintf("kustomize%s", kotsKinds.KustomizeVersion()), "build", filepath.Join(archiveDir, "overlays", "downstreams", downstreamName))
+	cmd := exec.Command(kustomize.GetKustomizePath(kotsKinds.KustomizeVersion()), "build", filepath.Join(archiveDir, "overlays", "downstreams", downstreamName))
 	out, err := cmd.Output()
 	if err != nil {
 		if ee, ok := err.(*exec.ExitError); ok {