Deploy kind cluster with golang

Author: maru-ava

.github/workflows/ci.yml

@@ -89,10 +89,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-go-for-project
+      # TODO(marun) Encapsulate execution with nix and monitoring configuration in a custom action
+      - uses: ./.github/actions/install-nix
+      - run: nix develop --command echo "dependencies installed"
       - name: Run e2e tests
         shell: bash
         run: bash -x ./scripts/tests.e2e.kube.sh
         env:
+          TMPNET_CHECK_MONITORING: true
           PROMETHEUS_USERNAME: ${{ secrets.PROMETHEUS_ID || '' }}
           PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
           LOKI_USERNAME: ${{ secrets.LOKI_ID || '' }}

bin/tmpnetctl

@@ -10,4 +10,4 @@ cd "${AVALANCHE_PATH}"
 if [[ ! -f ./build/tmpnetctl ]]; then
    ./scripts/build_tmpnetctl.sh
 fi
-./build/tmpnetctl
+./build/tmpnetctl "${@}"

scripts/tests.e2e.bootstrap_monitor.sh

@@ -9,14 +9,6 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then
   exit 255
 fi
 
-CMD=kind-with-registry.sh
-
-if ! command -v "${CMD}" &> /dev/null; then
-  echo "kind-with-registry.sh not found, have you run 'nix develop'?"
-  echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix"
-  exit 1
-fi
-
-"${CMD}"
+./bin/tmpnetctl start-kind-cluster
 
 KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e

scripts/tests.e2e.kube.sh

@@ -6,15 +6,14 @@ set -euo pipefail
 
 # TODO(marun)
 # - Support testing against a remote cluster
-# - Convert to golang to simplify reuse
-# - Make idempotent to simplify development and debugging
+# - Convert deloyment of collectors to golang to simplify reuse
 
 if ! [[ "$0" =~ scripts/tests.e2e.kube.sh ]]; then
   echo "must be run from repository root"
   exit 255
 fi
 
-./scripts/ensure_kube_cluster.sh
+./bin/tmpnetctl start-kind-cluster
 
 KUBE_CONTEXT="${KUBE_CONTEXT:-kind-kind}"
 # TODO(marun) Make the namespace configurable

tests/e2e/c/dynamic_fees.go

@@ -60,7 +60,7 @@ var _ = e2e.DescribeCChain("[Dynamic Fees]", func() {
 		env := e2e.GetEnv(tc)
 		publicNetwork := env.GetNetwork()
 
-		privateNetwork := tmpnet.NewDefaultNetwork("avalanchego-e2e-dynamic-fees")
+		privateNetwork := tmpnet.NewDefaultNetwork(tc.Log(), "avalanchego-e2e-dynamic-fees")
 		// Copy over the defaults from the normal test suite to include settings
 		// like the upgrade config.
 		privateNetwork.DefaultFlags = tmpnet.FlagsMap{}

tests/fixture/tmpnet/cmd/main.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"go.uber.org/zap"
 
 	"github.com/ava-labs/avalanchego/tests"
@@ -59,10 +60,12 @@ func main() {
 		runtime         string
 		avalancheGoPath string
 		pluginDir       string
-		kubeconfig      string
-		namespace       string
-		imageName       string
-		nodeCount       uint8
+		// TODO(marun) Update to camel-case?
+		kubeconfig  string
+		kubeContext string
+		namespace   string
+		imageName   string
+		nodeCount   uint8
 	)
 	startNetworkCmd := &cobra.Command{
 		Use:   "start-network",
@@ -151,7 +154,7 @@ func main() {
 		tmpnet.GetEnvWithDefault(tmpnet.AvalancheGoPluginDirEnvName, os.ExpandEnv("$HOME/.avalanchego/plugins")),
 		"[optional] the dir containing VM plugins",
 	)
-	startNetworkCmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "The path to a kubernetes configuration file for the target cluster")
+	SetKubeConfigFlags(startNetworkCmd.PersistentFlags(), &kubeconfig, &kubeContext)
 	startNetworkCmd.PersistentFlags().StringVar(&namespace, "namespace", "tmpnet", "The namespace in the target cluster to create nodes in")
 	startNetworkCmd.PersistentFlags().StringVar(&imageName, "image-name", "avaplatform/avalanchego:latest", "The name of the docker image to use for creating nodes")
 	startNetworkCmd.PersistentFlags().Uint8Var(&nodeCount, "node-count", tmpnet.DefaultNodeCount, "Number of nodes the network should initially consist of")
@@ -268,9 +271,40 @@ func main() {
 	)
 	rootCmd.AddCommand(checkLogsCmd)
 
+	startKindClusterCmd := &cobra.Command{
+		Use:   "start-kind-cluster",
+		Short: "Starts a local kind cluster with an integrated registry",
+		RunE: func(*cobra.Command, []string) error {
+			ctx, cancel := context.WithTimeout(context.Background(), tmpnet.DefaultNetworkTimeout)
+			defer cancel()
+			log, err := tests.LoggerForFormat("", rawLogFormat)
+			if err != nil {
+				return err
+			}
+			return tmpnet.StartKindCluster(ctx, log, kubeconfig, kubeContext)
+		},
+	}
+	SetKubeConfigFlags(startKindClusterCmd.PersistentFlags(), &kubeconfig, &kubeContext)
+	rootCmd.AddCommand(startKindClusterCmd)
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "tmpnetctl failed: %v\n", err)
 		os.Exit(1)
 	}
 	os.Exit(0)
 }
+
+func SetKubeConfigFlags(flagSet *pflag.FlagSet, kubeConfig *string, kubeContext *string) {
+	flagSet.StringVar(
+		kubeConfig,
+		"kubeconfig",
+		os.Getenv("KUBECONFIG"),
+		"The path to a kubernetes configuration file for the target cluster",
+	)
+	flagSet.StringVar(
+		kubeContext,
+		"",
+		"",
+		"The path to a kubernetes configuration file for the target cluster",
+	)
+}

tests/fixture/tmpnet/deploy_kind_cluster.go

@@ -0,0 +1,54 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package tmpnet
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"go.uber.org/zap"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+// TODO(marun) Maybe differentiate between configuration and endpoint errors
+func CheckClusterRunning(configPath string, configContext string) error {
+	clientset, err := GetClientset(configPath, configContext)
+	if err != nil {
+		return err
+	}
+	// Check if the configured context can reach a cluster endpoint
+	_, err = clientset.Discovery().ServerVersion()
+	return err
+}
+
+func StartKindCluster(ctx context.Context, log logging.Logger, configPath string, configContext string) error {
+	err := CheckClusterRunning(configPath, configContext)
+	if err == nil {
+		log.Info("kubernetes cluster already running",
+			zap.String("config", configPath),
+			zap.String("context", configContext),
+		)
+		return nil
+	}
+
+	log.Debug("kubernetes cluster not running",
+		zap.String("config", configPath),
+		zap.String("context", configContext),
+		zap.Error(err),
+	)
+
+	// Start a new kind cluster
+	ctx, cancel := context.WithTimeout(ctx, DefaultNetworkTimeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "kind-with-registry.sh")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to run kind-with-registry.sh: %w", err)
+	}
+	return nil
+}

tests/fixture/tmpnet/kube.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/portforward"
 	"k8s.io/client-go/transport/spdy"
 	"k8s.io/utils/pointer"
@@ -344,3 +345,34 @@ func enableLocalForwardForPod(
 	}
 	return forwardedPorts[0].Local, stopChan, nil
 }
+
+func GetClientConfig(path string, context string) (*restclient.Config, error) {
+	if len(context) == 0 {
+		// BuildConfigFromFlags uses the default context and will attempt to use an
+		// in-cluster config with the path as a fallback
+		return clientcmd.BuildConfigFromFlags("", path)
+	}
+
+	config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{
+			ExplicitPath: path,
+		},
+		&clientcmd.ConfigOverrides{
+			CurrentContext: context,
+		},
+	)
+	return config.ClientConfig()
+}
+
+func GetClientset(path string, context string) (*kubernetes.Clientset, error) {
+	clientConfig, err := GetClientConfig(path, context)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get client config: %w", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create clientset: %w", err)
+	}
+	return clientset, nil
+}

tests/fixture/tmpnet/node_pod.go

@@ -19,7 +19,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/ava-labs/avalanchego/config"
 
@@ -375,23 +374,12 @@ func (p *NodePod) IsHealthy(ctx context.Context) (bool, error) {
 
 func (p *NodePod) getKubeconfig() (*restclient.Config, error) {
 	// TODO(marun) inClusterConfig requires an empty path. How best to ensure this?
-	kubeconfigPath := os.Getenv("KUBECONFIG")
-	if kubeconfigPath == "" {
-		kubeconfigPath = p.runtimeConfig().Kubeconfig
-	}
-	return clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	return GetClientConfig(p.runtimeConfig().Kubeconfig, "")
 }
 
 func (p *NodePod) getClientset() (*kubernetes.Clientset, error) {
-	kubeconfig, err := p.getKubeconfig()
-	if err != nil {
-		return nil, err
-	}
-	clientset, err := kubernetes.NewForConfig(kubeconfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create clientset: %w", err)
-	}
-	return clientset, nil
+	// TODO(marun) Support supplying a context
+	return GetClientset(p.runtimeConfig().Kubeconfig, "")
 }
 
 func (p *NodePod) runtimeConfig() *KubeRuntimeConfig {