cluster: more robust linkage between cluster and registry

go.mod

@@ -33,6 +33,7 @@ require (
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 	gotest.tools/v3 v3.0.3 // indirect
+	k8s.io/api v0.19.2
 	k8s.io/apimachinery v0.19.2
 	k8s.io/cli-runtime v0.19.2
 	k8s.io/client-go v0.19.2

pkg/cluster/admin.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/tilt-dev/ctlptl/pkg/api"
+	"github.com/tilt-dev/localregistry-go"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/klog/v2"
 )
@@ -18,6 +19,7 @@ import (
 type Admin interface {
 	EnsureInstalled(ctx context.Context) error
 	Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error
+	LocalRegistryHosting(registry *api.Registry) *localregistry.LocalRegistryHostingV1
 	Delete(ctx context.Context, config *api.Cluster) error
 }
 
@@ -44,6 +46,10 @@ func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, r
 	return fmt.Errorf("docker-desktop Kubernetes clusters are only available on macos and windows")
 }
 
+func (a *dockerDesktopAdmin) LocalRegistryHosting(registry *api.Registry) *localregistry.LocalRegistryHostingV1 {
+	return nil
+}
+
 func (a *dockerDesktopAdmin) Delete(ctx context.Context, config *api.Cluster) error {
 	return fmt.Errorf("docker-desktop delete not implemented")
 }
@@ -120,30 +126,6 @@ containerdConfigPatches:
 		}
 	}
 
-	if registry != nil {
-		configMap := fmt.Sprintf(`
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: local-registry-hosting
-  namespace: kube-public
-data:
-  localRegistryHosting.v1: |
-    host: "localhost:%d"
-    help: "https://github.com/tilt-dev/ctlptl"
-`, registry.Status.HostPort)
-
-		_, _ = fmt.Fprintf(a.iostreams.ErrOut, "   Configurating kind for registry %s\n", registry.Name)
-		cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", "-")
-		cmd.Stdout = a.iostreams.Out
-		cmd.Stderr = a.iostreams.ErrOut
-		cmd.Stdin = strings.NewReader(configMap)
-		err := cmd.Run()
-		if err != nil {
-			return errors.Wrap(err, "configuring kind registry")
-		}
-	}
-
 	return nil
 }
 
@@ -156,6 +138,14 @@ func (a *kindAdmin) inKindNetwork(registry *api.Registry) bool {
 	return false
 }
 
+func (a *kindAdmin) LocalRegistryHosting(registry *api.Registry) *localregistry.LocalRegistryHostingV1 {
+	return &localregistry.LocalRegistryHostingV1{
+		Host:                   fmt.Sprintf("localhost:%d", registry.Status.HostPort),
+		HostFromClusterNetwork: fmt.Sprintf("%s:%d", registry.Name, registry.Status.ContainerPort),
+		Help:                   "https://github.com/tilt-dev/ctlptl",
+	}
+}
+
 func (a *kindAdmin) Delete(ctx context.Context, config *api.Cluster) error {
 	clusterName := config.Name
 	if !strings.HasPrefix(clusterName, "kind-") {

pkg/cluster/cluster.go

@@ -6,15 +6,19 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/pkg/errors"
 	"github.com/tilt-dev/ctlptl/pkg/api"
 	"github.com/tilt-dev/ctlptl/pkg/registry"
 	"github.com/tilt-dev/localregistry-go"
-	"k8s.io/apimachinery/pkg/api/errors"
+	"gopkg.in/yaml.v3"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/klog/v2"
@@ -38,8 +42,11 @@ type configLoader func() (clientcmdapi.Config, error)
 
 type registryController interface {
 	Apply(ctx context.Context, r *api.Registry) (*api.Registry, error)
+	List(ctx context.Context, options registry.ListOptions) (*api.RegistryList, error)
 }
 
+type clientLoader func(*rest.Config) (kubernetes.Interface, error)
+
 type Controller struct {
 	iostreams    genericclioptions.IOStreams
 	config       clientcmdapi.Config
@@ -49,6 +56,7 @@ type Controller struct {
 	configLoader configLoader
 	registryCtl  registryController
 	mu           sync.Mutex
+	clientLoader clientLoader
 }
 
 func DefaultController(iostreams genericclioptions.IOStreams) (*Controller, error) {
@@ -61,6 +69,10 @@ func DefaultController(iostreams genericclioptions.IOStreams) (*Controller, erro
 		return loader.RawConfig()
 	})
 
+	clientLoader := clientLoader(func(restConfig *rest.Config) (kubernetes.Interface, error) {
+		return kubernetes.NewForConfig(restConfig)
+	})
+
 	config, err := configLoader()
 	if err != nil {
 		return nil, err
@@ -72,6 +84,7 @@ func DefaultController(iostreams genericclioptions.IOStreams) (*Controller, erro
 		clients:      make(map[string]kubernetes.Interface),
 		admins:       make(map[Product]Admin),
 		configLoader: configLoader,
+		clientLoader: clientLoader,
 	}, nil
 }
 
@@ -156,7 +169,7 @@ func (c *Controller) client(name string) (kubernetes.Interface, error) {
 		return nil, err
 	}
 
-	client, err = kubernetes.NewForConfig(restConfig)
+	client, err = c.clientLoader(restConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -190,6 +203,34 @@ func (c *Controller) populateLocalRegistryHosting(ctx context.Context, cluster *
 	}
 
 	cluster.Status.LocalRegistryHosting = &hosting
+
+	if hosting.Host == "" {
+		return nil
+	}
+
+	// Let's try to find the registry corresponding to this cluster.
+	var port int
+	_, err = fmt.Sscanf(hosting.Host, "localhost:%d", &port)
+	if err != nil || port == 0 {
+		return err
+	}
+
+	registryCtl, err := c.registryController(ctx)
+	if err != nil {
+		return err
+	}
+
+	registryList, err := registryCtl.List(ctx, registry.ListOptions{FieldSelector: fmt.Sprintf("port=%d", port)})
+	if err != nil {
+		return err
+	}
+
+	if len(registryList.Items) == 0 {
+		return nil
+	}
+
+	cluster.Registry = registryList.Items[0].Name
+
 	return nil
 }
 
@@ -330,7 +371,7 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Clus
 	}
 
 	existingCluster, err := c.Get(ctx, desired.Name)
-	if err != nil && !errors.IsNotFound(err) {
+	if err != nil && !apierrors.IsNotFound(err) {
 		return nil, err
 	}
 
@@ -387,9 +428,45 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Clus
 		return nil, err
 	}
 
+	if needsCreate && desired.Registry != "" {
+		err = c.createRegistryHosting(ctx, admin, desired, reg)
+		if err != nil {
+			return nil, errors.Wrap(err, "configuring cluster registry")
+		}
+	}
+
 	return c.Get(ctx, desired.Name)
 }
 
+// Create a configmap on the cluster, so that other tools know that a registry
+// has been configured.
+func (c *Controller) createRegistryHosting(ctx context.Context, admin Admin, cluster *api.Cluster, reg *api.Registry) error {
+	hosting := admin.LocalRegistryHosting(reg)
+	if hosting == nil {
+		return nil
+	}
+
+	_, _ = fmt.Fprintf(c.iostreams.ErrOut, "   Configuring %s for registry %s\n", cluster.Name, reg.Name)
+	client, err := c.client(cluster.Name)
+	if err != nil {
+		return err
+	}
+
+	data, err := yaml.Marshal(hosting)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.CoreV1().ConfigMaps("kube-public").Create(ctx, &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "local-registry-hosting",
+			Namespace: "kube-public",
+		},
+		Data: map[string]string{"localRegistryHosting.v1": string(data)},
+	}, metav1.CreateOptions{})
+	return err
+}
+
 func (c *Controller) Delete(ctx context.Context, name string) error {
 	existing, err := c.Get(ctx, name)
 	if err != nil {
@@ -420,14 +497,15 @@ func (c *Controller) reloadConfigs() error {
 func (c *Controller) Get(ctx context.Context, name string) (*api.Cluster, error) {
 	ct, ok := c.config.Contexts[name]
 	if !ok {
-		return nil, errors.NewNotFound(groupResource, name)
+		return nil, apierrors.NewNotFound(groupResource, name)
 	}
 	cluster := &api.Cluster{
 		TypeMeta: typeMeta,
 		Name:     name,
 		Product:  productFromContext(ct, c.config.Clusters[ct.Cluster]).String(),
 	}
 	c.populateCluster(ctx, cluster)
+
 	return cluster, nil
 }