Add spegel distributed registry mirror

docs/adrs/embedded-registry.md

@@ -0,0 +1,43 @@
+# Package spegel Distributed Registry Mirror
+
+Date: 2023-12-07
+
+## Status
+
+Accepted
+
+## Context
+
+Embedded registry mirror support has been on the roadmap for some time, to address multiple challenges:
+* Upstream registries may enforce pull limits or otherwise throttle access to images.
+* In edge scenarios, bandwidth is at a premium, if external access is available at all.
+* Distributing airgap image tarballs to nodes, and ensuring that images remain available, is an ongoing
+  hurdle to adoption.
+* Deploying an in-cluster registry, or hosting a registry outside the cluster, put significant
+  burden on administrators, and suffer from chicken-or-egg bootstrapping issues.
+
+An ideal embedded registry would have several characteristics:
+* Allow stateless configuration such that nodes can come and go at any time.
+* Integrate into existing containerd registry mirror support.
+* Integrate into existing containerd image stores such that an additional copy of layer data is not required.
+* Use existing cluster authentication mechanisms to prevent unauthorized access to the registry.
+* Operate with minimal added CPU and memory overhead.
+
+## Decision
+
+* We will embed spegel within K3s, and use it to host a distributed registry mirror.
+* The distributed registry mirror will be enabled cluster-wide via server CLI flag.
+* Selection of upstream registries to mirror will be implemented via the existing `registries.yaml`
+configuration file.
+* The registry API will be served via HTTPS on every node's private IP at port 6443. On servers this will
+use the existing supervisor listener; on agents a new listener will be created for this purpose.
+* The default IPFS/libp2p port of 5001 will be used for P2P layer discovery.
+* Access to the registry API and P2P network will require proof of cluster membership, enforced via
+client certificate or preshared key.
+* Hybrid/multicloud support is out of scope; when the distributed registry mirror is enabled, cluster
+members are assumed to be directly accessible to each other via their internal IP on the listed ports.
+
+## Consequences
+
+* The size of our self-extracting binary and Docker images increase by several megabytes.
+* We take on the support burden of keeping spegel up to date, and supporting its use within K3s.

pkg/agent/config/config.go

@@ -16,6 +16,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -26,10 +27,12 @@ import (
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/pkg/errors"
+	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/rancher/wrangler/pkg/slice"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
@@ -47,8 +50,8 @@ const (
 // so this is somewhat computationally expensive on the server side, and is retried with jitter
 // to avoid having clients hammer on the server at fixed periods.
 // A call to this will bock until agent configuration is successfully returned by the
-// server.
-func Get(ctx context.Context, agent cmds.Agent, proxy proxy.Proxy) *config.Node {
+// server, or the context is cancelled.
+func Get(ctx context.Context, agent cmds.Agent, proxy proxy.Proxy) (*config.Node, error) {
 	var agentConfig *config.Node
 	var err error
 
@@ -64,7 +67,7 @@ func Get(ctx context.Context, agent cmds.Agent, proxy proxy.Proxy) *config.Node
 			cancel()
 		}
 	}, 5*time.Second, 1.0, true)
-	return agentConfig
+	return agentConfig, err
 }
 
 // KubeProxyDisabled returns a bool indicating whether or not kube-proxy has been disabled in the
@@ -501,6 +504,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		SELinux:                  envInfo.EnableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
 		ImageServiceEndpoint:     envInfo.ImageServiceEndpoint,
+		EmbeddedRegistry:         controlConfig.EmbeddedRegistry,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
 		FlannelExternalIP:        controlConfig.FlannelExternalIP,
@@ -663,7 +667,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.NodeLabels = envInfo.Labels
 	nodeConfig.AgentConfig.ImageCredProvBinDir = envInfo.ImageCredProvBinDir
 	nodeConfig.AgentConfig.ImageCredProvConfig = envInfo.ImageCredProvConfig
-	nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry
 	nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM
 	nodeConfig.AgentConfig.DisableNPC = controlConfig.DisableNPC
 	nodeConfig.AgentConfig.Rootless = envInfo.Rootless
@@ -675,6 +678,35 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.LogFile = cmds.LogConfig.LogFile
 	nodeConfig.AgentConfig.AlsoLogToStderr = cmds.LogConfig.AlsoLogToStderr
 
+	privRegistries, err := registries.GetPrivateRegistries(envInfo.PrivateRegistry)
+	if err != nil {
+		return nil, err
+	}
+	nodeConfig.AgentConfig.Registry = privRegistries.Registry
+
+	if nodeConfig.EmbeddedRegistry {
+		psk, err := hex.DecodeString(controlConfig.IPSECPSK)
+		if err != nil {
+			return nil, err
+		}
+		if len(psk) < 32 {
+			return nil, errors.New("insufficient PSK bytes")
+		}
+
+		conf := spegel.DefaultRegistry
+		conf.ExternalAddress = nodeConfig.AgentConfig.NodeIP
+		conf.InternalAddress = controlConfig.Loopback(false)
+		conf.RegistryPort = strconv.Itoa(controlConfig.SupervisorPort)
+		conf.ClientCAFile = clientCAFile
+		conf.ClientCertFile = clientK3sControllerCert
+		conf.ClientKeyFile = clientK3sControllerKey
+		conf.ServerCAFile = serverCAFile
+		conf.ServerCertFile = servingKubeletCert
+		conf.ServerKeyFile = servingKubeletKey
+		conf.PSK = psk[:32]
+		conf.InjectMirror(nodeConfig)
+	}
+
 	if err := validateNetworkConfig(nodeConfig); err != nil {
 		return nil, err
 	}

pkg/agent/containerd/config.go

@@ -1,6 +1,7 @@
 package containerd
 
 import (
+	"net"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -10,6 +11,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/templates"
 	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
 )
@@ -36,6 +38,7 @@ func writeContainerdConfig(cfg *config.Node, containerdConfig templates.Containe
 
 // writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
 func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	mirrorAddr := net.JoinHostPort(spegel.DefaultRegistry.InternalAddress, spegel.DefaultRegistry.RegistryPort)
 	registry := containerdConfig.PrivateRegistryConfig
 	hosts := map[string]templates.HostConfig{}
 
@@ -57,12 +60,17 @@ func writeContainerdHosts(cfg *config.Node, containerdConfig templates.Container
 		// structure, which is defined in rancher/wharfie.
 		for _, endpoint := range mirror.Endpoints {
 			if endpointURL, err := url.Parse(endpoint); err == nil {
-				config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
+				re := templates.RegistryEndpoint{
 					OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
 					Config:       registry.Configs[endpointURL.Host],
 					Rewrites:     mirror.Rewrites,
 					URI:          endpoint,
-				})
+				}
+				// Do not apply rewrites to the embedded registry endpoint
+				if endpointURL.Host == mirrorAddr {
+					re.Rewrites = nil
+				}
+				config.Endpoints = append(config.Endpoints, re)
 			}
 		}
 		hosts[host] = config

pkg/agent/containerd/config_linux.go

@@ -18,7 +18,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/opencontainers/runc/libcontainer/userns"
 	"github.com/pkg/errors"
-	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/util"
@@ -40,11 +39,6 @@ func getContainerdArgs(cfg *config.Node) []string {
 // setupContainerdConfig generates the containerd.toml, using a template combined with various
 // runtime configurations and registry mirror settings provided by the administrator.
 func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
-	privRegistries, err := registries.GetPrivateRegistries(cfg.AgentConfig.PrivateRegistry)
-	if err != nil {
-		return err
-	}
-
 	isRunningInUserNS := userns.RunningInUserNS()
 	_, _, controllers := cgroups.CheckCgroups()
 	// "/sys/fs/cgroup" is namespaced
@@ -72,7 +66,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		SystemdCgroup:         cfg.AgentConfig.Systemd,
 		IsRunningInUserNS:     isRunningInUserNS,
 		EnableUnprivileged:    kernel.CheckKernelVersion(4, 11, 0),
-		PrivateRegistryConfig: privRegistries.Registry,
+		PrivateRegistryConfig: cfg.AgentConfig.Registry,
 		ExtraRuntimes:         extraRuntimes,
 		Program:               version.Program,
 		NoDefaultEndpoint:     cfg.Containerd.NoDefault,

pkg/agent/containerd/config_windows.go

@@ -11,7 +11,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	util3 "github.com/k3s-io/k3s/pkg/util"
 	"github.com/pkg/errors"
-	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/sirupsen/logrus"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 )
@@ -27,11 +26,6 @@ func getContainerdArgs(cfg *config.Node) []string {
 // setupContainerdConfig generates the containerd.toml, using a template combined with various
 // runtime configurations and registry mirror settings provided by the administrator.
 func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
-	privRegistries, err := registries.GetPrivateRegistries(cfg.AgentConfig.PrivateRegistry)
-	if err != nil {
-		return err
-	}
-
 	if cfg.SELinux {
 		logrus.Warn("SELinux isn't supported on windows")
 	}
@@ -41,7 +35,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		DisableCgroup:         true,
 		SystemdCgroup:         false,
 		IsRunningInUserNS:     false,
-		PrivateRegistryConfig: privRegistries.Registry,
+		PrivateRegistryConfig: cfg.AgentConfig.Registry,
 		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}