Add embedded registry implementation

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/agent/config/config.go

@@ -16,6 +16,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
@@ -679,6 +681,29 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 	nodeConfig.AgentConfig.Registry = privRegistries.Registry
 
+	if nodeConfig.EmbeddedRegistry {
+		psk, err := hex.DecodeString(controlConfig.IPSECPSK)
+		if err != nil {
+			return nil, err
+		}
+		if len(psk) < 32 {
+			return nil, errors.New("insufficient PSK bytes")
+		}
+
+		conf := spegel.DefaultRegistry
+		conf.ExternalAddress = nodeConfig.AgentConfig.NodeIP
+		conf.InternalAddress = controlConfig.Loopback(false)
+		conf.RegistryPort = strconv.Itoa(controlConfig.SupervisorPort)
+		conf.ClientCAFile = clientCAFile
+		conf.ClientCertFile = clientK3sControllerCert
+		conf.ClientKeyFile = clientK3sControllerKey
+		conf.ServerCAFile = serverCAFile
+		conf.ServerCertFile = servingKubeletCert
+		conf.ServerKeyFile = servingKubeletKey
+		conf.PSK = psk[:32]
+		conf.InjectMirror(nodeConfig)
+	}
+
 	if err := validateNetworkConfig(nodeConfig); err != nil {
 		return nil, err
 	}

pkg/agent/containerd/config.go

@@ -1,6 +1,7 @@
 package containerd
 
 import (
+	"net"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -10,6 +11,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/templates"
 	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
 )
@@ -36,6 +38,7 @@ func writeContainerdConfig(cfg *config.Node, containerdConfig templates.Containe
 
 // writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
 func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	mirrorAddr := net.JoinHostPort(spegel.DefaultRegistry.ExternalAddress, spegel.DefaultRegistry.RegistryPort)
 	registry := containerdConfig.PrivateRegistryConfig
 	hosts := map[string]templates.HostConfig{}
 
@@ -57,12 +60,17 @@ func writeContainerdHosts(cfg *config.Node, containerdConfig templates.Container
 		// structure, which is defined in rancher/wharfie.
 		for _, endpoint := range mirror.Endpoints {
 			if endpointURL, err := url.Parse(endpoint); err == nil {
-				config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
+				re := templates.RegistryEndpoint{
 					OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
 					Config:       registry.Configs[endpointURL.Host],
 					Rewrites:     mirror.Rewrites,
 					URI:          endpoint,
-				})
+				}
+				// Do not apply rewrites to the embedded registry endpoint
+				if endpointURL.Host == mirrorAddr {
+					re.Rewrites = nil
+				}
+				config.Endpoints = append(config.Endpoints, re)
 			}
 		}
 		hosts[host] = config

pkg/agent/run.go

@@ -29,6 +29,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
 	"github.com/k3s-io/k3s/pkg/nodeconfig"
 	"github.com/k3s-io/k3s/pkg/rootless"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -100,6 +101,16 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	nodeConfig.AgentConfig.EnableIPv4 = enableIPv4
 	nodeConfig.AgentConfig.EnableIPv6 = enableIPv6
 
+	if nodeConfig.EmbeddedRegistry {
+		if nodeConfig.Docker || nodeConfig.ContainerRuntimeEndpoint != "" {
+			return errors.New("embedded registry mirror requires embedded containerd")
+		}
+
+		if err := spegel.DefaultRegistry.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to start embedded registry")
+		}
+	}
+
 	if err := setupCriCtlConfig(cfg, nodeConfig); err != nil {
 		return err
 	}

pkg/cli/agent/agent.go

@@ -2,21 +2,27 @@ package agent
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 
 	"github.com/erikdubbelboer/gspt"
+	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
+	"github.com/k3s-io/k3s/pkg/authenticator"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/datadir"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
+	apiauth "k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
 func Run(ctx *cli.Context) error {
@@ -96,5 +102,37 @@ func Run(ctx *cli.Context) error {
 		}
 	}
 
+	// Until the agent is run and retrieves config from the server, we won't know
+	// if the embedded registry is enabled. If it is not enabled, these are not
+	// used as the registry is never started.
+	conf := spegel.DefaultRegistry
+	conf.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir)
+	conf.HandlerFunc = func(conf *spegel.Config, router *mux.Router) error {
+		// Create and bind a new authenticator using the configured client CA
+		authArgs := []string{"--client-ca-file=" + conf.ClientCAFile}
+		auth, err := authenticator.FromArgs(authArgs)
+		if err != nil {
+			return err
+		}
+		conf.AuthFunc = func() apiauth.Request {
+			return auth
+		}
+
+		// Create a new server and listen on the configured port
+		server := &http.Server{
+			Handler: router,
+			Addr:    ":" + conf.RegistryPort,
+			TLSConfig: &tls.Config{
+				ClientAuth: tls.RequestClientCert,
+			},
+		}
+		go func() {
+			if err := server.ListenAndServeTLS(conf.ServerCertFile, conf.ServerKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				logrus.Fatalf("registry server failed: %v", err)
+			}
+		}()
+		return nil
+	}
+
 	return agent.Run(contextCtx, cfg)
 }

pkg/cli/server/server.go

@@ -11,6 +11,7 @@ import (
 
 	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/erikdubbelboer/gspt"
+	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/agent"
 	"github.com/k3s-io/k3s/pkg/agent/loadbalancer"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
@@ -20,6 +21,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/k3s/pkg/rootless"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
@@ -28,6 +30,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
 	kubeapiserverflag "k8s.io/component-base/cli/flag"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	utilsnet "k8s.io/utils/net"
@@ -548,6 +551,23 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		return agent.RunStandalone(ctx, agentConfig)
 	}
 
+	if cfg.EmbeddedRegistry {
+		conf := spegel.DefaultRegistry
+		conf.Bootstrapper = spegel.NewChainingBootstrapper(
+			spegel.NewServerBootstrapper(&serverConfig.ControlConfig),
+			spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir),
+			spegel.NewSelfBootstrapper(),
+		)
+		conf.HandlerFunc = func(_ *spegel.Config, router *mux.Router) error {
+			router.NotFoundHandler = serverConfig.ControlConfig.Runtime.Handler
+			serverConfig.ControlConfig.Runtime.Handler = router
+			return nil
+		}
+		conf.AuthFunc = func() authenticator.Request {
+			return serverConfig.ControlConfig.Runtime.Authenticator
+		}
+	}
+
 	return agent.Run(ctx, agentConfig)
 }
 

pkg/daemons/config/types.go

@@ -149,6 +149,7 @@ type CriticalControlArgs struct {
 	DisableServiceLB      bool         `cli:"disable-service-lb"`
 	EncryptSecrets        bool         `cli:"secrets-encryption"`
 	MultiClusterCIDR      bool         `cli:"multi-cluster-cidr"`
+	EmbeddedRegistry      bool         `cli:"embedded-registry"`
 	FlannelBackend        string       `cli:"flannel-backend"`
 	FlannelIPv6Masq       bool         `cli:"flannel-ipv6-masq"`
 	FlannelExternalIP     bool         `cli:"flannel-external-ip"`
@@ -183,7 +184,6 @@ type Control struct {
 	DisableKubeProxy         bool
 	DisableScheduler         bool
 	DisableServiceLB         bool
-	EmbeddedRegistry         bool
 	Rootless                 bool
 	ServiceLBNamespace       string
 	EnablePProf              bool