-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add embedded registry implementation
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
- Loading branch information
Showing
10 changed files
with
919 additions
and
90 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,177 @@ | ||
| package spegel | ||
|
|
||
| import ( | ||
| "context" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/k3s-io/k3s/pkg/clientaccess" | ||
| "github.com/k3s-io/k3s/pkg/daemons/config" | ||
| "github.com/k3s-io/k3s/pkg/util" | ||
| "github.com/k3s-io/k3s/pkg/version" | ||
| "github.com/libp2p/go-libp2p/core/peer" | ||
| "github.com/pkg/errors" | ||
| "github.com/rancher/wrangler/pkg/merr" | ||
| "github.com/sirupsen/logrus" | ||
| "github.com/xenitab/spegel/pkg/routing" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/labels" | ||
| "k8s.io/apimachinery/pkg/util/wait" | ||
| ) | ||
|
|
||
| // explicit interface checks | ||
| var _ routing.Bootstrapper = &selfBootstrapper{} | ||
| var _ routing.Bootstrapper = &agentBootstrapper{} | ||
| var _ routing.Bootstrapper = &serverBootstrapper{} | ||
| var _ routing.Bootstrapper = &chainingBootstrapper{} | ||
|
|
||
| type selfBootstrapper struct { | ||
| id string | ||
| } | ||
|
|
||
| // NewSelfBootstrapper returns a stub p2p bootstrapper that just returns its own ID | ||
| func NewSelfBootstrapper() routing.Bootstrapper { | ||
| return &selfBootstrapper{} | ||
| } | ||
|
|
||
| func (s *selfBootstrapper) Run(_ context.Context, id string) error { | ||
| s.id = id | ||
| return nil | ||
| } | ||
|
|
||
| func (s *selfBootstrapper) GetAddress() (*peer.AddrInfo, error) { | ||
| return peer.AddrInfoFromString(s.id) | ||
| } | ||
|
|
||
| type agentBootstrapper struct { | ||
| server string | ||
| token string | ||
| clientCert string | ||
| clientKey string | ||
| } | ||
|
|
||
| // NewAgentBootstrapper returns a p2p bootstrapper that retrieves a peer address from its server | ||
| func NewAgentBootstrapper(server, token, dataDir string) routing.Bootstrapper { | ||
| return &agentBootstrapper{ | ||
| clientCert: filepath.Join(dataDir, "agent", "client-kubelet.crt"), | ||
| clientKey: filepath.Join(dataDir, "agent", "client-kubelet.key"), | ||
| server: server, | ||
| token: token, | ||
| } | ||
| } | ||
|
|
||
| func (c *agentBootstrapper) Run(ctx context.Context, _ string) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (c *agentBootstrapper) GetAddress() (*peer.AddrInfo, error) { | ||
| if c.server == "" || c.token == "" { | ||
| return nil, errors.New("cannot get addresses without server and token") | ||
| } | ||
|
|
||
| withCert := clientaccess.WithClientCertificate(c.clientCert, c.clientKey) | ||
| info, err := clientaccess.ParseAndValidateToken(c.server, c.token, withCert) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| addr, err := info.Get("/v1-" + version.Program + "/p2p") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return peer.AddrInfoFromString(string(addr)) | ||
| } | ||
|
|
||
| type serverBootstrapper struct { | ||
| controlConfig *config.Control | ||
| } | ||
|
|
||
| // NewServerBootstrapper returns a p2p bootstrapper that returns an address from a random cluster member. | ||
| func NewServerBootstrapper(controlConfig *config.Control) routing.Bootstrapper { | ||
| return &serverBootstrapper{ | ||
| controlConfig: controlConfig, | ||
| } | ||
| } | ||
|
|
||
| func (s *serverBootstrapper) Run(_ context.Context, id string) error { | ||
| s.controlConfig.Runtime.ClusterControllerStarts["spegel-p2p"] = func(ctx context.Context) { | ||
| nodes := s.controlConfig.Runtime.Core.Core().V1().Node() | ||
| wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (bool, error) { | ||
| nodeName := os.Getenv("NODE_NAME") | ||
| if nodeName == "" { | ||
| return false, nil | ||
| } | ||
| node, err := nodes.Get(nodeName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| return false, nil | ||
| } | ||
| if node.Annotations == nil { | ||
| node.Annotations = map[string]string{} | ||
| } | ||
| node.Annotations[P2pAddressAnnotation] = id | ||
| if _, err = nodes.Update(node); err != nil { | ||
| return false, nil | ||
| } | ||
| logrus.Infof("Node P2P address annotation added: %s", id) | ||
| return true, nil | ||
| }) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (s *serverBootstrapper) GetAddress() (*peer.AddrInfo, error) { | ||
| if s.controlConfig.Runtime.Core == nil { | ||
| return nil, errors.New("runtime core not ready") | ||
| } | ||
|
|
||
| nodes := s.controlConfig.Runtime.Core.Core().V1().Node() | ||
| labelSelector := labels.Set{util.ControlPlaneRoleLabelKey: "true"}.String() | ||
| nodeList, err := nodes.List(metav1.ListOptions{LabelSelector: labelSelector}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| for _, node := range nodeList.Items { | ||
| if addr, ok := node.Annotations[P2pAddressAnnotation]; ok { | ||
| if info, err := peer.AddrInfoFromString(addr); err == nil { | ||
| return info, nil | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil, errors.New("no p2p control-plane nodes found") | ||
| } | ||
|
|
||
| type chainingBootstrapper struct { | ||
| bootstrappers []routing.Bootstrapper | ||
| } | ||
|
|
||
| // NewChainingBoostrapper returns a p2p bootstrapper that passes through to a list of bootstrappers. | ||
| func NewChainingBootstrapper(bootstrappers ...routing.Bootstrapper) routing.Bootstrapper { | ||
| return &chainingBootstrapper{ | ||
| bootstrappers: bootstrappers, | ||
| } | ||
| } | ||
|
|
||
| func (c *chainingBootstrapper) Run(ctx context.Context, id string) error { | ||
| errs := merr.Errors{} | ||
| for _, b := range c.bootstrappers { | ||
| if err := b.Run(ctx, id); err != nil { | ||
| errs = append(errs, err) | ||
| } | ||
| } | ||
| return merr.NewErrors(errs...) | ||
| } | ||
|
|
||
| func (c *chainingBootstrapper) GetAddress() (*peer.AddrInfo, error) { | ||
| errs := merr.Errors{} | ||
| for _, b := range c.bootstrappers { | ||
| addr, err := b.GetAddress() | ||
| if err == nil { | ||
| return addr, nil | ||
| } | ||
| errs = append(errs, err) | ||
| } | ||
| return nil, merr.NewErrors(errs...) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| package spegel | ||
|
|
||
| import ( | ||
| "github.com/k3s-io/k3s/pkg/daemons/config" | ||
| "github.com/rancher/wharfie/pkg/registries" | ||
| ) | ||
|
|
||
| // InjectMirror configures TLS for the registry mirror client, and adds the mirror address as an endpoint | ||
| // to all configured registries. | ||
| func (c *Config) InjectMirror(nodeConfig *config.Node) error { | ||
| registry := nodeConfig.AgentConfig.Registry | ||
|
|
||
| if registry.Configs == nil { | ||
| registry.Configs = map[string]registries.RegistryConfig{} | ||
| } | ||
| registry.Configs[c.Address] = registries.RegistryConfig{ | ||
| TLS: ®istries.TLSConfig{ | ||
| CAFile: c.ServerCAFile, | ||
| CertFile: c.ClientCertFile, | ||
| KeyFile: c.ClientKeyFile, | ||
| }, | ||
| } | ||
|
|
||
| if registry.Mirrors == nil { | ||
| registry.Mirrors = map[string]registries.Mirror{} | ||
| } | ||
| for host, mirror := range registry.Mirrors { | ||
| mirror.Endpoints = append([]string{"https://" + c.Address}, mirror.Endpoints...) | ||
| registry.Mirrors[host] = mirror | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.