Dev

go.mod

@@ -74,6 +74,7 @@ replace (
 )
 
 require (
+	github.com/Microsoft/hcsshim v0.12.6
 	github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000
 	github.com/blang/semver/v4 v4.0.0
 	github.com/containerd/aufs v1.0.0
@@ -182,7 +183,6 @@ require (
 	github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/Microsoft/hcsshim v0.12.6 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/Rican7/retry v0.1.0 // indirect
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect

pkg/agent/containerd/config.go

@@ -71,7 +71,7 @@ func writeContainerdHosts(cfg *config.Node, containerdConfig templates.Container
 }
 
 // cleanContainerdHosts removes any registry host config dirs containing a hosts.toml file
-// with a header that indicates it was created by k8e, or directories where a hosts.toml
+// with a header that indicates it was created by k3s, or directories where a hosts.toml
 // is about to be written.  Unmanaged directories not containing this file, or containing
 // a file without the header, are left alone.
 func cleanContainerdHosts(dir string, hosts HostConfigs) error {
@@ -81,7 +81,7 @@ func cleanContainerdHosts(dir string, hosts HostConfigs) error {
 		os.RemoveAll(hostsDir)
 	}
 
-	// clean directories that contain a hosts.toml with a header indicating it was  created by k8e
+	// clean directories that contain a hosts.toml with a header indicating it was  created by k3s
 	ents, err := os.ReadDir(dir)
 	if err != nil && !os.IsNotExist(err) {
 		return err

pkg/agent/run.go

@@ -90,11 +90,6 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node")
 	}
 
-	conntrackConfig, err := getConntrackConfig(nodeConfig)
-	if err != nil {
-		return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
-	}
-	syssetup.Configure(enableIPv6, conntrackConfig)
 	nodeConfig.AgentConfig.EnableIPv4 = enableIPv4
 	nodeConfig.AgentConfig.EnableIPv6 = enableIPv6
 

pkg/agent/templates/templates.go

@@ -1,9 +1,14 @@
 package templates
 
 import (
+	"bytes"
+	"net/url"
+	"text/template"
+
 	"github.com/rancher/wharfie/pkg/registries"
 
 	"github.com/xiaods/k8e/pkg/daemons/config"
+	"github.com/xiaods/k8e/pkg/version"
 )
 
 type ContainerdRuntimeConfig struct {
@@ -17,7 +22,89 @@ type ContainerdConfig struct {
 	SystemdCgroup         bool
 	IsRunningInUserNS     bool
 	EnableUnprivileged    bool
+	NoDefaultEndpoint     bool
 	PrivateRegistryConfig *registries.Registry
 	ExtraRuntimes         map[string]ContainerdRuntimeConfig
 	Program               string
 }
+
+type RegistryEndpoint struct {
+	OverridePath bool
+	URL          *url.URL
+	Rewrites     map[string]string
+	Config       registries.RegistryConfig
+}
+
+type HostConfig struct {
+	Default   *RegistryEndpoint
+	Program   string
+	Endpoints []RegistryEndpoint
+}
+
+var HostsTomlHeader = "# File generated by " + version.Program + ". DO NOT EDIT.\n"
+
+const HostsTomlTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT.
+{{ with $e := .Default }}
+{{- if $e.URL }}
+server = "{{ $e.URL }}"
+capabilities = ["pull", "resolve", "push"]
+{{ end }}
+{{- if $e.Config.TLS }}
+{{- if $e.Config.TLS.CAFile }}
+ca = [{{ printf "%q" $e.Config.TLS.CAFile }}]
+{{- end }}
+{{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }}
+client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]]
+{{- end }}
+{{- if $e.Config.TLS.InsecureSkipVerify }}
+skip_verify = true
+{{- end }}
+{{ end }}
+{{ end }}
+[host]
+{{ range $e := .Endpoints -}}
+[host."{{ $e.URL }}"]
+  capabilities = ["pull", "resolve"]
+  {{- if $e.OverridePath }}
+  override_path = true
+  {{- end }}
+{{- if $e.Config.TLS }}
+  {{- if $e.Config.TLS.CAFile }}
+  ca = [{{ printf "%q" $e.Config.TLS.CAFile }}]
+  {{- end }}
+  {{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }}
+  client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]]
+  {{- end }}
+  {{- if $e.Config.TLS.InsecureSkipVerify }}
+  skip_verify = true
+  {{- end }}
+{{ end }}
+{{- if $e.Rewrites }}
+  [host."{{ $e.URL }}".rewrite]
+  {{- range $pattern, $replace := $e.Rewrites }}
+    "{{ $pattern }}" = "{{ $replace }}"
+  {{- end }}
+{{ end }}
+{{ end -}}
+`
+
+func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}

pkg/agent/templates/templates_linux.go

@@ -3,13 +3,14 @@
 package templates
 
 import (
-	"bytes"
 	"text/template"
 )
 
 const ContainerdConfigTemplate = `
+{{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
 version = 2
+
 [plugins."io.containerd.internal.v1.opt"]
   path = "{{ .NodeConfig.Containerd.Opt }}"
 [plugins."io.containerd.grpc.v1.cri"]
@@ -18,16 +19,19 @@ version = 2
   enable_selinux = {{ .NodeConfig.SELinux }}
   enable_unprivileged_ports = {{ .EnableUnprivileged }}
   enable_unprivileged_icmp = {{ .EnableUnprivileged }}
+
 {{- if .DisableCgroup}}
   disable_cgroup = true
 {{end}}
 {{- if .IsRunningInUserNS }}
   disable_apparmor = true
   restrict_oom_score_adj = true
 {{end}}
+
 {{- if .NodeConfig.AgentConfig.PauseImage }}
   sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
 {{end}}
+
 {{- if .NodeConfig.AgentConfig.Snapshotter }}
 [plugins."io.containerd.grpc.v1.cri".containerd]
   snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
@@ -40,19 +44,11 @@ cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket
 [plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
 enable_keychain = true
 {{end}}
+
+[plugins."io.containerd.snapshotter.v1.stargz".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
 {{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
 {{range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
 [plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth]
@@ -61,16 +57,15 @@ enable_keychain = true
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
 {{end}}
-{{ if $v.TLS }}
-[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
 {{end}}
 {{end}}
 {{end}}
 {{end}}
+
+{{- if not .NodeConfig.NoFlannel }}
+[plugins."io.containerd.grpc.v1.cri".cni]
+  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
 {{end}}
 
 {{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }}
@@ -81,21 +76,14 @@ enable_keychain = true
 
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
   runtime_type = "io.containerd.runc.v2"
+
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
   SystemdCgroup = {{ .SystemdCgroup }}
+
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
+
 {{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
 {{range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
 [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
@@ -104,15 +92,9 @@ enable_keychain = true
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
 {{end}}
-{{ if $v.TLS }}
-[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-{{end}}
 {{end}}
 {{end}}
+
 {{range $k, $v := .ExtraRuntimes}}
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}"]
   runtime_type = "{{$v.RuntimeType}}"
@@ -122,12 +104,9 @@ enable_keychain = true
 {{end}}
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
-}
+// Linux config templates do not need fixups
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		return s
+	},
+}