DNS: use gateway instead of embedded docker DNS ++

- before starting the cluster, gather environment info via tools node
- use hostIP/gatewayIP for DNS fix
- revamp of custom entrypoint scripts

cmd/image/imageImport.go

@@ -27,9 +27,9 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/rancher/k3d/v4/cmd/util"
+	"github.com/rancher/k3d/v4/pkg/client"
 	l "github.com/rancher/k3d/v4/pkg/logger"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
-	"github.com/rancher/k3d/v4/pkg/tools"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 )
 
@@ -63,7 +63,7 @@ So if a file './rancher/k3d-tools' exists, k3d will try to import it instead of
 			errOccured := false
 			for _, cluster := range clusters {
 				l.Log().Infof("Importing image(s) into cluster '%s'", cluster.Name)
-				if err := tools.ImageImportIntoClusterMulti(cmd.Context(), runtimes.SelectedRuntime, images, &cluster, loadImageOpts); err != nil {
+				if err := client.ImageImportIntoClusterMulti(cmd.Context(), runtimes.SelectedRuntime, images, &cluster, loadImageOpts); err != nil {
 					l.Log().Errorf("Failed to import image(s) into cluster '%s': %+v", cluster.Name, err)
 					errOccured = true
 				}

pkg/client/cluster.go

@@ -70,15 +70,23 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 	/*
 	 * Step 2: Pre-Start Configuration
 	 */
-	// TODO: ClusterRun: add cluster configuration step here
+	_, err := EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster)
+	if err != nil {
+		return err
+	}
+	envInfo, err := GatherEnvironmentInfo(ctx, runtime, &clusterConfig.Cluster)
+	if err != nil {
+		return err
+	}
 
 	/*
 	 * Step 3: Start Containers
 	 */
 	if err := ClusterStart(ctx, runtime, &clusterConfig.Cluster, k3d.ClusterStartOpts{
-		WaitForServer: clusterConfig.ClusterCreateOpts.WaitForServer,
-		Timeout:       clusterConfig.ClusterCreateOpts.Timeout, // TODO: here we should consider the time used so far
-		NodeHooks:     clusterConfig.ClusterCreateOpts.NodeHooks,
+		WaitForServer:   clusterConfig.ClusterCreateOpts.WaitForServer,
+		Timeout:         clusterConfig.ClusterCreateOpts.Timeout, // TODO: here we should consider the time used so far
+		NodeHooks:       clusterConfig.ClusterCreateOpts.NodeHooks,
+		EnvironmentInfo: envInfo,
 	}); err != nil {
 		return fmt.Errorf("Failed Cluster Start: %+v", err)
 	}
@@ -859,10 +867,11 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	 */
 	if initNode != nil {
 		l.Log().Infoln("Starting the initializing server...")
-		if err := NodeStart(ctx, runtime, initNode, k3d.NodeStartOpts{
+		if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
 			Wait:            true, // always wait for the init node
 			NodeHooks:       startClusterOpts.NodeHooks,
 			ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
+			EnvironmentInfo: startClusterOpts.EnvironmentInfo,
 		}); err != nil {
 			return fmt.Errorf("Failed to start initializing server node: %+v", err)
 		}
@@ -872,9 +881,10 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	 * Server Nodes
 	 */
 	l.Log().Infoln("Starting servers...")
-	nodeStartOpts := k3d.NodeStartOpts{
-		Wait:      true,
-		NodeHooks: startClusterOpts.NodeHooks,
+	nodeStartOpts := &k3d.NodeStartOpts{
+		Wait:            true,
+		NodeHooks:       startClusterOpts.NodeHooks,
+		EnvironmentInfo: startClusterOpts.EnvironmentInfo,
 	}
 	for _, serverNode := range servers {
 		if err := NodeStart(ctx, runtime, serverNode, nodeStartOpts); err != nil {
@@ -909,8 +919,9 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		currentHelperNode := helperNode
 
 		helperWG.Go(func() error {
-			nodeStartOpts := k3d.NodeStartOpts{
-				NodeHooks: currentHelperNode.HookActions,
+			nodeStartOpts := &k3d.NodeStartOpts{
+				NodeHooks:       currentHelperNode.HookActions,
+				EnvironmentInfo: startClusterOpts.EnvironmentInfo,
 			}
 			if currentHelperNode.Role == k3d.LoadBalancerRole {
 				nodeStartOpts.Wait = true
@@ -940,36 +951,6 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		return err
 	}
 
-	// network magic
-	dockerDNSIP := "127.0.0.11"
-	hostIP, err := GetHostIP(ctx, runtime, cluster)
-	if err != nil {
-		l.Log().Warnf("Failed to get HostIP to inject as host.k3d.internal: %+v", err)
-		return nil
-	}
-	cmd := `iptables-save \
-	| sed \
-		-e "s/-d ` + dockerDNSIP + `/-d ` + hostIP.String() + `/g" \
-		-e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
-		-e "s/--to-source :53/--to-source ` + hostIP.String() + `:53/g"\
-	| iptables-restore
-cp /etc/resolv.conf /etc/resolv.conf.original
-sed -e "s/` + dockerDNSIP + `/` + hostIP.String() + `/g" /etc/resolv.conf.original >/etc/resolv.conf
-`
-	for _, node := range cluster.Nodes {
-		logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", cmd})
-		if err != nil {
-			msg := fmt.Sprintf("error applying network magic: %+v", err)
-			readlogs, err := ioutil.ReadAll(logreader)
-			if err != nil {
-				l.Log().Errorf("error reading the logs from failed network magic exec process in node %s: %v", node.Name, err)
-			} else {
-				msg += fmt.Sprintf("\nLogs: %s", string(readlogs))
-			}
-			return fmt.Errorf(msg)
-		}
-	}
-
 	return nil
 }
 

pkg/client/environment.go

@@ -0,0 +1,44 @@
+/*
+Copyright © 2020-2021 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package client
+
+import (
+	"context"
+
+	"github.com/rancher/k3d/v4/pkg/runtimes"
+
+	k3d "github.com/rancher/k3d/v4/pkg/types"
+)
+
+func GatherEnvironmentInfo(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.EnvironmentInfo, error) {
+	envInfo := &k3d.EnvironmentInfo{}
+
+	hostIP, err := GetHostIP(ctx, runtime, cluster)
+	if err != nil {
+		return envInfo, err
+	}
+
+	envInfo.HostGateway = hostIP
+
+	return envInfo, nil
+
+}

pkg/client/fixes.go

@@ -32,7 +32,7 @@ import (
 
 // FIXME: FixCgroupV2 - to be removed when fixed upstream
 func EnableCgroupV2FixIfNeeded(runtime runtimes.Runtime) {
-	if _, isSet := os.LookupEnv(fixes.EnvFixCgroupV2); !isSet {
+	if _, isSet := os.LookupEnv(string(fixes.EnvFixCgroupV2)); !isSet {
 		runtimeInfo, err := runtime.Info()
 		if err != nil {
 			l.Log().Warnf("Failed to get runtime information: %+v", err)
@@ -45,7 +45,7 @@ func EnableCgroupV2FixIfNeeded(runtime runtimes.Runtime) {
 		}
 		if cgroupVersion == 2 {
 			l.Log().Debugf("Detected CgroupV2, enabling custom entrypoint (disable by setting %s=false)", fixes.EnvFixCgroupV2)
-			if err := os.Setenv(fixes.EnvFixCgroupV2, "true"); err != nil {
+			if err := os.Setenv(string(fixes.EnvFixCgroupV2), "true"); err != nil {
 				l.Log().Errorf("Detected CgroupsV2 but failed to enable k3d's hotfix (try `export %s=true`): %+v", fixes.EnvFixCgroupV2, err)
 			}
 		}

pkg/client/host.go

@@ -57,7 +57,13 @@ func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net
 
 		// Docker (for Desktop) on MacOS or Windows
 		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
-			ip, err := resolveHostnameFromInside(ctx, rtime, cluster.Nodes[0], "host.docker.internal")
+
+			toolsNode, err := EnsureToolsNode(ctx, rtime, cluster)
+			if err != nil {
+				return nil, err
+			}
+
+			ip, err := resolveHostnameFromInside(ctx, rtime, toolsNode, "host.docker.internal")
 			if err != nil {
 				return nil, err
 			}

pkg/client/node.go

@@ -283,10 +283,11 @@ func NodeRun(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, node
 		return err
 	}
 
-	if err := NodeStart(ctx, runtime, node, k3d.NodeStartOpts{
-		Wait:      nodeCreateOpts.Wait,
-		Timeout:   nodeCreateOpts.Timeout,
-		NodeHooks: nodeCreateOpts.NodeHooks,
+	if err := NodeStart(ctx, runtime, node, &k3d.NodeStartOpts{
+		Wait:            nodeCreateOpts.Wait,
+		Timeout:         nodeCreateOpts.Timeout,
+		NodeHooks:       nodeCreateOpts.NodeHooks,
+		EnvironmentInfo: nodeCreateOpts.EnvironmentInfo,
 	}); err != nil {
 		return err
 	}
@@ -295,33 +296,16 @@ func NodeRun(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, node
 }
 
 // NodeStart starts an existing node
-func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, nodeStartOpts k3d.NodeStartOpts) error {
+func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, nodeStartOpts *k3d.NodeStartOpts) error {
 
 	// return early, if the node is already running
 	if node.State.Running {
 		l.Log().Infof("Node %s is already running", node.Name)
 		return nil
 	}
 
-	// FIXME: FixCgroupV2 - to be removed when fixed upstream
-	if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
-		EnableCgroupV2FixIfNeeded(runtime)
-		if fixes.FixCgroupV2Enabled() {
-
-			if nodeStartOpts.NodeHooks == nil {
-				nodeStartOpts.NodeHooks = []k3d.NodeHook{}
-			}
-
-			nodeStartOpts.NodeHooks = append(nodeStartOpts.NodeHooks, k3d.NodeHook{
-				Stage: k3d.LifecycleStagePreStart,
-				Action: actions.WriteFileAction{
-					Runtime: runtime,
-					Content: fixes.CgroupV2Entrypoint,
-					Dest:    "/bin/entrypoint.sh",
-					Mode:    0744,
-				},
-			})
-		}
+	if err := enableFixes(ctx, runtime, node, nodeStartOpts); err != nil {
+		return err
 	}
 
 	startTime := time.Now()
@@ -371,6 +355,79 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
 	return nil
 }
 
+func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, nodeStartOpts *k3d.NodeStartOpts) error {
+
+	if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
+
+		// FIXME: FixCgroupV2 - to be removed when fixed upstream
+		// auto-enable, if needed
+		EnableCgroupV2FixIfNeeded(runtime)
+
+		// early exit if we don't need any fix
+		if !fixes.FixEnabledAny() {
+			l.Log().Debugln("No fix enabled.")
+			return nil
+		}
+
+		// ensure nodehook list
+		if nodeStartOpts.NodeHooks == nil {
+			nodeStartOpts.NodeHooks = []k3d.NodeHook{}
+		}
+
+		// write umbrella entrypoint
+		nodeStartOpts.NodeHooks = append(nodeStartOpts.NodeHooks, k3d.NodeHook{
+			Stage: k3d.LifecycleStagePreStart,
+			Action: actions.WriteFileAction{
+				Runtime: runtime,
+				Content: fixes.K3DEntrypoint,
+				Dest:    "/bin/k3d-entrypoint.sh",
+				Mode:    0744,
+			},
+		})
+
+		// DNS Fix
+		if fixes.FixEnabled(fixes.EnvFixDNS) {
+			l.Log().Debugf("ENABLING DNS MAGIC!!!")
+
+			if nodeStartOpts.EnvironmentInfo == nil || nodeStartOpts.EnvironmentInfo.HostGateway == nil {
+				return fmt.Errorf("Cannot enable DNS fix, as Host Gateway IP is missing!")
+			}
+
+			data := []byte(strings.ReplaceAll(string(fixes.DNSMagicEntrypoint), "GATEWAY_IP", nodeStartOpts.EnvironmentInfo.HostGateway.String()))
+
+			nodeStartOpts.NodeHooks = append(nodeStartOpts.NodeHooks, k3d.NodeHook{
+				Stage: k3d.LifecycleStagePreStart,
+				Action: actions.WriteFileAction{
+					Runtime: runtime,
+					Content: data,
+					Dest:    "/bin/k3d-entrypoint-dns.sh",
+					Mode:    0744,
+				},
+			})
+		}
+
+		// CGroupsV2Fix
+		if fixes.FixEnabled(fixes.EnvFixCgroupV2) {
+			l.Log().Debugf("ENABLING CGROUPSV2 MAGIC!!!")
+
+			if nodeStartOpts.NodeHooks == nil {
+				nodeStartOpts.NodeHooks = []k3d.NodeHook{}
+			}
+
+			nodeStartOpts.NodeHooks = append(nodeStartOpts.NodeHooks, k3d.NodeHook{
+				Stage: k3d.LifecycleStagePreStart,
+				Action: actions.WriteFileAction{
+					Runtime: runtime,
+					Content: fixes.CgroupV2Entrypoint,
+					Dest:    "/bin/k3d-entrypoint-cgroupv2.sh",
+					Mode:    0744,
+				},
+			})
+		}
+	}
+	return nil
+}
+
 // NodeCreate creates a new containerized k3s node
 func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, createNodeOpts k3d.NodeCreateOpts) error {
 	// FIXME: FixCgroupV2 - to be removed when fixed upstream
@@ -742,15 +799,15 @@ func NodeReplace(ctx context.Context, runtime runtimes.Runtime, old, new *k3d.No
 
 	// start new node
 	l.Log().Infof("Starting new node %s...", new.Name)
-	if err := NodeStart(ctx, runtime, new, k3d.NodeStartOpts{Wait: true, NodeHooks: new.HookActions}); err != nil {
+	if err := NodeStart(ctx, runtime, new, &k3d.NodeStartOpts{Wait: true, NodeHooks: new.HookActions}); err != nil {
 		if err := NodeDelete(ctx, runtime, new, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
 			return fmt.Errorf("Failed to start new node. Also failed to rollback: %+v", err)
 		}
 		if err := runtime.RenameNode(ctx, old, oldNameOriginal); err != nil {
 			return fmt.Errorf("Failed to start new node. Also failed to rename %s back to %s: %+v", old.Name, oldNameOriginal, err)
 		}
 		old.Name = oldNameOriginal
-		if err := NodeStart(ctx, runtime, old, k3d.NodeStartOpts{Wait: true}); err != nil {
+		if err := NodeStart(ctx, runtime, old, &k3d.NodeStartOpts{Wait: true}); err != nil {
 			return fmt.Errorf("Failed to start new node. Also failed to restart old node: %+v", err)
 		}
 		return fmt.Errorf("Failed to start new node. Rolled back: %+v", err)

pkg/client/registry.go

@@ -43,7 +43,7 @@ func RegistryRun(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Registr
 		return nil, fmt.Errorf("Failed to create registry: %+v", err)
 	}
 
-	if err := NodeStart(ctx, runtime, regNode, k3d.NodeStartOpts{}); err != nil {
+	if err := NodeStart(ctx, runtime, regNode, &k3d.NodeStartOpts{}); err != nil {
 		return nil, fmt.Errorf("Failed to start registry: %+v", err)
 	}