Add DNS server to host-agent to use native host resolver

jandubois · jandubois · commit 1cd3aa663b34 · 2021-10-01T21:55:24.000-07:00
This is is required to correctly resolve hostnames while using conditional
forwarding (split-DNS) when connected to a VPN.

The hostagent must be compiled with CGO_ENABLED=1 to use the native resolver.

Signed-off-by: Jan Dubois &lt;jan.dubois@suse.com&gt;
diff --git a/Makefile b/Makefile
@@ -11,7 +11,7 @@ PACKAGE := github.com/lima-vm/lima
 VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 VERSION_TRIMMED := $(VERSION:v%=%)
 
-GO_BUILD := CGO_ENABLED=0 $(GO) build -ldflags="-s -w -X $(PACKAGE)/pkg/version.Version=$(VERSION)"
+GO_BUILD := $(GO) build -ldflags="-s -w -X $(PACKAGE)/pkg/version.Version=$(VERSION)"
 
 .PHONY: all
 all: binaries
@@ -39,16 +39,18 @@ _output/bin/nerdctl.lima:
 
 .PHONY: _output/bin/limactl
 _output/bin/limactl:
-	$(GO_BUILD) -o $@ ./cmd/limactl
+	# The hostagent must be compiled with CGO_ENABLED=1 so that net.LookupIP() in the DNS server
+	# calls the native resolver library and not the simplistic version in the Go library.
+	CGO_ENABLED=1 $(GO_BUILD) -o $@ ./cmd/limactl
 
 .PHONY: _output/share/lima/lima-guestagent.Linux-x86_64
 _output/share/lima/lima-guestagent.Linux-x86_64:
-	GOOS=linux GOARCH=amd64 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
 	chmod 644 $@
 
 .PHONY: _output/share/lima/lima-guestagent.Linux-aarch64
 _output/share/lima/lima-guestagent.Linux-aarch64:
-	GOOS=linux GOARCH=arm64 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
+	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
 	chmod 644 $@
 
 .PHONY: install
diff --git a/docs/internal.md b/docs/internal.md
@@ -108,4 +108,6 @@ The volume label is "cidata", as defined by [cloud-init NoCloud](https://cloudin
 - `LIMA_CIDATA_MOUNTS_%d_MOUNTPOINT`: the N-th mount point of Lima mounts (N=0, 1, ...)
 - `LIMA_CIDATA_CONTAINERD_USER`: set to "1" if rootless containerd to be set up
 - `LIMA_CIDATA_CONTAINERD_SYSTEM`: set to "1" if system-wide containerd to be set up
-- `LIMA_CIDATA_SLIRP_GATEWAY`: set to the IP address of the host on the SLIRP network. `192.168.5.2`.
+- `LIMA_CIDATA_SLIRP_GATEWAY`: set to the IP address of the host on the SLIRP network. `192.168.5.2`.
+- `LIMA_CIDATA_SLIRP_DNS`: set to the IP address of the DNS on the SLIRP network. `192.168.5.3`.
+- `LIMA_CIDATA_UDP_DNS_LOCAL_PORT`: set to the udp port number of the hostagent dns server (or 0 when not enabled).
diff --git a/docs/network.md b/docs/network.md
@@ -20,6 +20,14 @@ The loopback addresses of the host is `192.168.5.2` and is accessible from the g
 
 The DNS.
 
+If `useHostResolver` in `lima.yaml` is true, then the hostagent is going to run a DNS server over udp, using the same socket number as `ssh.localPort`. This server does a local lookup using the native host resolver, so will deal correctly with VPN configurations and split-DNS setups, as well a mDNS (for this the hostagent has to be compiled with `CGO_ENABLED=1`).
+
+This udp port is then forwarded via iptables rules to `192.168.5.3:53`, overriding the DNS provided by QEMU via slirp.
+
+During initial cloud-init bootstrap, `iptables` may not yet be installed. In that case the repo server is determined using the slirp DNS. After `iptables` has been installed, the forwarding rule is applied, switching over to the hostagent DNS.
+
+If `useHostResoler` is false, then DNS servers can be configured manually in `lima.yaml` via the `dns` setting. If that list is empty, then Lima will either use the slirp DNS (on Linux), or the nameservers from the `en0` host interface (on macOS).
+
 ## `vde_vmnet` (192.168.105.0/24)
 
 [`vde_vmnet`](https://github.com/lima-vm/vde_vmnet) is required for adding another guest IP that is accessible from
diff --git a/examples/alpine.yaml b/examples/alpine.yaml
@@ -1,7 +1,7 @@
 images:
-- location: https://github.com/lima-vm/alpine-lima/releases/download/v0.1.4/alpine-lima-std-3.13.5-x86_64.iso
+- location: https://github.com/lima-vm/alpine-lima/releases/download/v0.1.5/alpine-lima-std-3.13.5-x86_64.iso
   arch: "x86_64"
-  digest: "sha512:ef627d902645744ab9f62cfe9f741f84c9d544b36da113aef9410489db8d2430b9c37581c5958b4d7fc64dfb5267f4147a376a747cca053c8c43c65fc68474e0"
+  digest: "sha512:7b5c6f3691b88293cf94091c1d74629ee3227a5f5548d24497309d00f91338800d315d4a907b6c45a10d1a6ceb2bb7dc5b8a72d9be2e251f6979886ae0bf1274"
 
 mounts:
 - location: "~"
diff --git a/go.mod b/go.mod
@@ -16,6 +16,7 @@ require (
 	github.com/lima-vm/sshocker v0.2.2
 	github.com/mattn/go-isatty v0.0.14
 	github.com/mattn/go-shellwords v1.0.12
+	github.com/miekg/dns v1.1.43
 	github.com/norouter/norouter v0.6.4
 	github.com/nxadm/tail v1.4.8
 	github.com/opencontainers/go-digest v1.0.0
@@ -46,6 +47,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
 	golang.org/x/text v0.3.6 // indirect
diff --git a/go.sum b/go.sum
@@ -580,6 +580,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1f
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mibk/dupl v1.0.0/go.mod h1:pCr4pNxxIbFGvtyCOi0c7LVjmV6duhKWV+ex5vh38ME=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
diff --git a/pkg/cidata/cidata.TEMPLATE.d/boot/07-host-dns-setup.sh b/pkg/cidata/cidata.TEMPLATE.d/boot/07-host-dns-setup.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eux
+
+# Wait until iptables has been installed; 30-install-packages.sh will call this script again
+if command -v iptables >/dev/null 2>&1; then
+	if [ "${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}" -ne 0 ]; then
+		# Only add the rule once
+		if ! iptables-save | grep "udp.*${LIMA_CIDATA_SLIRP_GATEWAY}:${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}"; then
+			iptables -t nat -A OUTPUT -d "${LIMA_CIDATA_SLIRP_DNS}" -p udp --dport 53 -j DNAT \
+				--to-destination "${LIMA_CIDATA_SLIRP_GATEWAY}:${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}"
+		fi
+	fi
+fi
diff --git a/pkg/cidata/cidata.TEMPLATE.d/boot/30-install-packages.sh b/pkg/cidata/cidata.TEMPLATE.d/boot/30-install-packages.sh
@@ -10,6 +10,14 @@ update_fuse_conf() {
 	fi
 }
 
+INSTALL_IPTABLES=0
+if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ] || [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+	INSTALL_IPTABLES=1
+fi
+if [ "${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}" -ne 0 ]; then
+	INSTALL_IPTABLES=1
+fi
+
 # Install minimum dependencies
 if command -v apt-get >/dev/null 2>&1; then
 	DEBIAN_FRONTEND=noninteractive
@@ -20,7 +28,7 @@ if command -v apt-get >/dev/null 2>&1; then
 			apt-get install -y sshfs
 		fi
 	fi
-	if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ] || [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+	if [ "${INSTALL_IPTABLES}" = 1 ]; then
 		if [ ! -e /usr/sbin/iptables ]; then
 			apt-get install -y iptables
 		fi
@@ -36,7 +44,7 @@ elif command -v dnf >/dev/null 2>&1; then
 			dnf install -y fuse-sshfs
 		fi
 	fi
-	if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ] || [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+	if [ "${INSTALL_IPTABLES}" = 1 ]; then
 		if [ ! -e /usr/sbin/iptables ]; then
 			dnf install -y iptables
 		fi
@@ -63,7 +71,7 @@ elif command -v zypper >/dev/null 2>&1; then
 			zypper install -y sshfs
 		fi
 	fi
-	if [ "${LIMA_CIDATA_CONTAINERD_SYSTEM}" = 1 ] || [ "${LIMA_CIDATA_CONTAINERD_USER}" = 1 ]; then
+	if [ "${INSTALL_IPTABLES}" = 1 ]; then
 		if [ ! -e /usr/sbin/iptables ]; then
 			zypper install -y iptables
 		fi
@@ -80,6 +88,17 @@ elif command -v apk >/dev/null 2>&1; then
 			apk add sshfs
 		fi
 	fi
+	if [ "${INSTALL_IPTABLES}" = 1 ]; then
+		if ! command -v iptables >/dev/null 2>&1; then
+			apk update
+			apk add iptables
+		fi
+	fi
+fi
+
+if [ "${LIMA_CIDATA_UDP_DNS_LOCAL_PORT}" -ne 0 ]; then
+	# Try to setup iptables rule again, in case we just installed iptables
+	"${LIMA_CIDATA_MNT}/boot/07-host-dns-setup.sh"
 fi
 
 # update_fuse_conf has to be called after installing all the packages,
diff --git a/pkg/cidata/cidata.TEMPLATE.d/lima.env b/pkg/cidata/cidata.TEMPLATE.d/lima.env
@@ -14,4 +14,6 @@ LIMA_CIDATA_CONTAINERD_SYSTEM=1
 {{- else}}
 LIMA_CIDATA_CONTAINERD_SYSTEM=
 {{- end}}
-LIMA_CIDATA_SLIRP_GATEWAY={{ .SlirpGateway }}
+LIMA_CIDATA_SLIRP_DNS={{.SlirpDNS}}
+LIMA_CIDATA_SLIRP_GATEWAY={{.SlirpGateway}}
+LIMA_CIDATA_UDP_DNS_LOCAL_PORT={{.UDPDNSLocalPort}}
diff --git a/pkg/cidata/cidata.TEMPLATE.d/user-data b/pkg/cidata/cidata.TEMPLATE.d/user-data
@@ -42,5 +42,4 @@ resolv_conf:
   {{- range $ns := $.DNSAddresses }}
   - {{$ns}}
   {{- end }}
-  - {{.SlirpDNS}}
 {{- end }}
diff --git a/pkg/cidata/cidata.go b/pkg/cidata/cidata.go
@@ -90,13 +90,13 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML) error {
 		return err
 	}
 	args := TemplateArgs{
-		Name:         name,
-		User:         u.Username,
-		UID:          uid,
-		Containerd:   Containerd{System: *y.Containerd.System, User: *y.Containerd.User},
-		SlirpNICName: qemu.SlirpNICName,
-		SlirpGateway: qemu.SlirpGateway,
-		SlirpDNS:     qemu.SlirpDNS,
+		Name:            name,
+		User:            u.Username,
+		UID:             uid,
+		Containerd:      Containerd{System: *y.Containerd.System, User: *y.Containerd.User},
+		SlirpNICName:    qemu.SlirpNICName,
+		SlirpGateway:    qemu.SlirpGateway,
+		SlirpDNS:        qemu.SlirpDNS,
 	}
 
 	// change instance id on every boot so network config will be processed again
@@ -131,7 +131,10 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML) error {
 	if err != nil {
 		return err
 	}
-	if len(y.DNS) > 0 {
+	if *y.UseHostResolver {
+		args.UDPDNSLocalPort = y.SSH.LocalPort
+		args.DNSAddresses = append(args.DNSAddresses, qemu.SlirpDNS)
+	} else if len(y.DNS) > 0 {
 		for _, addr := range y.DNS {
 			args.DNSAddresses = append(args.DNSAddresses, addr.String())
 		}
diff --git a/pkg/cidata/template.go b/pkg/cidata/template.go
@@ -28,19 +28,20 @@ type Network struct {
 	Interface  string
 }
 type TemplateArgs struct {
-	Name         string // instance name
-	IID          string // instance id
-	User         string // user name
-	UID          int
-	SSHPubKeys   []string
-	Mounts       []string // abs path, accessible by the User
-	Containerd   Containerd
-	Networks     []Network
-	SlirpNICName string
-	SlirpGateway string
-	SlirpDNS     string
-	Env          map[string]string
-	DNSAddresses []string
+	Name            string // instance name
+	IID             string // instance id
+	User            string // user name
+	UID             int
+	SSHPubKeys      []string
+	Mounts          []string // abs path, accessible by the User
+	Containerd      Containerd
+	Networks        []Network
+	SlirpNICName    string
+	SlirpGateway    string
+	SlirpDNS        string
+	UDPDNSLocalPort int
+	Env             map[string]string
+	DNSAddresses    []string
 }
 
 func ValidateTemplateArgs(args TemplateArgs) error {
diff --git a/pkg/hostagent/dns.go b/pkg/hostagent/dns.go
@@ -0,0 +1,123 @@
+// This file has been adapted from https://github.com/norouter/norouter/blob/v0.6.4/pkg/agent/dns/dns.go
+
+package hostagent
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+	"github.com/sirupsen/logrus"
+)
+
+type Handler struct {
+	clientConfig *dns.ClientConfig
+	clients      []*dns.Client
+}
+
+func newStaticClientConfig(ips []net.IP) (*dns.ClientConfig, error) {
+	s := ``
+	for _, ip := range ips {
+		s += fmt.Sprintf("nameserver %s\n", ip.String())
+	}
+	r := strings.NewReader(s)
+	return dns.ClientConfigFromReader(r)
+}
+
+func newHandler() (dns.Handler, error) {
+	cc, err := dns.ClientConfigFromFile("/etc/resolv.conf")
+	if err != nil {
+		fallbackIPs := []net.IP{net.ParseIP("8.8.8.8"), net.ParseIP("1.1.1.1")}
+		logrus.WithError(err).Warnf("failed to detect system DNS, falling back to %v", fallbackIPs)
+		cc, err = newStaticClientConfig(fallbackIPs)
+		if err != nil {
+			return nil, err
+		}
+	}
+	clients := []*dns.Client{
+		{}, // UDP
+		{Net: "tcp"},
+	}
+	h := &Handler{
+		clientConfig: cc,
+		clients:      clients,
+	}
+	return h, nil
+}
+
+func (h *Handler) handleQuery(w dns.ResponseWriter, req *dns.Msg) {
+	var (
+		reply   dns.Msg
+		handled bool
+	)
+	reply.SetReply(req)
+	for _, q := range reply.Question {
+		switch q.Qtype {
+		case dns.TypeA:
+			addrs, err := net.LookupIP(q.Name)
+			if err == nil && len(addrs) > 0 {
+				for _, ip := range addrs {
+					if ip.To4() != nil {
+						a := &dns.A{
+							Hdr: dns.RR_Header{
+								Name:   q.Name,
+								Rrtype: dns.TypeA,
+								Class:  dns.ClassINET,
+							},
+							A: ip.To4(),
+						}
+						reply.Answer = append(reply.Answer, a)
+						handled = true
+						break
+					}
+				}
+			}
+		}
+	}
+	if handled {
+		_ = w.WriteMsg(&reply)
+		return
+	}
+	h.handleDefault(w, req)
+}
+
+func (h *Handler) handleDefault(w dns.ResponseWriter, req *dns.Msg) {
+	for _, client := range h.clients {
+		for _, srv := range h.clientConfig.Servers {
+			addr := fmt.Sprintf("%s:%s", srv, h.clientConfig.Port)
+			reply, _, err := client.Exchange(req, addr)
+			if err == nil {
+				_ = w.WriteMsg(reply)
+				return
+			}
+		}
+	}
+	var reply dns.Msg
+	reply.SetReply(req)
+	_ = w.WriteMsg(&reply)
+}
+
+func (h *Handler) ServeDNS(w dns.ResponseWriter, req *dns.Msg) {
+	switch req.Opcode {
+	case dns.OpcodeQuery:
+		h.handleQuery(w, req)
+	default:
+		h.handleDefault(w, req)
+	}
+}
+
+func (a *HostAgent) StartDNS() (*dns.Server, error) {
+	h, err := newHandler()
+	if err != nil {
+		panic(err)
+	}
+	addr := fmt.Sprintf("127.0.0.1:%d", a.y.SSH.LocalPort)
+	server := &dns.Server{Net: "udp", Addr: addr, Handler: h}
+	go func() {
+		if e := server.ListenAndServe(); e != nil {
+			panic(e)
+		}
+	}()
+	return server, nil
+}
diff --git a/pkg/hostagent/hostagent.go b/pkg/hostagent/hostagent.go
@@ -143,6 +143,14 @@ func (a *HostAgent) Run(ctx context.Context) error {
 		a.emitEvent(ctx, exitingEv)
 	}()
 
+	if *a.y.UseHostResolver {
+		dnsServer, err := a.StartDNS()
+		if err != nil {
+			return fmt.Errorf("cannot start DNS server: %w", err)
+		}
+		defer func() { _ = dnsServer.Shutdown() }()
+	}
+
 	qCmd := exec.CommandContext(ctx, a.qExe, a.qArgs...)
 	qStdout, err := qCmd.StdoutPipe()
 	if err != nil {
diff --git a/pkg/limayaml/default.yaml b/pkg/limayaml/default.yaml
@@ -184,6 +184,14 @@ networks:
 # env:
 #   KEY: value
 
+# The host agent implements a DNS server that looks up host names on the host
+# using the local system resolver. This means changing VPN and network settings
+# are reflected automatically into the guest, including conditional forward,
+# and mDNS lookup:
+# Default: true
+useHostResolver: true
+
+# If useHostResolver is false, then the following rules apply for configuring dns:
 # Explicitly set DNS addresses for qemu user-mode networking. By default qemu picks *one*
 # nameserver from the host config and forwards all queries to this server. On macOS
 # Lima adds the nameservers configured for the "en0" interface to the list. In case this
diff --git a/pkg/limayaml/defaults.go b/pkg/limayaml/defaults.go
diff --git a/pkg/limayaml/limayaml.go b/pkg/limayaml/limayaml.go
diff --git a/pkg/limayaml/validate.go b/pkg/limayaml/validate.go
diff --git a/pkg/sysprof/sysprof_darwin.go b/pkg/sysprof/sysprof_darwin.go
