Merge pull request #3072 from jandubois/based-on

Assemble templates using the new `base` setting

Author: AkihiroSuda

.github/workflows/test.yml

@@ -253,7 +253,8 @@ jobs:
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
       with:
-        fetch-depth: 1
+        # To avoid "failed to load YAML file \"templates/experimental/riscv64.yaml\": can't parse builtin Lima version \"3f3a6f6\": 3f3a6f6 is not in dotted-tri format"
+        fetch-depth: 0
     - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34  # v5.3.0
       with:
         go-version: 1.24.x

cmd/limactl/start.go

@@ -205,7 +205,9 @@ func loadOrCreateInstance(cmd *cobra.Command, args []string, createOnly bool) (*
 			return nil, err
 		}
 	}
-
+	if err := tmpl.Embed(cmd.Context(), true, true); err != nil {
+		return nil, err
+	}
 	yqExprs, err := editflags.YQExpressions(flags, true)
 	if err != nil {
 		return nil, err

cmd/limactl/template.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -26,6 +27,9 @@ func newTemplateCommand() *cobra.Command {
 		// The template command is still hidden because the subcommands and options are still under development
 		// and subject to change at any time.
 		Hidden: true,
+		PreRun: func(*cobra.Command, []string) {
+			logrus.Warn("`limactl template` is experimental")
+		},
 	}
 	templateCommand.AddCommand(
 		newTemplateCopyCommand(),
@@ -46,30 +50,89 @@ var templateCopyExample = `  Template locators are local files, file://, https:/
   # Copy default template to STDOUT
   limactl template copy template://default -
 
-  # Copy template from web location to local file
-  limactl template copy https://example.com/lima.yaml mighty-machine.yaml
+  # Copy template from web location to local file and embed all external references
+  # (this does not embed template:// references)
+  limactl template copy --embed https://example.com/lima.yaml mighty-machine.yaml
 `
 
 func newTemplateCopyCommand() *cobra.Command {
 	templateCopyCommand := &cobra.Command{
-		Use:     "copy TEMPLATE DEST",
+		Use:     "copy [OPTIONS] TEMPLATE DEST",
 		Short:   "Copy template",
 		Long:    "Copy a template via locator to a local file",
 		Example: templateCopyExample,
 		Args:    WrapArgsError(cobra.ExactArgs(2)),
 		RunE:    templateCopyAction,
 	}
+	templateCopyCommand.Flags().Bool("embed", false, "embed external dependencies into template")
+	templateCopyCommand.Flags().Bool("embed-all", false, "embed all dependencies into template")
+	templateCopyCommand.Flags().Bool("fill", false, "fill defaults")
+	templateCopyCommand.Flags().Bool("verbatim", false, "don't make locators absolute")
 	return templateCopyCommand
 }
 
 func templateCopyAction(cmd *cobra.Command, args []string) error {
+	embed, err := cmd.Flags().GetBool("embed")
+	if err != nil {
+		return err
+	}
+	embedAll, err := cmd.Flags().GetBool("embed-all")
+	if err != nil {
+		return err
+	}
+	fill, err := cmd.Flags().GetBool("fill")
+	if err != nil {
+		return err
+	}
+	verbatim, err := cmd.Flags().GetBool("verbatim")
+	if err != nil {
+		return err
+	}
+	if fill {
+		embedAll = true
+	}
+	if embedAll {
+		embed = true
+	}
+	if embed && verbatim {
+		return errors.New("--verbatim cannot be used with any of --embed, --embed-all, or --fill")
+	}
 	tmpl, err := limatmpl.Read(cmd.Context(), "", args[0])
 	if err != nil {
 		return err
 	}
 	if len(tmpl.Bytes) == 0 {
 		return fmt.Errorf("don't know how to interpret %q as a template locator", args[0])
 	}
+	if !verbatim {
+		if embed {
+			// Embed default base.yaml only when fill is true.
+			if err := tmpl.Embed(cmd.Context(), embedAll, fill); err != nil {
+				return err
+			}
+		} else {
+			if err := tmpl.UseAbsLocators(); err != nil {
+				return err
+			}
+		}
+	}
+	if fill {
+		limaDir, err := dirnames.LimaDir()
+		if err != nil {
+			return err
+		}
+		// Load() will merge the template with override.yaml and default.yaml via FillDefaults().
+		// FillDefaults() needs the potential instance directory to validate host templates using {{.Dir}}.
+		filePath := filepath.Join(limaDir, tmpl.Name+".yaml")
+		tmpl.Config, err = limayaml.Load(tmpl.Bytes, filePath)
+		if err != nil {
+			return err
+		}
+		tmpl.Bytes, err = limayaml.Marshal(tmpl.Config, false)
+		if err != nil {
+			return err
+		}
+	}
 	writer := cmd.OutOrStdout()
 	target := args[1]
 	if target != "-" {
@@ -116,10 +179,14 @@ func templateValidateAction(cmd *cobra.Command, args []string) error {
 		if tmpl.Name == "" {
 			return fmt.Errorf("can't determine instance name from template locator %q", arg)
 		}
+		// Embed default base.yaml only when fill is true.
+		if err := tmpl.Embed(cmd.Context(), true, fill); err != nil {
+			return err
+		}
 		// Load() will merge the template with override.yaml and default.yaml via FillDefaults().
 		// FillDefaults() needs the potential instance directory to validate host templates using {{.Dir}}.
-		instDir := filepath.Join(limaDir, tmpl.Name)
-		y, err := limayaml.Load(tmpl.Bytes, instDir)
+		filePath := filepath.Join(limaDir, tmpl.Name+".yaml")
+		y, err := limayaml.Load(tmpl.Bytes, filePath)
 		if err != nil {
 			return err
 		}

hack/cache-common-inc.sh

@@ -18,17 +18,18 @@ function error_exit() {
 # ```
 function download_template_if_needed() {
 	local template="$1"
-	case "${template}" in
-	https://*)
-		tmp_yaml=$(mktemp -d)/template.yaml
-		curl -sSLf "${template}" >"${tmp_yaml}" || return
-		echo "${tmp_yaml}"
-		;;
-	*)
-		test -f "${template}" || return
-		echo "${template}"
-		;;
-	esac
+	tmp_yaml="$(mktemp -d)/template.yaml"
+	# The upgrade test doesn't have limactl installed first. The old version wouldn't support `limactl tmpl` anyways.
+	if command -v limactl >/dev/null; then
+		limactl tmpl copy --embed-all "${template}" "${tmp_yaml}" || return
+	else
+		if [[ $template == https://* ]]; then
+			curl -sSLf "${template}" >"${tmp_yaml}" || return
+		else
+			cp "${template}" "${tmp_yaml}"
+		fi
+	fi
+	echo "${tmp_yaml}"
 }
 
 # e.g.

hack/test-port-forwarding.pl

@@ -212,11 +212,11 @@ sub JoinHostPort {
 # "ipv4" and "ipv6" will be replaced by the actual host ipv4 and ipv6 addresses.
 __DATA__
 portForwards:
-  # We can't test that port 22 will be blocked because the guestagent has
-  # been ignoring it since startup, so the log message is in the part of
-  # the log we skipped.
-  # skip: 127.0.0.1 22 → 127.0.0.1 2222
-  # ignore: 127.0.0.1 sshLocalPort
+# We can't test that port 22 will be blocked because the guestagent has
+# been ignoring it since startup, so the log message is in the part of
+# the log we skipped.
+# skip: 127.0.0.1 22 → 127.0.0.1 2222
+# ignore: 127.0.0.1 sshLocalPort
 
 - guestIP: 127.0.0.2
   guestPortRange: [3000, 3009]

hack/test-templates.sh

@@ -17,7 +17,8 @@ if [ "$#" -ne 1 ]; then
 	exit 1
 fi
 
-FILE="$1"
+# Resolve any ../ fragments in the filename because they are invalid in relative template locators
+FILE="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
 NAME="$(basename -s .yaml "$FILE")"
 OS_HOST="$(uname -o)"
 
@@ -135,6 +136,9 @@ if [[ -n ${CHECKS["port-forwards"]} ]]; then
 	limactl validate "$FILE_HOST"
 fi
 
+INFO "Make sure template embedding copies \"$FILE_HOST\" exactly"
+diff -u <(echo -n "base: $FILE_HOST" | limactl tmpl copy --embed - -) "$FILE_HOST"
+
 function diagnose() {
 	NAME="$1"
 	set -x +e
@@ -207,7 +211,7 @@ fi
 
 if [[ -n ${CHECKS["set-user"]} ]]; then
 	INFO 'Testing that user settings can be provided by lima.yaml'
-	limactl shell "$NAME" grep "^john:x:4711:4711:John Doe:/home/john-john" /etc/passwd
+	limactl shell "$NAME" grep "^john:x:4711:4711:John Doe:/home/john-john:/usr/bin/bash" /etc/passwd
 fi
 
 if [[ -n ${CHECKS["proxy-settings"]} ]]; then

hack/test-templates/test-misc.yaml

@@ -3,21 +3,7 @@
 # - snapshots
 # - (More to come)
 #
-# This template requires Lima v1.0.0-alpha.0 or later.
-images:
-# Try to use release-yyyyMMdd image if available. Note that release-yyyyMMdd will be removed after several months.
-- location: "https://cloud-images.ubuntu.com/releases/22.04/release-20220902/ubuntu-22.04-server-cloudimg-amd64.img"
-  arch: "x86_64"
-  digest: "sha256:c777670007cc5f132417b9e0bc01367ccfc2a989951ffa225bb1952917c3aa81"
-- location: "https://cloud-images.ubuntu.com/releases/22.04/release-20220902/ubuntu-22.04-server-cloudimg-arm64.img"
-  arch: "aarch64"
-  digest: "sha256:9620f479bd5a6cbf1e805654d41b27f4fc56ef20f916c8331558241734de81ae"
-# Fallback to the latest release image.
-# Hint: run `limactl prune` to invalidate the cache
-- location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img"
-  arch: "x86_64"
-- location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img"
-  arch: "aarch64"
+base: template://ubuntu-22.04
 
 # 9p is not compatible with `limactl snapshot`
 mountTypesUnsupported: ["9p"]
@@ -26,8 +12,6 @@ mounts:
   writable: true
 - location: "/tmp/lima test dir with spaces"
   writable: true
-- location: "/tmp/lima"
-  writable: true
 
 param:
   BOOT: boot
@@ -64,3 +48,5 @@ user:
   comment: John Doe
   home: "/home/{{.User}}-{{.User}}"
   uid: 4711
+  # Ubuntu has identical /bin/bash and /usr/bin/bash
+  shell: /usr/bin/bash

pkg/limatmpl/abs.go

@@ -0,0 +1,131 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package limatmpl
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+// UseAbsLocators will replace all relative template locators with absolute ones, so this template
+// can be stored anywhere and still reference the same base templates and files.
+func (tmpl *Template) UseAbsLocators() error {
+	err := tmpl.useAbsLocators()
+	return tmpl.ClearOnError(err)
+}
+
+func (tmpl *Template) useAbsLocators() error {
+	if err := tmpl.Unmarshal(); err != nil {
+		return err
+	}
+	basePath, err := basePath(tmpl.Locator)
+	if err != nil {
+		return err
+	}
+	for i, baseLocator := range tmpl.Config.Base {
+		absLocator, err := absPath(baseLocator.URL, basePath)
+		if err != nil {
+			return err
+		}
+		if i == 0 {
+			// base can be either a single string (URL), or a single locator object, or a list whose first element can be either a string or an object
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.base | select(type == \"!!str\")) |= %q\n", absLocator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.base | select(type == \"!!map\") | .url) |= %q\n", absLocator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.base | select(type == \"!!seq\" and (.[0] | type) == \"!!str\") | .[0]) |= %q\n", absLocator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.base | select(type == \"!!seq\" and (.[0] | type) == \"!!map\") | .[0].url) |= %q\n", absLocator))
+		} else {
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.base[%d] | select(type == \"!!str\")) |= %q\n", i, absLocator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.base[%d] | select(type == \"!!map\") | .url) |= %q\n", i, absLocator))
+		}
+	}
+	for i, p := range tmpl.Config.Probes {
+		if p.File != nil {
+			absLocator, err := absPath(p.File.URL, basePath)
+			if err != nil {
+				return err
+			}
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.probes[%d].file | select(type == \"!!str\")) = %q\n", i, absLocator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.probes[%d].file | select(type == \"!!map\") | .url) = %q\n", i, absLocator))
+		}
+	}
+	for i, p := range tmpl.Config.Provision {
+		if p.File != nil {
+			absLocator, err := absPath(p.File.URL, basePath)
+			if err != nil {
+				return err
+			}
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.provision[%d].file | select(type == \"!!str\")) = %q\n", i, absLocator))
+			tmpl.expr.WriteString(fmt.Sprintf("| ($a.provision[%d].file | select(type == \"!!map\") | .url) = %q\n", i, absLocator))
+		}
+	}
+	return tmpl.evalExpr()
+}
+
+// withVolume adds the volume name of the current working directory to a path without volume name.
+// On Windows filepath.Abs() only returns a "rooted" name, but does not add the volume name.
+// withVolume also normalizes all path separators to the platform native one.
+func withVolume(path string) (string, error) {
+	if runtime.GOOS == "windows" && filepath.VolumeName(path) == "" {
+		root, err := filepath.Abs("/")
+		if err != nil {
+			return "", err
+		}
+		path = filepath.VolumeName(root) + path
+	}
+	return filepath.Clean(path), nil
+}
+
+// basePath returns the locator in absolute format, but without the filename part.
+func basePath(locator string) (string, error) {
+	u, err := url.Parse(locator)
+	// Single-letter schemes will be drive names on Windows, e.g. "c:/foo"
+	if err == nil && len(u.Scheme) > 1 {
+		// path.Dir("") returns ".", which must be removed for url.JoinPath() to do the right thing later
+		return u.Scheme + "://" + strings.TrimSuffix(path.Dir(path.Join(u.Host, u.Path)), "."), nil
+	}
+	base, err := filepath.Abs(filepath.Dir(locator))
+	if err != nil {
+		return "", err
+	}
+	return withVolume(base)
+}
+
+// absPath either returns the locator directly, or combines it with the basePath if the locator is a relative path.
+func absPath(locator, basePath string) (string, error) {
+	if locator == "" {
+		return "", errors.New("locator is empty")
+	}
+	u, err := url.Parse(locator)
+	if err == nil && len(u.Scheme) > 1 {
+		return locator, nil
+	}
+	// Check for rooted locator; filepath.IsAbs() returns false on Windows when the volume name is missing
+	volumeLen := len(filepath.VolumeName(locator))
+	if locator[volumeLen] != '/' && locator[volumeLen] != filepath.Separator {
+		switch {
+		case basePath == "":
+			return "", errors.New("basePath is empty")
+		case basePath == "-":
+			return "", errors.New("can't use relative paths when reading template from STDIN")
+		case strings.Contains(locator, "../"):
+			return "", fmt.Errorf("relative locator path %q must not contain '../' segments", locator)
+		case volumeLen != 0:
+			return "", fmt.Errorf("relative locator path %q must not include a volume name", locator)
+		}
+		u, err = url.Parse(basePath)
+		if err != nil {
+			return "", err
+		}
+		if len(u.Scheme) > 1 {
+			return u.JoinPath(locator).String(), nil
+		}
+		locator = filepath.Join(basePath, locator)
+	}
+	return withVolume(locator)
+}