frontend: add required paths to LLB and use it with --parents

cache/contenthash/checksum.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"sync/atomic"
 
+	cerrdefs "github.com/containerd/errdefs"
 	iradix "github.com/hashicorp/go-immutable-radix/v2"
 	simplelru "github.com/hashicorp/golang-lru/v2/simplelru"
 	"github.com/moby/buildkit/cache"
@@ -59,6 +60,7 @@ type ChecksumOpts struct {
 	Wildcard        bool
 	IncludePatterns []string
 	ExcludePatterns []string
+	RequiredPaths   []string
 }
 
 func Checksum(ctx context.Context, ref cache.ImmutableRef, path string, opts ChecksumOpts, s session.Group) (digest.Digest, error) {
@@ -690,6 +692,17 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
 	cc.tree = txn.Commit()
 	cc.dirty = updated
 
+	// Validate that all required paths exist.
+	for _, requiredPath := range opts.RequiredPaths {
+		found := slices.ContainsFunc(includedPaths, func(includedPath *includedPath) bool {
+			return strings.HasPrefix(includedPath.path, requiredPath)
+		})
+
+		if !found {
+			return "", nil, errors.Wrapf(cerrdefs.ErrNotFound, "%q", requiredPath)
+		}
+	}
+
 	return origPrefix, includedPaths, nil
 }
 

client/llb/fileop.go

@@ -569,6 +569,7 @@ type CopyInfo struct {
 	CopyDirContentsOnly            bool
 	IncludePatterns                []string
 	ExcludePatterns                []string
+	RequiredPaths                  []string
 	AttemptUnpack                  bool
 	CreateDestPath                 bool
 	AllowWildcard                  bool
@@ -603,6 +604,7 @@ func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base
 		Owner:                            a.info.ChownOpt.marshal(base),
 		IncludePatterns:                  a.info.IncludePatterns,
 		ExcludePatterns:                  a.info.ExcludePatterns,
+		RequiredPaths:                    a.info.RequiredPaths,
 		AllowWildcard:                    a.info.AllowWildcard,
 		AllowEmptyWildcard:               a.info.AllowEmptyWildcard,
 		FollowSymlink:                    a.info.FollowSymlinks,
@@ -647,6 +649,9 @@ func (a *fileActionCopy) addCaps(f *FileOp) {
 	if len(a.info.IncludePatterns) != 0 || len(a.info.ExcludePatterns) != 0 {
 		addCap(&f.constraints, pb.CapFileCopyIncludeExcludePatterns)
 	}
+	if len(a.info.RequiredPaths) != 0 {
+		addCap(&f.constraints, pb.CapFileCopyRequiredPaths)
+	}
 	if a.info.AlwaysReplaceExistingDestPaths {
 		addCap(&f.constraints, pb.CapFileCopyAlwaysReplaceExistingDestPaths)
 	}

frontend/dockerfile/dockerfile2llb/convert.go

@@ -1223,7 +1223,7 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
 	// Run command can potentially access any file. Mark the full filesystem as used.
 	d.paths["/"] = struct{}{}
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if len(c.Files) > 0 {
 		if len(args) != 1 || !c.PrependShell {
 			return errors.Errorf("parsing produced an invalid run command: %v", args)
@@ -1606,6 +1606,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		} else {
 			validateCopySourcePath(src, &cfg)
 			var patterns []string
+			var requiredPaths []string
 			if cfg.parents {
 				// detect optional pivot point
 				parent, pattern, ok := strings.Cut(src, "/./")
@@ -1622,6 +1623,12 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				}
 
 				patterns = []string{strings.TrimPrefix(pattern, "/")}
+
+				// determine if we want to require any paths to exist.
+				// we only require a path to exist if wildcards aren't present.
+				if !containsWildcards(src) && !containsWildcards(pattern) {
+					requiredPaths = []string{filepath.Join(src, pattern)}
+				}
 			}
 
 			src, err = system.NormalizePath("/", src, d.platform.OS, false)
@@ -1639,6 +1646,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				FollowSymlinks:      true,
 				CopyDirContentsOnly: true,
 				IncludePatterns:     patterns,
+				RequiredPaths:       requiredPaths,
 				AttemptUnpack:       unpack,
 				CreateDestPath:      true,
 				AllowWildcard:       true,
@@ -1760,7 +1768,7 @@ func dispatchOnbuild(d *dispatchState, c *instructions.OnbuildCommand) error {
 func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Linter) error {
 	validateUsedOnce(c, &d.cmd, lint)
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if c.PrependShell {
 		if len(d.image.Config.Shell) == 0 {
 			msg := linter.RuleJSONArgsRecommended.Format(c.Name())
@@ -1776,7 +1784,7 @@ func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Lint
 func dispatchEntrypoint(d *dispatchState, c *instructions.EntrypointCommand, lint *linter.Linter) error {
 	validateUsedOnce(c, &d.entrypoint, lint)
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if c.PrependShell {
 		if len(d.image.Config.Shell) == 0 {
 			msg := linter.RuleJSONArgsRecommended.Format(c.Name())
@@ -2692,3 +2700,15 @@ func (emptyEnvs) Get(string) (string, bool) {
 func (emptyEnvs) Keys() []string {
 	return nil
 }
+
+func containsWildcards(name string) bool {
+	for i := 0; i < len(name); i++ {
+		switch name[i] {
+		case '*', '?', '[':
+			return true
+		case '\\':
+			i++
+		}
+	}
+	return false
+}

frontend/dockerfile/dockerfile_parents_test.go

@@ -18,6 +18,7 @@ import (
 var parentsTests = integration.TestFuncs(
 	testCopyParents,
 	testCopyRelativeParents,
+	testCopyParentsMissingDirectory,
 )
 
 func init() {
@@ -180,3 +181,100 @@ eot
 		require.NoError(t, err)
 	}
 }
+
+func testCopyParentsMissingDirectory(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM alpine AS base
+WORKDIR /test
+RUN <<eot
+	set -ex
+	mkdir -p a/b/c/d/e
+	touch a/b/c/d/foo
+	touch a/b/c/d/e/bay
+eot
+
+FROM alpine AS normal
+COPY --from=base --parents /test/a/b/c/d /out/
+RUN <<eot
+	set -ex
+	[ -d /out/test/a/b/c/d/e ]
+	[ -f /out/test/a/b/c/d/e/bay ]
+	[ ! -d /out/e ]
+	[ ! -d /out/a ]
+eot
+
+FROM alpine AS withpivot
+COPY --from=base --parents /test/a/b/./c/d /out/
+RUN <<eot
+	set -ex
+	[ -d /out/c/d/e ]
+	[ -f /out/c/d/foo ]
+	[ ! -d /out/a ]
+	[ ! -d /out/e ]
+eot
+
+FROM alpine AS nonexistentfile
+COPY --from=base --parents /test/nonexistent-file /out/
+
+FROM alpine AS wildcard-nonexistent
+COPY --from=base --parents /test/a/b2*/c /out/
+RUN <<eot
+	set -ex
+	[ -d /out ]
+	[ ! -d /out/a ]
+eot
+
+FROM alpine AS wildcard-afterpivot
+COPY --from=base --parents /test/a/b/./c2* /out/
+RUN <<eot
+	set -ex
+	[ -d /out ]
+	[ ! -d /out/a ]
+	[ ! -d /out/c* ]
+eot
+`)
+
+	dir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	type test struct {
+		target     string
+		errorRegex any
+	}
+
+	tests := []test{
+		{"normal", nil},
+		{"withpivot", nil},
+		{"nonexistentfile", `failed to calculate checksum of ref.*: "/test/nonexistent-file": not found`},
+		{"wildcard-nonexistent", nil},
+		{"wildcard-afterpivot", nil},
+	}
+
+	for _, tt := range tests {
+		t.Logf("target: %s", tt.target)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			FrontendAttrs: map[string]string{
+				"target": tt.target,
+			},
+			LocalMounts: map[string]fsutil.FS{
+				dockerui.DefaultLocalNameDockerfile: dir,
+				dockerui.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+
+		if tt.errorRegex != nil {
+			require.Error(t, err)
+			require.Regexp(t, tt.errorRegex, err.Error())
+		} else {
+			require.NoError(t, err)
+		}
+	}
+}

solver/llbsolver/ops/file.go

@@ -105,7 +105,7 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol
 			markInvalid(action.Input)
 			processOwner(p.Owner, selectors)
 			if action.SecondaryInput != -1 && int(action.SecondaryInput) < f.numInputs {
-				addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink, p.IncludePatterns, p.ExcludePatterns)
+				addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink, p.IncludePatterns, p.ExcludePatterns, p.RequiredPaths)
 				p.Src = path.Base(p.Src)
 			}
 			dt, err = json.Marshal(p)
@@ -215,13 +215,14 @@ func (f *fileOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) {
 	}, nil
 }
 
-func addSelector(m map[int][]opsutils.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns []string) {
+func addSelector(m map[int][]opsutils.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns, requiredPaths []string) {
 	s := opsutils.Selector{
 		Path:            sel,
 		FollowLinks:     followLinks,
 		Wildcard:        wildcard && containsWildcards(sel),
 		IncludePatterns: includePatterns,
 		ExcludePatterns: excludePatterns,
+		RequiredPaths:   requiredPaths,
 	}
 
 	m[idx] = append(m[idx], s)
@@ -284,15 +285,15 @@ func processOwner(chopt *pb.ChownOpt, selectors map[int][]opsutils.Selector) err
 			if u.ByName.Input < 0 {
 				return errors.Errorf("invalid user index %d", u.ByName.Input)
 			}
-			addSelector(selectors, int(u.ByName.Input), "/etc/passwd", false, true, nil, nil)
+			addSelector(selectors, int(u.ByName.Input), "/etc/passwd", false, true, nil, nil, nil)
 		}
 	}
 	if chopt.Group != nil {
 		if u, ok := chopt.Group.User.(*pb.UserOpt_ByName); ok {
 			if u.ByName.Input < 0 {
 				return errors.Errorf("invalid user index %d", u.ByName.Input)
 			}
-			addSelector(selectors, int(u.ByName.Input), "/etc/group", false, true, nil, nil)
+			addSelector(selectors, int(u.ByName.Input), "/etc/group", false, true, nil, nil, nil)
 		}
 	}
 	return nil

solver/llbsolver/ops/opsutils/contenthash.go

@@ -21,6 +21,7 @@ type Selector struct {
 	FollowLinks     bool
 	IncludePatterns []string
 	ExcludePatterns []string
+	RequiredPaths   []string
 }
 
 func (sel Selector) HasWildcardOrFilters() bool {
@@ -52,6 +53,7 @@ func NewContentHashFunc(selectors []Selector) solver.ResultBasedCacheFunc {
 						FollowLinks:     sel.FollowLinks,
 						IncludePatterns: sel.IncludePatterns,
 						ExcludePatterns: sel.ExcludePatterns,
+						RequiredPaths:   sel.RequiredPaths,
 					},
 					s,
 				)

solver/pb/caps.go

@@ -73,6 +73,7 @@ const (
 	CapFileBase                               apicaps.CapID = "file.base"
 	CapFileRmWildcard                         apicaps.CapID = "file.rm.wildcard"
 	CapFileCopyIncludeExcludePatterns         apicaps.CapID = "file.copy.includeexcludepatterns"
+	CapFileCopyRequiredPaths                  apicaps.CapID = "file.copy.requiredpaths"
 	CapFileRmNoFollowSymlink                  apicaps.CapID = "file.rm.nofollowsymlink"
 	CapFileCopyAlwaysReplaceExistingDestPaths apicaps.CapID = "file.copy.alwaysreplaceexistingdestpaths"
 	CapFileCopyModeStringFormat               apicaps.CapID = "file.copy.modestring"
@@ -451,6 +452,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapFileCopyRequiredPaths,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapFileCopyAlwaysReplaceExistingDestPaths,
 		Enabled: true,

solver/pb/ops.proto

@@ -355,6 +355,9 @@ message FileActionCopy {
 	bool alwaysReplaceExistingDestPaths = 14;
 	// mode in non-octal format
 	string modeStr = 15;
+	// required paths that must be included in the copy. This is only used when
+	// include_patterns has at least one pattern.
+	repeated string required_paths = 16;
 }
 
 message FileActionMkFile {