converter: support appending files to bootstrap layer

Support `AppendFiles` option on `converter.Merge` method,
`AppendFiles` specifies the files that need to be appended
to the bootstrap layer.

This option allows the user to place some files in the nydus
bootstrap layer, for example, the user can place metadata
info generated at build time, e.g. named `image/metadata.json`.

And this usage does not affect the `image/image.boot` bootstrap
file that is accessed by the nydus runtime.

Signed-off-by: Yan Song <yansong.ys@antgroup.com>

pkg/converter/convert_unix.go

@@ -25,6 +25,7 @@ import (
 	"github.com/containerd/containerd/archive"
 	"github.com/containerd/containerd/archive/compression"
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/images/converter"
@@ -624,13 +625,23 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
 		return nil, errors.Wrap(err, "merge bootstrap")
 	}
 
+	bootstrapRa, err := local.OpenReader(targetBootstrapPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "open bootstrap reader")
+	}
+	defer bootstrapRa.Close()
+
+	files := append([]File{
+		{
+			Name:   EntryBootstrap,
+			Reader: content.NewReader(bootstrapRa),
+			Size:   bootstrapRa.Size(),
+		},
+	}, opt.AppendFiles...)
 	var rc io.ReadCloser
 
 	if opt.WithTar {
-		rc, err = packToTar(targetBootstrapPath, fmt.Sprintf("image/%s", EntryBootstrap), false)
-		if err != nil {
-			return nil, errors.Wrap(err, "pack bootstrap to tar")
-		}
+		rc = packToTar(files, false)
 	} else {
 		rc, err = os.Open(targetBootstrapPath)
 		if err != nil {

pkg/converter/types.go

@@ -125,6 +125,8 @@ type MergeOption struct {
 	Timeout *time.Duration
 	// Encrypt encrypts the bootstrap layer if it's specified.
 	Encrypt Encrypter
+	// AppendFiles specifies the files that need to be appended to the bootstrap layer.
+	AppendFiles []File
 }
 
 type UnpackOption struct {

pkg/converter/utils.go

@@ -14,14 +14,19 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
 
 	"github.com/containerd/containerd/content"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+type File struct {
+	Name   string
+	Reader io.Reader
+	Size   int64
+}
+
 type writeCloser struct {
 	closed bool
 	io.WriteCloser
@@ -83,39 +88,27 @@ func newSeekReader(ra io.ReaderAt) *seekReader {
 	}
 }
 
-// packToTar makes .tar(.gz) stream of file named `name` and return reader.
-func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
-	fi, err := os.Stat(src)
-	if err != nil {
-		return nil, err
-	}
-
+// packToTar packs files to .tar(.gz) stream then return reader.
+func packToTar(files []File, compress bool) io.ReadCloser {
 	dirHdr := &tar.Header{
-		Name:     filepath.Dir(name),
+		Name:     "image",
 		Mode:     0755,
 		Typeflag: tar.TypeDir,
 	}
 
-	hdr := &tar.Header{
-		Name: name,
-		Mode: 0444,
-		Size: fi.Size(),
-	}
-
-	reader, writer := io.Pipe()
+	pr, pw := io.Pipe()
 
 	go func() {
 		// Prepare targz writer
 		var tw *tar.Writer
 		var gw *gzip.Writer
 		var err error
-		var file *os.File
 
 		if compress {
-			gw = gzip.NewWriter(writer)
+			gw = gzip.NewWriter(pw)
 			tw = tar.NewWriter(gw)
 		} else {
-			tw = tar.NewWriter(writer)
+			tw = tar.NewWriter(pw)
 		}
 
 		defer func() {
@@ -137,30 +130,30 @@ func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
 				finalErr = err2
 			}
 
-			writer.CloseWithError(finalErr)
+			pw.CloseWithError(finalErr)
 		}()
 
-		file, err = os.Open(src)
-		if err != nil {
-			return
-		}
-		defer file.Close()
-
 		// Write targz stream
 		if err = tw.WriteHeader(dirHdr); err != nil {
 			return
 		}
 
-		if err = tw.WriteHeader(hdr); err != nil {
-			return
-		}
-
-		if _, err = io.Copy(tw, file); err != nil {
-			return
+		for _, file := range files {
+			hdr := tar.Header{
+				Name: filepath.Join("image", file.Name),
+				Mode: 0444,
+				Size: file.Size,
+			}
+			if err = tw.WriteHeader(&hdr); err != nil {
+				return
+			}
+			if _, err = io.Copy(tw, file.Reader); err != nil {
+				return
+			}
 		}
 	}()
 
-	return reader, nil
+	return pr
 }
 
 // Copied from containerd/containerd project, copyright The containerd Authors.