nydusify(feat): support for specifing log file and concurrently processing external model manifests

Signed-off-by: Yang Kaiyong <yangkaiyong.yky@antgroup.com>

Author: Yang Kaiyong

contrib/nydusify/cmd/nydusify.go

@@ -179,21 +179,8 @@ func getPrefetchPatterns(c *cli.Context) (string, error) {
 	return patterns, nil
 }
 
-func main() {
-	logrus.SetFormatter(&logrus.TextFormatter{
-		FullTimestamp: true,
-	})
-
-	version := fmt.Sprintf("\nVersion	: %s\nRevision	: %s\nGo version	: %s\nBuild time	: %s", gitVersion, revision, runtime.Version(), buildTime)
-
-	app := &cli.App{
-		Name:    "Nydusify",
-		Usage:   "Nydus utility tool to build, convert, verify and view container images",
-		Version: version,
-	}
-
-	// global options
-	app.Flags = []cli.Flag{
+func getGlobalFlags() []cli.Flag {
+	return []cli.Flag{
 		&cli.BoolFlag{
 			Name:     "debug",
 			Aliases:  []string{"D"},
@@ -208,7 +195,30 @@ func main() {
 			Usage:   "Set log level (panic, fatal, error, warn, info, debug, trace)",
 			EnvVars: []string{"LOG_LEVEL"},
 		},
+		&cli.StringFlag{
+			Name:     "log-file",
+			Required: false,
+			Usage:    "Write logs to a file",
+			EnvVars:  []string{"LOG_FILE"},
+		},
 	}
+}
+
+func main() {
+	logrus.SetFormatter(&logrus.TextFormatter{
+		FullTimestamp: true,
+	})
+
+	version := fmt.Sprintf("\nVersion	: %s\nRevision	: %s\nGo version	: %s\nBuild time	: %s", gitVersion, revision, runtime.Version(), buildTime)
+
+	app := &cli.App{
+		Name:    "Nydusify",
+		Usage:   "Nydus utility tool to build, convert, verify and view container images",
+		Version: version,
+	}
+
+	// global options
+	app.Flags = getGlobalFlags()
 
 	app.Commands = []*cli.Command{
 		{
@@ -1439,4 +1449,13 @@ func setupLogLevel(c *cli.Context) {
 	}
 
 	logrus.SetLevel(logLevel)
+
+	if c.String("log-file") != "" {
+		f, err := os.OpenFile(c.String("log-file"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+		if err != nil {
+			logrus.Errorf("failed to open log file: %+v", err)
+			return
+		}
+		logrus.SetOutput(f)
+	}
 }

contrib/nydusify/cmd/nydusify_test.go

@@ -12,6 +12,9 @@ import (
 	"os"
 	"testing"
 
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
 )
@@ -341,3 +344,50 @@ func TestGetPrefetchPatterns(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "/", patterns)
 }
+
+func TestGetGlobalFlags(t *testing.T) {
+	flags := getGlobalFlags()
+	require.Equal(t, 3, len(flags))
+}
+
+func TestSetupLogLevelWithLogFile(t *testing.T) {
+	logFilePath := "test_log_file.log"
+	defer os.Remove(logFilePath)
+
+	c := &cli.Context{}
+
+	patches := gomonkey.ApplyMethodSeq(c, "String", []gomonkey.OutputCell{
+		{Values: []interface{}{"info"}, Times: 1},
+		{Values: []interface{}{"test_log_file.log"}, Times: 2},
+	})
+	defer patches.Reset()
+	setupLogLevel(c)
+
+	file, err := os.Open(logFilePath)
+	assert.NoError(t, err)
+	assert.NotNil(t, file)
+	file.Close()
+
+	logrusOutput := logrus.StandardLogger().Out
+	assert.NotNil(t, logrusOutput)
+
+	logrus.Info("This is a test log message")
+	content, err := os.ReadFile(logFilePath)
+	assert.NoError(t, err)
+	assert.Contains(t, string(content), "This is a test log message")
+}
+
+func TestSetupLogLevelWithInvalidLogFile(t *testing.T) {
+
+	c := &cli.Context{}
+
+	patches := gomonkey.ApplyMethodSeq(c, "String", []gomonkey.OutputCell{
+		{Values: []interface{}{"info"}, Times: 1},
+		{Values: []interface{}{"test/test_log_file.log"}, Times: 2},
+	})
+	defer patches.Reset()
+	setupLogLevel(c)
+
+	logrusOutput := logrus.StandardLogger().Out
+	assert.NotNil(t, logrusOutput)
+}

contrib/nydusify/go.mod

@@ -39,7 +39,7 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
-	github.com/BraveY/snapshotter-converter v0.0.6 // indirect
+	github.com/BraveY/snapshotter-converter v0.0.5 // indirect
 	github.com/CloudNativeAI/model-spec v0.0.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect

contrib/nydusify/pkg/converter/converter.go

@@ -262,7 +262,6 @@ func convertModelArtifact(ctx context.Context, opt Opt) error {
 		ChunkSize:      opt.ChunkSize,
 		FromDir:        contextDir,
 		AttributesPath: attributesPath,
-		Crc32:          true,
 	}
 	_, externalBlobDigest, err := packWithAttributes(ctx, packOption, tmpDir)
 	if err != nil {

contrib/nydusify/pkg/external/modctl/remote.go

@@ -10,11 +10,13 @@ import (
 	"io"
 	"os"
 	"strconv"
+	"sync"
 
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/snapshotter/external/backend"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 
 	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
 	pkgPvd "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
@@ -80,14 +82,34 @@ func initRemoteHandler(handler *RemoteHandler) error {
 }
 
 func (handler *RemoteHandler) Handle(ctx context.Context) (*backend.Backend, []backend.FileAttribute, error) {
-	var fileAttrs []backend.FileAttribute
+	var (
+		fileAttrs []backend.FileAttribute
+		mu        sync.Mutex
+		eg        *errgroup.Group
+	)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	eg, ctx = errgroup.WithContext(ctx)
+	eg.SetLimit(5)
+
 	for idx, layer := range handler.manifest.Layers {
-		fa, err := handler.handle(ctx, layer, int32(idx))
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "handle layer failed")
-		}
-		fileAttrs = append(fileAttrs, fa...)
+		eg.Go(func() error {
+			fa, err := handler.handle(ctx, layer, int32(idx))
+			if err != nil {
+				cancel()
+				return err
+			}
+			mu.Lock()
+			fileAttrs = append(fileAttrs, fa...)
+			mu.Unlock()
+			return nil
+		})
 	}
+
+	if err := eg.Wait(); err != nil {
+		return nil, nil, errors.Wrap(err, "wait for handle failed")
+	}
+
 	bkd, err := handler.backend()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "get backend failed")

contrib/nydusify/pkg/snapshotter/external/external.go

@@ -95,7 +95,7 @@ func RemoteHandle(ctx context.Context, opts Options) error {
 		attributes = append(attributes, Attribute{
 			Pattern: p,
 		})
-		logrus.Infof("file attr: %s, file_mode: %o", p, file.Mode)
+		logrus.Debugf("file attr: %s, file_mode: %o", p, file.Mode)
 	}
 
 	backendBytes, err := json.MarshalIndent(bkd, "", "  ")