nydusify(feat): add crc32 in file attributes

Read CRC32 from external models' manifest and pass it to builder.

Signed-off-by: Yang Kaiyong <yangkaiyong.yky@antgroup.com>

Author: Yang Kaiyong

contrib/nydusify/go.mod

@@ -39,7 +39,7 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
-	github.com/BraveY/snapshotter-converter v0.0.5 // indirect
+	github.com/BraveY/snapshotter-converter v0.0.6 // indirect
 	github.com/CloudNativeAI/model-spec v0.0.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect

contrib/nydusify/go.sum

@@ -5,6 +5,10 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIS
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw=
 github.com/BraveY/snapshotter-converter v0.0.5 h1:h3zAB31u16EOkshS2J9Nx40RiWSjH6zd5baOSmjLCOg=
 github.com/BraveY/snapshotter-converter v0.0.5/go.mod h1:nOVwsdXqdeltxr12x0t0JIbYDD+cdmdBx0HA2pYpxQY=
+github.com/BraveY/snapshotter-converter v0.0.6-0.20250409034316-66511579fa6d h1:00wAtig4otPLOMJN+CZHvG4MWm+g4NMY6j0K7eYEFNk=
+github.com/BraveY/snapshotter-converter v0.0.6-0.20250409034316-66511579fa6d/go.mod h1:nOVwsdXqdeltxr12x0t0JIbYDD+cdmdBx0HA2pYpxQY=
+github.com/BraveY/snapshotter-converter v0.0.6-0.20250409042404-e997e14906b7 h1:c9aFn0vSkXe1nrGe5mONSRs18/BXJKEiSiHvZyaXlBE=
+github.com/BraveY/snapshotter-converter v0.0.6-0.20250409042404-e997e14906b7/go.mod h1:nOVwsdXqdeltxr12x0t0JIbYDD+cdmdBx0HA2pYpxQY=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/CloudNativeAI/model-spec v0.0.2 h1:uCO86kMk8wwadn8vKs0wT4petig5crByTIngdO3L2cQ=
 github.com/CloudNativeAI/model-spec v0.0.2/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk=

contrib/nydusify/pkg/converter/converter.go

@@ -262,6 +262,7 @@ func convertModelArtifact(ctx context.Context, opt Opt) error {
 		ChunkSize:      opt.ChunkSize,
 		FromDir:        contextDir,
 		AttributesPath: attributesPath,
+		Crc32:          true,
 	}
 	_, externalBlobDigest, err := packWithAttributes(ctx, packOption, tmpDir)
 	if err != nil {

contrib/nydusify/pkg/external/modctl/remote.go

@@ -38,6 +38,20 @@ type RemoteHandler struct {
 	blobs []backend.Blob
 }
 
+type FileCrcList struct {
+	Files []FileCrcInfo `json:"files"`
+}
+
+type FileCrcInfo struct {
+	FilePath  string `json:"file_path"`
+	ChunkCrcs string `json:"chunk_crcs"`
+}
+
+const (
+	filePathKey = "org.cnai.model.filepath"
+	crcsKey     = "org.cnai.nydus.crcs"
+)
+
 func NewRemoteHandler(ctx context.Context, imageRef string, plainHTTP bool) (*RemoteHandler, error) {
 	remoter, err := pkgPvd.DefaultRemote(imageRef, true)
 	if err != nil {
@@ -154,13 +168,27 @@ func (handler *RemoteHandler) handle(ctx context.Context, layer ocispec.Descript
 		return nil, errors.Wrap(err, "read tar blob failed")
 	}
 
+	var fileCrcList = FileCrcList{}
+	var fileCrcMap = make(map[string]string)
+	if layer.Annotations != nil {
+		if c, ok := layer.Annotations[crcsKey]; ok {
+			if err := json.Unmarshal([]byte(c), &fileCrcList); err != nil {
+				return nil, errors.Wrap(err, "unmarshal crcs failed")
+			}
+			for _, f := range fileCrcList.Files {
+				fileCrcMap[f.FilePath] = f.ChunkCrcs
+			}
+		}
+	}
+
 	blobInfo := handler.blobs[index].Config
 	fileAttrs := make([]backend.FileAttribute, len(files))
 	hackFile := os.Getenv("HACK_FILE")
 	for idx, f := range files {
 		if hackFile != "" && f.name == hackFile {
 			hackFileWrapper(&f)
 		}
+
 		fileAttrs[idx] = backend.FileAttribute{
 			BlobID:                 blobInfo.Digest,
 			BlobIndex:              uint32(index),
@@ -172,6 +200,9 @@ func (handler *RemoteHandler) handle(ctx context.Context, layer ocispec.Descript
 			Type:                   "external",
 			Mode:                   f.mode,
 		}
+		if crcs, ok := fileCrcMap[f.name]; ok {
+			fileAttrs[idx].Crcs = crcs
+		}
 	}
 	return fileAttrs, nil
 }

contrib/nydusify/pkg/external/modctl/remote_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/snapshotter/external/backend"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type MockRemote struct {
@@ -95,13 +96,32 @@ func TestRemoteHandler_Handle(t *testing.T) {
 		MaybeWithHTTPFunc: func(error) {},
 	}
 
+	fileCrcInfo := &FileCrcInfo{
+		ChunkCrcs: "0x1234,0x5678",
+		FilePath:  "file1.txt",
+	}
+	fileCrcList := &FileCrcList{
+		Files: []FileCrcInfo{
+			*fileCrcInfo,
+		},
+	}
+	crcs, err := json.Marshal(fileCrcList)
+	require.NoError(t, err)
+	annotations := map[string]string{
+		filePathKey: "file1.txt",
+		crcsKey:     string(crcs),
+	}
 	handler := &RemoteHandler{
 		ctx:      context.Background(),
 		imageRef: "test-image",
 		remoter:  mockRemote,
 		manifest: ocispec.Manifest{
 			Layers: []ocispec.Descriptor{
-				{MediaType: "test-media-type", Digest: "test-digest"},
+				{
+					MediaType:   "test-media-type",
+					Digest:      "test-digest",
+					Annotations: annotations,
+				},
 			},
 		},
 		blobs: []backend.Blob{
@@ -119,6 +139,15 @@ func TestRemoteHandler_Handle(t *testing.T) {
 	assert.NotNil(t, backend)
 	assert.NotEmpty(t, fileAttrs)
 	assert.Equal(t, 3, len(fileAttrs))
+	assert.Equal(t, fileCrcInfo.ChunkCrcs, fileAttrs[0].Crcs)
+	assert.Equal(t, "", fileAttrs[1].Crcs)
+
+	handler.manifest.Layers[0].Annotations = map[string]string{
+		filePathKey: "file1.txt",
+		crcsKey:     "0x1234,0x5678",
+	}
+	_, _, err = handler.Handle(context.Background())
+	assert.Error(t, err)
 }
 
 func TestGetModelConfig(t *testing.T) {

contrib/nydusify/pkg/snapshotter/external/backend/backend.go

@@ -43,6 +43,7 @@ type FileAttribute struct {
 	Chunk0CompressedOffset uint64
 	Type                   string
 	Mode                   uint32
+	Crcs                   string
 }
 
 type File struct {

contrib/nydusify/pkg/snapshotter/external/external.go

@@ -90,8 +90,8 @@ func RemoteHandle(ctx context.Context, opts Options) error {
 	}
 	attributes := []Attribute{}
 	for _, file := range fileAttrs {
-		p := fmt.Sprintf("/%s type=%s file_size=%d blob_index=%d blob_id=%s chunk_size=%s chunk_0_compressed_offset=%d compressed_size=%s",
-			file.RelativePath, file.Type, file.FileSize, file.BlobIndex, file.BlobID, file.ChunkSize, file.Chunk0CompressedOffset, file.BlobSize)
+		p := fmt.Sprintf("/%s type=%s file_size=%d blob_index=%d blob_id=%s chunk_size=%s chunk_0_compressed_offset=%d compressed_size=%s crcs=%s",
+			file.RelativePath, file.Type, file.FileSize, file.BlobIndex, file.BlobID, file.ChunkSize, file.Chunk0CompressedOffset, file.BlobSize, file.Crcs)
 		attributes = append(attributes, Attribute{
 			Pattern: p,
 		})