nydus-image(feat): add crc32 in external chunks

Signed-off-by: Yang Kaiyong <yangkaiyong.yky@antgroup.com>

Author: Yang Kaiyong

builder/src/attributes.rs

@@ -11,6 +11,7 @@ use gix_attributes::parse;
 use gix_attributes::parse::Kind;
 
 const KEY_TYPE: &str = "type";
+const KEY_CRCS: &str = "crcs";
 const VAL_EXTERNAL: &str = "external";
 
 pub struct Parser {}
@@ -24,6 +25,7 @@ pub struct Item {
 #[derive(Clone, Debug, Eq, PartialEq, Default)]
 pub struct Attributes {
     pub items: HashMap<PathBuf, HashMap<String, String>>,
+    pub crcs: HashMap<PathBuf, Vec<u32>>,
 }
 
 impl Attributes {
@@ -33,6 +35,7 @@ impl Attributes {
         let _items = parse(&content);
 
         let mut items = HashMap::new();
+        let mut crcs = HashMap::new();
         for _item in _items {
             let _item = _item?;
             if let Kind::Pattern(pattern) = _item.0 {
@@ -43,15 +46,32 @@ impl Attributes {
                 let mut current_path = path.clone();
                 let mut attributes = HashMap::new();
                 let mut _type = String::new();
+                let mut crc_arr = vec![];
                 for line in _item.1 {
                     let line = line?;
                     let name = line.name.as_str();
                     let state = line.state.as_bstr().unwrap_or_default();
                     if name == KEY_TYPE {
                         _type = state.to_string();
                     }
+                    if name == KEY_CRCS {
+                        crc_arr = state
+                            .to_string()
+                            .split(',')
+                            .map(|s| {
+                                let trimmed = s.trim();
+                                let hex_str = if trimmed.starts_with("0x") {
+                                    &trimmed[2..]
+                                } else {
+                                    trimmed
+                                };
+                                u32::from_str_radix(hex_str, 16).map_err(|e| anyhow::anyhow!(e))
+                            })
+                            .collect::<Result<Vec<u32>, _>>()?;
+                    }
                     attributes.insert(name.to_string(), state.to_string());
                 }
+                crcs.insert(path.clone(), crc_arr);
                 items.insert(path, attributes);
 
                 // process parent directory
@@ -69,7 +89,7 @@ impl Attributes {
             }
         }
 
-        Ok(Attributes { items })
+        Ok(Attributes { items, crcs })
     }
 
     fn check_external(&self, attributes: &HashMap<String, String>) -> bool {
@@ -99,6 +119,10 @@ impl Attributes {
     pub fn get_values<P: AsRef<Path>>(&self, path: P) -> Option<&HashMap<String, String>> {
         self.items.get(path.as_ref())
     }
+
+    pub fn get_crcs<P: AsRef<Path>>(&self, path: P) -> Option<&Vec<u32>> {
+        self.crcs.get(path.as_ref())
+    }
 }
 
 #[cfg(test)]
@@ -113,17 +137,25 @@ mod tests {
         let file = TempFile::new().unwrap();
         fs::write(
             file.as_path(),
-            "/foo type=external
-            /bar type=external
+            "/foo type=external crcs=0x1234,0x5678
+            /bar type=external crcs=0x1234,0x5678
             /models/foo/bar type=external",
         )
         .unwrap();
 
         let attributes = Attributes::from(file.as_path()).unwrap();
-        let _attributes: HashMap<String, String> = [("type".to_string(), "external".to_string())]
-            .iter()
-            .cloned()
-            .collect();
+        let _attributes_base: HashMap<String, String> =
+            [("type".to_string(), "external".to_string())]
+                .iter()
+                .cloned()
+                .collect();
+        let _attributes: HashMap<String, String> = [
+            ("type".to_string(), "external".to_string()),
+            ("crcs".to_string(), "0x1234,0x5678".to_string()),
+        ]
+        .iter()
+        .cloned()
+        .collect();
 
         let items_map: HashMap<PathBuf, HashMap<String, String>> = vec![
             Item {
@@ -136,21 +168,22 @@ mod tests {
             },
             Item {
                 pattern: PathBuf::from("/models"),
-                attributes: _attributes.clone(),
+                attributes: _attributes_base.clone(),
             },
             Item {
                 pattern: PathBuf::from("/models/foo"),
-                attributes: _attributes.clone(),
+                attributes: _attributes_base.clone(),
             },
             Item {
                 pattern: PathBuf::from("/models/foo/bar"),
-                attributes: _attributes.clone(),
+                attributes: _attributes_base.clone(),
             },
         ]
         .into_iter()
         .map(|item| (item.pattern, item.attributes))
         .collect();
 
-        assert_eq!(attributes, Attributes { items: items_map });
+        assert_eq!(attributes.items, items_map);
+        assert_eq!(attributes.get_crcs("/foo"), Some(&vec![0x1234, 0x5678]))
     }
 }

builder/src/core/node.rs

@@ -366,7 +366,7 @@ impl Node {
 
             let chunk_data = &mut data_buf[0..uncompressed_size as usize];
             let (mut chunk, mut chunk_info) =
-                self.read_file_chunk(ctx, reader, chunk_data, blob_mgr.external)?;
+                self.read_file_chunk(ctx, reader, chunk_data, i, blob_mgr.external)?;
             if let Some(h) = inode_hasher.as_mut() {
                 h.digest_update(chunk.id().as_ref());
             }
@@ -432,6 +432,7 @@ impl Node {
         ctx: &BuildContext,
         reader: &mut R,
         buf: &mut [u8],
+        index: u32,
         external: bool,
     ) -> Result<(ChunkWrapper, Option<BlobChunkInfoV2Ondisk>)> {
         let mut chunk = self.inode.create_chunk();
@@ -475,6 +476,22 @@ impl Node {
             }
         }
 
+        if ctx.crc_checker != crc::Algorithm::None && external {
+            println!("crc checker is enabled for external blob");
+            if let Some(crcs) = ctx.attributes.get_crcs(self.target()) {
+                println!("matched crcs for file {}", self.target().display());
+                if (index as usize) >= crcs.len() {
+                    return Err(anyhow!(
+                        "invalid crc index {} for file {}",
+                        index,
+                        self.target().display()
+                    ));
+                }
+                chunk.set_has_crc(true);
+                chunk.set_crc32(crcs[index as usize]);
+            }
+        }
+
         if ctx.cipher != crypt::Algorithm::None && !external {
             chunk.set_encrypted(true);
         }
@@ -999,12 +1016,12 @@ impl Node {
 
 #[cfg(test)]
 mod tests {
-    use std::io::BufReader;
+    use std::{collections::HashMap, io::BufReader};
 
     use nydus_utils::{digest, BufReaderInfo};
     use vmm_sys_util::tempfile::TempFile;
 
-    use crate::{ArtifactWriter, BlobCacheGenerator, HashChunkDict};
+    use crate::{attributes::Attributes, ArtifactWriter, BlobCacheGenerator, HashChunkDict};
 
     use super::*;
 
@@ -1212,4 +1229,73 @@ mod tests {
         node.remove_xattr(OsStr::new("system.posix_acl_default.key"));
         assert!(!node.inode.has_xattr());
     }
+
+    #[test]
+    fn test_read_file_chunk_crc_valid_index() {
+        let mut ctx = BuildContext::default();
+        ctx.crc_checker = crc::Algorithm::Crc32Iscsi;
+        ctx.attributes = Attributes {
+            crcs: HashMap::new(),
+            ..Default::default()
+        };
+        let target = PathBuf::from("/test_file");
+        ctx.attributes
+            .crcs
+            .insert(target.clone(), vec![0x12345678, 0x87654321]);
+
+        let node = Node::new(
+            InodeWrapper::new(RafsVersion::V5),
+            NodeInfo {
+                path: target.clone(),
+                target: target.clone(),
+                ..Default::default()
+            },
+            1,
+        );
+
+        let mut reader = std::io::Cursor::new(vec![0u8; 1024]);
+        let mut buf = [0u8; 1024];
+
+        print!("target: {}", node.target().display());
+        let result = node.read_file_chunk(&ctx, &mut reader, &mut buf, 1, true);
+        assert!(result.is_ok());
+        let (chunk, _) = result.unwrap();
+        assert_eq!(chunk.crc32(), 0x87654321);
+        assert!(chunk.has_crc());
+
+        // test invalid crc index
+        let result = node.read_file_chunk(&ctx, &mut reader, &mut buf, 2, true);
+        assert!(result.is_err());
+        let err = result.unwrap_err().to_string();
+        assert!(err.contains("invalid crc index 2 for file /test_file"));
+    }
+
+    #[test]
+    fn test_read_file_chunk_crc_invalid_index() {
+        let mut ctx = BuildContext::default();
+        ctx.crc_checker = crc::Algorithm::Crc32Iscsi;
+        ctx.attributes = Attributes {
+            crcs: HashMap::new(),
+            ..Default::default()
+        };
+        let target = PathBuf::from("/test_file");
+        ctx.attributes.crcs.insert(target.clone(), vec![0x12345678]);
+
+        let node = Node::new(
+            InodeWrapper::new(RafsVersion::V5),
+            NodeInfo {
+                path: target.clone(),
+                ..Default::default()
+            },
+            1,
+        );
+
+        let mut reader = std::io::Cursor::new(vec![0u8; 1024]);
+        let mut buf = [0u8; 1024];
+
+        let result = node.read_file_chunk(&ctx, &mut reader, &mut buf, 1, true);
+        assert!(result.is_err());
+        let err = result.unwrap_err().to_string();
+        assert!(err.contains("invalid crc index 1 for file /test_file"));
+    }
 }