smoke: add test for external blob

Signed-off-by: Yan Song <[email protected]>

builder/src/attributes.rs

@@ -10,79 +10,112 @@ use anyhow::Result;
 use gix_attributes::parse;
 use gix_attributes::parse::Kind;
 
-pub struct Attribute {}
+const KEY_TYPE: &str = "type";
+const VAL_EXTERNAL: &str = "external";
 
-impl Attribute {
+pub struct Parser {}
+
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Item {
+    pub pattern: PathBuf,
+    pub attributes: HashMap<String, String>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Attributes {
+    pub items: Vec<Item>,
+}
+
+impl Attributes {
     /// Parse nydus attributes from a file.
-    pub fn parse<P: AsRef<Path>>(path: P) -> Result<HashMap<PathBuf, u32>> {
+    pub fn from<P: AsRef<Path>>(path: P) -> Result<Attributes> {
         let content = fs::read(path)?;
-        let attributes = parse(&content);
-        let mut result = HashMap::new();
-        for attribute in attributes {
-            let attribute = attribute?;
-            if let Kind::Pattern(pattern) = attribute.0 {
-                let mut path: Option<PathBuf> = None;
-                let mut backend_index: Option<u32> = None;
-                for line in attribute.1 {
-                    let line = line?;
-                    if line.name.as_str() == "type"
-                        && line.state.as_bstr().unwrap_or_default() == "external"
-                    {
-                        let _path = PathBuf::from(pattern.text.to_string());
-                        if !_path.is_absolute() {
-                            path = Some(path::Path::new("/").join(_path));
-                        }
-                    }
-                    if line.name.as_str() == "backend_index" {
-                        backend_index = Some(
-                            line.state
-                                .as_bstr()
-                                .unwrap_or_default()
-                                .to_string()
-                                .parse()?,
-                        );
-                    }
+        let _items = parse(&content);
+
+        let mut items = Vec::new();
+        for _item in _items {
+            let _item = _item?;
+            if let Kind::Pattern(pattern) = _item.0 {
+                let mut path = PathBuf::from(pattern.text.to_string());
+                if !path.is_absolute() {
+                    path = path::Path::new("/").join(path);
                 }
-                match (path, backend_index) {
-                    (Some(path), Some(backend_index)) => {
-                        result.insert(path, backend_index);
-                    }
-                    _ => {}
+                let mut attributes = HashMap::new();
+                for line in _item.1 {
+                    let line = line?;
+                    let name = line.name.as_str();
+                    let state = line.state.as_bstr().unwrap_or_default();
+                    attributes.insert(name.to_string(), state.to_string());
                 }
+                items.push(Item {
+                    pattern: path,
+                    attributes,
+                });
             }
         }
-        Ok(result)
+
+        Ok(Attributes { items })
+    }
+
+    fn check_external(&self, item: &Item) -> bool {
+        item.attributes.get(KEY_TYPE) == Some(&VAL_EXTERNAL.to_string())
+    }
+
+    pub fn is_external<P: AsRef<Path>>(&self, path: P) -> bool {
+        self.items
+            .iter()
+            .any(|item| item.pattern == path.as_ref() && self.check_external(item))
+    }
+
+    pub fn is_prefix_external<P: AsRef<Path>>(&self, target: P) -> bool {
+        self.items
+            .iter()
+            .any(|item| item.pattern.starts_with(&target) && self.check_external(item))
     }
 }
 
 #[cfg(test)]
 mod tests {
     use std::{collections::HashMap, fs, path::PathBuf};
 
-    use super::Attribute;
+    use super::{Attributes, Item};
     use vmm_sys_util::tempfile::TempFile;
 
     #[test]
     fn test_attribute_parse() {
         let file = TempFile::new().unwrap();
         fs::write(
             file.as_path(),
-            "/foo type=external backend_index=0
-            /bar type=external backend_index=0
-            /models/foo type=external backend_index=1",
+            "/foo type=external
+            /bar type=external
+            /models/foo type=external",
         )
         .unwrap();
-        let paths = Attribute::parse(file.as_path()).unwrap();
-        assert_eq!(
-            paths,
-            [
-                (PathBuf::from("/foo"), 0),
-                (PathBuf::from("/bar"), 0),
-                (PathBuf::from("/models/foo"), 1)
-            ]
+
+        let attributes = Attributes::from(file.as_path()).unwrap();
+        let _attributes: HashMap<String, String> = [("type".to_string(), "external".to_string())]
             .iter()
             .cloned()
-            .collect::<HashMap<PathBuf, u32>>()
+            .collect();
+
+        assert_eq!(
+            attributes,
+            Attributes {
+                items: vec![
+                    Item {
+                        pattern: PathBuf::from("/foo"),
+                        attributes: _attributes.clone()
+                    },
+                    Item {
+                        pattern: PathBuf::from("/bar"),
+                        attributes: _attributes.clone()
+                    },
+                    Item {
+                        pattern: PathBuf::from("/models/foo"),
+                        attributes: _attributes.clone()
+                    }
+                ]
+            },
         );
     }
 }

builder/src/compact.rs

@@ -21,6 +21,7 @@ use nydus_utils::{digest, try_round_up_4k};
 use serde::{Deserialize, Serialize};
 use sha2::Digest;
 
+use crate::attributes::Attributes;
 use crate::core::context::Artifact;
 
 use super::core::blob::Blob;
@@ -615,7 +616,7 @@ impl BlobCompactor {
             false,
             Features::new(),
             false,
-            HashMap::new(),
+            Attributes::default(),
         );
         let mut bootstrap_mgr =
             BootstrapManager::new(Some(ArtifactStorage::SingleFile(d_bootstrap)), None);
@@ -1142,7 +1143,7 @@ mod tests {
             false,
             Features::new(),
             false,
-            HashMap::new(),
+            Attributes::default(),
         );
 
         let mut compactor = blob_compactor_load_and_dedup_chunks().unwrap();
@@ -1246,7 +1247,7 @@ mod tests {
             false,
             Features::new(),
             false,
-            HashMap::new(),
+            Attributes::default(),
         );
         let mut blob_ctx1 = BlobContext::new(
             "blob_id1".to_owned(),

builder/src/core/context.rs

@@ -44,6 +44,7 @@ use nydus_utils::digest::DigestData;
 use nydus_utils::{compress, digest, div_round_up, round_down, try_round_up_4k, BufReaderInfo};
 
 use super::node::ChunkSource;
+use crate::attributes::Attributes;
 use crate::core::tree::TreeNode;
 use crate::{ChunkDict, Feature, Features, HashChunkDict, Prefetch, PrefetchPolicy, WhiteoutSpec};
 
@@ -613,7 +614,6 @@ impl BlobContext {
         blob_ctx
             .blob_meta_header
             .set_encrypted(features.contains(BlobFeatures::ENCRYPTED));
-        println!("EXTERNAL {}", features.contains(BlobFeatures::EXTERNAL));
         blob_ctx
             .blob_meta_header
             .set_external(features.contains(BlobFeatures::EXTERNAL));
@@ -1328,7 +1328,7 @@ pub struct BuildContext {
     pub blob_cache_generator: Option<BlobCacheGenerator>,
 
     /// Nydus attributes for different build behavior.
-    pub attributes: HashMap<PathBuf, u32>,
+    pub attributes: Attributes,
 }
 
 impl BuildContext {
@@ -1349,7 +1349,7 @@ impl BuildContext {
         blob_inline_meta: bool,
         features: Features,
         encrypt: bool,
-        attributes: HashMap<PathBuf, u32>,
+        attributes: Attributes,
     ) -> Self {
         // It's a flag for images built with new nydus-image 2.2 and newer.
         let mut blob_features = BlobFeatures::CAP_TAR_TOC;
@@ -1454,7 +1454,7 @@ impl Default for BuildContext {
             configuration: Arc::new(ConfigV2::default()),
             blob_cache_generator: None,
 
-            attributes: HashMap::new(),
+            attributes: Attributes::default(),
         }
     }
 }

builder/src/core/node.rs

@@ -334,7 +334,6 @@ impl Node {
                     .layered_chunk_dict
                     .add_chunk(chunk.clone(), ctx.digester);
             }
-            println!("CHUNK {:?} {}", self.target(), chunk);
             self.chunks.push(NodeChunk {
                 source: ChunkSource::Build,
                 inner: chunk,

builder/src/directory.rs

@@ -71,7 +71,7 @@ impl FilesystemTreeBuilder {
 
             let (mut child, mut external_child) = (Tree::new(child.clone()), Tree::new(child));
 
-            let external = ctx.attributes.get(&target).is_some();
+            let external = ctx.attributes.is_external(&target);
             if external {
                 info!("ignore external file data: {:?}", path);
             }
@@ -92,11 +92,8 @@ impl FilesystemTreeBuilder {
                 external_trees.push(external_child);
             } else {
                 trees.push(child.clone());
-                for (path, _) in &ctx.attributes {
-                    if path.starts_with(&target) {
-                        external_trees.push(external_child);
-                        break;
-                    }
+                if ctx.attributes.is_prefix_external(target) {
+                    external_trees.push(external_child);
                 }
             };
         }

builder/src/stargz.rs

@@ -911,7 +911,9 @@ impl Builder for StargzBuilder {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{ArtifactStorage, ConversionType, Features, Prefetch, WhiteoutSpec};
+    use crate::{
+        attributes::Attributes, ArtifactStorage, ConversionType, Features, Prefetch, WhiteoutSpec,
+    };
 
     #[test]
     fn test_build_stargz_toc() {
@@ -937,7 +939,7 @@ mod tests {
             false,
             Features::new(),
             false,
-            HashMap::new(),
+            Attributes::default(),
         );
         ctx.fs_version = RafsVersion::V6;
         ctx.conversion_type = ConversionType::EStargzToRafs;

builder/src/tarball.rs

@@ -665,9 +665,8 @@ impl Builder for TarballBuilder {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
-
     use super::*;
+    use crate::attributes::Attributes;
     use crate::{ArtifactStorage, Features, Prefetch, WhiteoutSpec};
     use nydus_utils::{compress, digest};
 
@@ -694,7 +693,7 @@ mod tests {
             false,
             Features::new(),
             false,
-            HashMap::new(),
+            Attributes::default(),
         );
         let mut bootstrap_mgr = BootstrapManager::new(
             Some(ArtifactStorage::FileDir((tmp_dir, String::new()))),
@@ -730,7 +729,7 @@ mod tests {
             false,
             Features::new(),
             true,
-            HashMap::new(),
+            Attributes::default(),
         );
         let mut bootstrap_mgr = BootstrapManager::new(
             Some(ArtifactStorage::FileDir((tmp_dir, String::new()))),

smoke/go.mod

@@ -33,11 +33,13 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect
 	google.golang.org/grpc v1.60.1 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect