[WIP] support external blob

Cargo.toml

@@ -95,6 +95,7 @@ default = [
     "backend-s3",
     "backend-http-proxy",
     "backend-localdisk",
+    "backend-external",
 ]
 virtiofs = [
     "nydus-service/virtiofs",
@@ -115,6 +116,7 @@ backend-localdisk = [
 backend-oss = ["nydus-storage/backend-oss"]
 backend-registry = ["nydus-storage/backend-registry"]
 backend-s3 = ["nydus-storage/backend-s3"]
+backend-external = ["nydus-storage/backend-external"]
 
 [workspace]
 members = [

api/src/config.rs

@@ -25,6 +25,9 @@ pub struct ConfigV2 {
     pub id: String,
     /// Configuration information for storage backend.
     pub backend: Option<BackendConfigV2>,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration information for local cache system.
     pub cache: Option<CacheConfigV2>,
     /// Configuration information for RAFS filesystem.
@@ -40,6 +43,7 @@ impl Default for ConfigV2 {
             version: 2,
             id: String::new(),
             backend: None,
+            external_backends: Vec::new(),
             cache: None,
             rafs: None,
             internal: ConfigV2Internal::default(),
@@ -54,6 +58,7 @@ impl ConfigV2 {
             version: 2,
             id: id.to_string(),
             backend: None,
+            external_backends: Vec::new(),
             cache: None,
             rafs: None,
             internal: ConfigV2Internal::default(),
@@ -959,6 +964,9 @@ pub struct BlobCacheEntryConfigV2 {
     /// Configuration information for storage backend.
     #[serde(default)]
     pub backend: BackendConfigV2,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration information for local cache system.
     #[serde(default)]
     pub cache: CacheConfigV2,
@@ -1022,6 +1030,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
             version: c.version,
             id: c.id.clone(),
             backend: Some(c.backend.clone()),
+            external_backends: c.external_backends.clone(),
             cache: Some(c.cache.clone()),
             rafs: None,
             internal: ConfigV2Internal::default(),
@@ -1292,6 +1301,19 @@ struct CacheConfig {
     pub prefetch_config: BlobPrefetchConfig,
 }
 
+/// Additional configuration information for external backend, its items
+/// will be merged to the configuration from image.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct ExternalBackendConfig {
+    /// External backend identifier to merge.
+    pub patch: HashMap<String, String>,
+    /// External backend type.
+    #[serde(rename = "type")]
+    pub kind: String,
+    /// External backend config items to merge.
+    pub config: HashMap<String, String>,
+}
+
 impl TryFrom<&CacheConfig> for CacheConfigV2 {
     type Error = std::io::Error;
 
@@ -1333,6 +1355,9 @@ struct FactoryConfig {
     pub id: String,
     /// Configuration for storage backend.
     pub backend: BackendConfig,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration for blob cache manager.
     #[serde(default)]
     pub cache: CacheConfig,
@@ -1393,6 +1418,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
             version: 2,
             id: v.device.id,
             backend: Some(backend),
+            external_backends: v.device.external_backends,
             cache: Some(cache),
             rafs: Some(rafs),
             internal: ConfigV2Internal::default(),
@@ -1482,6 +1508,9 @@ pub(crate) struct BlobCacheEntryConfig {
     ///
     /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
     backend_config: Value,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    external_backends: Vec<ExternalBackendConfig>,
     /// Type of blob cache, corresponding to `FactoryConfig::CacheConfig::cache_type`.
     ///
     /// Possible value: "fscache", "filecache".
@@ -1517,6 +1546,7 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
             version: 2,
             id: v.id.clone(),
             backend: (&backend_config).try_into()?,
+            external_backends: v.external_backends.clone(),
             cache: (&cache_config).try_into()?,
             metadata_path: v.metadata_path.clone(),
         })

builder/Cargo.toml

@@ -27,6 +27,7 @@ nydus-api = { version = "0.3", path = "../api" }
 nydus-rafs = { version = "0.3", path = "../rafs" }
 nydus-storage = { version = "0.6", path = "../storage", features = ["backend-localfs"] }
 nydus-utils = { version = "0.4", path = "../utils" }
+gix-attributes = "0.22.0"
 
 [package.metadata.docs.rs]
 all-features = true

builder/src/attributes.rs

@@ -0,0 +1,121 @@
+// Copyright 2024 Nydus Developers. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::{fs, path};
+
+use anyhow::Result;
+use gix_attributes::parse;
+use gix_attributes::parse::Kind;
+
+const KEY_TYPE: &str = "type";
+const VAL_EXTERNAL: &str = "external";
+
+pub struct Parser {}
+
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Item {
+    pub pattern: PathBuf,
+    pub attributes: HashMap<String, String>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Attributes {
+    pub items: Vec<Item>,
+}
+
+impl Attributes {
+    /// Parse nydus attributes from a file.
+    pub fn from<P: AsRef<Path>>(path: P) -> Result<Attributes> {
+        let content = fs::read(path)?;
+        let _items = parse(&content);
+
+        let mut items = Vec::new();
+        for _item in _items {
+            let _item = _item?;
+            if let Kind::Pattern(pattern) = _item.0 {
+                let mut path = PathBuf::from(pattern.text.to_string());
+                if !path.is_absolute() {
+                    path = path::Path::new("/").join(path);
+                }
+                let mut attributes = HashMap::new();
+                for line in _item.1 {
+                    let line = line?;
+                    let name = line.name.as_str();
+                    let state = line.state.as_bstr().unwrap_or_default();
+                    attributes.insert(name.to_string(), state.to_string());
+                }
+                items.push(Item {
+                    pattern: path,
+                    attributes,
+                });
+            }
+        }
+
+        Ok(Attributes { items })
+    }
+
+    fn check_external(&self, item: &Item) -> bool {
+        item.attributes.get(KEY_TYPE) == Some(&VAL_EXTERNAL.to_string())
+    }
+
+    pub fn is_external<P: AsRef<Path>>(&self, path: P) -> bool {
+        self.items
+            .iter()
+            .any(|item| item.pattern == path.as_ref() && self.check_external(item))
+    }
+
+    pub fn is_prefix_external<P: AsRef<Path>>(&self, target: P) -> bool {
+        self.items
+            .iter()
+            .any(|item| item.pattern.starts_with(&target) && self.check_external(item))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{collections::HashMap, fs, path::PathBuf};
+
+    use super::{Attributes, Item};
+    use vmm_sys_util::tempfile::TempFile;
+
+    #[test]
+    fn test_attribute_parse() {
+        let file = TempFile::new().unwrap();
+        fs::write(
+            file.as_path(),
+            "/foo type=external
+            /bar type=external
+            /models/foo type=external",
+        )
+        .unwrap();
+
+        let attributes = Attributes::from(file.as_path()).unwrap();
+        let _attributes: HashMap<String, String> = [("type".to_string(), "external".to_string())]
+            .iter()
+            .cloned()
+            .collect();
+
+        assert_eq!(
+            attributes,
+            Attributes {
+                items: vec![
+                    Item {
+                        pattern: PathBuf::from("/foo"),
+                        attributes: _attributes.clone()
+                    },
+                    Item {
+                        pattern: PathBuf::from("/bar"),
+                        attributes: _attributes.clone()
+                    },
+                    Item {
+                        pattern: PathBuf::from("/models/foo"),
+                        attributes: _attributes.clone()
+                    }
+                ]
+            },
+        );
+    }
+}