Add import folder & fix relative path logic

Dockerfile

@@ -26,9 +26,8 @@ CMD ["/app/summit", "0.0.0.0", "--port", "5000", "--root", "/app"]
 FROM alpine:3.20 AS vessel
 VOLUME /app/state
 VOLUME /app/config.toml
+VOLUME /import
 EXPOSE 5001
 WORKDIR /app
 COPY --from=rust-builder /vessel .
-# TODO: Remove
-COPY ./test/import/stone.index /app/state/public/volatile/x86_64/stone.index
-CMD ["/app/vessel", "0.0.0.0", "--port", "5001", "--root", "/app"]
+CMD ["/app/vessel", "0.0.0.0", "--port", "5001", "--root", "/app", "--import", "/import"]

crates/vessel/Cargo.toml

@@ -10,13 +10,16 @@ service-types = { path = "../service-types" }
 clap.workspace = true
 color-eyre.workspace = true
 futures.workspace = true
+hex.workspace = true
 http.workspace = true
 moss.workspace = true
+sha2.workspace = true
 serde.workspace = true
 sqlx.workspace = true
 stone.workspace = true
 strum.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 tracing.workspace = true
+url.workspace = true
 uuid.workspace = true

crates/vessel/src/api.rs

@@ -44,8 +44,8 @@ async fn import_packages(request: api::Request<api::v1::vessel::Build>, state: S
         .collectables
         .into_iter()
         .filter_map(|c| {
-            matches!(c.kind, collectable::Kind::Package).then_some(c.uri.parse().map(|uri| worker::Package {
-                uri,
+            matches!(c.kind, collectable::Kind::Package).then_some(c.uri.parse().map(|url| worker::Package {
+                url,
                 sha256sum: c.sha256sum,
             }))
         })
@@ -83,8 +83,8 @@ pub enum Error {
     #[error("invalid endpoint")]
     InvalidEndpoint(#[source] uuid::Error),
     /// Url cannot be parsed from string
-    #[error("invalid uri")]
-    InvalidUrl(#[from] http::uri::InvalidUri),
+    #[error("invalid url")]
+    InvalidUrl(#[from] url::ParseError),
     /// Failed to load endpoint from DB
     #[error("load endpoint")]
     LoadEndpoint(#[source] database::Error),

crates/vessel/src/main.rs

@@ -19,6 +19,7 @@ async fn main() -> Result<()> {
         port,
         config,
         root,
+        import,
     } = Args::parse();
 
     let config = Config::load(config.unwrap_or_else(|| root.join("config.toml"))).await?;
@@ -31,6 +32,10 @@ async fn main() -> Result<()> {
 
     let (worker_sender, worker_task) = worker::run(&state).await?;
 
+    if let Some(directory) = import {
+        let _ = worker_sender.send(worker::Message::ImportDirectory(directory));
+    }
+
     let mut http = Server::new(Role::RepositoryManager, &config, &state)
         .merge_api(api::service(state.db.clone(), worker_sender))
         .start((host, port))
@@ -60,4 +65,6 @@ struct Args {
     config: Option<PathBuf>,
     #[arg(long, short, default_value = ".")]
     root: PathBuf,
+    #[arg(long)]
+    import: Option<PathBuf>,
 }

crates/vessel/src/worker.rs

@@ -1,15 +1,17 @@
 use std::{
+    ffi::OsStr,
     os::unix::fs::MetadataExt,
     path::{Path, PathBuf},
 };
 
 use color_eyre::eyre::{self, eyre, Context, Result};
 use futures::{stream, StreamExt, TryStreamExt};
-use http::Uri;
 use moss::db::meta;
 use service::{api, request, Endpoint};
+use sha2::{Digest, Sha256};
 use tokio::{fs, sync::mpsc, time::Instant};
 use tracing::{error, info, info_span, Instrument};
+use url::Url;
 
 use crate::collection;
 
@@ -23,11 +25,12 @@ pub enum Message {
         endpoint: Endpoint,
         packages: Vec<Package>,
     },
+    ImportDirectory(PathBuf),
 }
 
 #[derive(Debug)]
 pub struct Package {
-    pub uri: Uri,
+    pub url: Url,
     pub sha256sum: String,
 }
 
@@ -120,6 +123,28 @@ async fn handle_message(state: &State, message: Message) -> Result<()> {
             .instrument(span)
             .await
         }
+        Message::ImportDirectory(directory) => {
+            let span = info_span!("import_directory", directory = directory.to_string_lossy().to_string());
+
+            async move {
+                info!("Import started");
+
+                let stones = enumerate_stones(&directory).context("enumerate stones")?;
+                let num_stones = stones.len();
+
+                if num_stones > 0 {
+                    import_packages(state, stones).await.context("import packages")?;
+
+                    info!(num_stones, "All stones imported");
+                } else {
+                    info!("No stones to import");
+                }
+
+                Ok(())
+            }
+            .instrument(span)
+            .await
+        }
     }
 }
 
@@ -204,11 +229,11 @@ fn import_package(
     let id = moss::package::Id::from(package.sha256sum.clone());
 
     let pool_dir = relative_pool_dir(&source_id)?;
-    let file_name = Path::new(package.uri.path())
+    let file_name = Path::new(package.url.path())
         .file_name()
         .ok_or(eyre!("Invalid archive, no file name in URI"))?;
     let target_path = pool_dir.join(file_name);
-    let full_path = state.state_dir.join(&target_path);
+    let full_path = state.state_dir.join("public").join(&target_path);
 
     meta.uri = Some(target_path.to_string_lossy().to_string());
 
@@ -262,7 +287,7 @@ fn import_package(
 async fn download_package(state_dir: &Path, package: Package) -> Result<(Package, PathBuf)> {
     let path = download_path(state_dir, &package.sha256sum).await?;
 
-    request::download_and_verify(package.uri.to_string().parse()?, &path, &package.sha256sum).await?;
+    request::download_and_verify(package.url.clone(), &path, &package.sha256sum).await?;
 
     Ok((package, path))
 }
@@ -296,7 +321,7 @@ fn relative_pool_dir(source_id: &str) -> Result<PathBuf> {
         portion = &lower[0..4];
     }
 
-    Ok(Path::new("public").join("pool").join(portion).join(lower))
+    Ok(Path::new("pool").join(portion).join(lower))
 }
 
 fn hardlink_or_copy(from: &Path, to: &Path) -> Result<()> {
@@ -380,3 +405,36 @@ async fn reindex(state: &State) -> Result<()> {
 
     Ok(())
 }
+
+fn enumerate_stones(dir: &Path) -> Result<Vec<Package>> {
+    use std::fs::{self, File};
+    use std::io;
+
+    let contents = fs::read_dir(dir).context("read directory")?;
+
+    let mut files = vec![];
+
+    for entry in contents {
+        let entry = entry.context("read directory entry")?;
+        let path = entry.path();
+        let meta = entry.metadata().context("read directory entry metadata")?;
+
+        if meta.is_file() && path.extension() == Some(OsStr::new("stone")) {
+            let url = format!("file://{}", path.to_string_lossy())
+                .parse()
+                .context("invalid file uri")?;
+
+            let mut hasher = Sha256::default();
+
+            io::copy(&mut File::open(&path).context("open file")?, &mut hasher).context("hash file")?;
+
+            let sha256sum = hex::encode(hasher.finalize());
+
+            files.push(Package { url, sha256sum });
+        } else if meta.is_dir() {
+            files.extend(enumerate_stones(&path)?);
+        }
+    }
+
+    Ok(files)
+}

test/docker-compose.yaml

@@ -1,5 +1,4 @@
 services:
-  # Legacy
   caddy:
     image: caddy:2
     ports: 
@@ -8,6 +7,7 @@ services:
       - vessel-state:/vessel
       - summit-state:/summit
       - ./legacy/Caddyfile:/etc/caddy/Caddyfile
+  # Legacy
   summit:
     image: serpentos/summit:legacy
     volumes:
@@ -17,25 +17,8 @@ services:
     depends_on:
       vessel:
         condition: service_started
-  # vessel:
-  #   image: serpentos/vessel:legacy
-  #   ports:
-  #     - "5001:5001"
-  #   volumes:
-  #     - vessel-state:/app/state
-  #     - ./legacy/vessel/.seed:/app/state/.seed
-  #     - ./import:/import
-  #   depends_on:
-  #     caddy:
-  #       condition: service_started
-  vessel:
-    image: serpentos/vessel
-    ports:
-      - "5001:5001"
-    volumes:
-      - vessel-state:/app/state
-      - ./vessel/.privkey:/app/state/.privkey
-      - ./vessel/config.toml:/app/config.toml
+      avalanche:
+        condition: service_started
   avalanche:
     image: serpentos/avalanche:legacy
     ports:
@@ -44,6 +27,19 @@ services:
     volumes:
       - avalanche-state:/app/state
       - ./legacy/avalanche/.seed:/app/state/.seed
+  # Rust
+  vessel:
+    image: serpentos/vessel
+    ports:
+      - "5001:5001"
+    volumes:
+      - vessel-state:/app/state
+      - ./vessel/.privkey:/app/state/.privkey
+      - ./vessel/config.toml:/app/config.toml
+      - ./import:/import
+    depends_on:
+      caddy:
+        condition: service_started
 
 volumes:
   summit-state:

test/legacy/bootstrap.sh

@@ -59,11 +59,6 @@ curl "http://127.0.0.1:${PORTS['summit']}/api/v1/endpoints/create" \
   -b $cookies \
   -H 'Content-Type: application/json' \
   --data-raw '{"request":{"id":"official","summary":"Indexes stuff","instanceURI":"http://vessel:'"${PORTS['vessel']}"'","pubkey":"'"${KEYS['vessel']}"'","adminName":"admin","adminEmail":"[email protected]"}}'
-# vessel-rs auto-accepts
-# curl -b $cookies "http://127.0.0.1:${PORTS['vessel']}/vsl/accept/${KEYS['summit']}" 
-
-# echo "Importing stones"
-# curl -b $cookies "http://127.0.0.1:${PORTS['vessel']}/vsl/import?importPath=%2Fimport"
 
 echo "Restarting summit"
 docker compose restart summit