upload chkblobs to gsfs as well as mapblobs

crates/bwmapserver/src/gsfs.rs

@@ -33,19 +33,17 @@ async fn read_file_as_stream(
     })
 }
 
-pub async fn gsfs_put_mapblob(
+async fn gsfs_put(
     client: &Client,
     endpoint: &str,
-    path: impl AsRef<Path> + 'static,
-    mapblob_hash: &str,
+    src: impl Stream<Item = Result<Bytes, std::io::Error>> + Sync + Send + 'static,
+    dst: impl AsRef<str> + 'static,
 ) -> Result<()> {
+    anyhow::ensure!(dst.as_ref().starts_with("/"), "dst must start with /");
+
     let response = client
-        .put(format!(
-            "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}"
-        ))
-        .body(Body::wrap_stream(
-            read_file_as_stream(path, 1024 * 1024).await?,
-        ))
+        .put(format!("{endpoint}/api/fs/scmscx.com{}", dst.as_ref()))
+        .body(Body::wrap_stream(src))
         .send()
         .await?;
 
@@ -60,15 +58,15 @@ pub async fn gsfs_put_mapblob(
     Ok(())
 }
 
-pub async fn gsfs_get_mapblob(
+async fn gsfs_get(
     client: &Client,
     endpoint: &str,
-    mapblob_hash: &str,
+    path: impl AsRef<str> + 'static,
 ) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>> {
+    anyhow::ensure!(path.as_ref().starts_with("/"), "path must start with /");
+
     let response = client
-        .get(format!(
-            "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}"
-        ))
+        .get(format!("{endpoint}/api/fs/scmscx.com{}", path.as_ref()))
         .send()
         .await?;
 
@@ -82,3 +80,49 @@ pub async fn gsfs_get_mapblob(
 
     Ok(response.bytes_stream())
 }
+
+pub async fn gsfs_put_mapblob(
+    client: &Client,
+    endpoint: &str,
+    path: impl AsRef<Path> + 'static,
+    mapblob_hash: &str,
+) -> Result<()> {
+    gsfs_put(
+        client,
+        endpoint,
+        read_file_as_stream(path, 1024 * 1024).await?,
+        format!("/mapblob/{mapblob_hash}"),
+    )
+    .await
+}
+
+pub async fn gsfs_get_mapblob(
+    client: &Client,
+    endpoint: &str,
+    mapblob_hash: &str,
+) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>> {
+    gsfs_get(client, endpoint, format!("/mapblob/{mapblob_hash}")).await
+}
+
+pub async fn gsfs_put_chkblob(
+    client: &Client,
+    endpoint: &str,
+    path: impl AsRef<Path> + 'static,
+    chkblob_hash: &str,
+) -> Result<()> {
+    gsfs_put(
+        client,
+        endpoint,
+        read_file_as_stream(path, 1024 * 1024).await?,
+        format!("/chkblob/{chkblob_hash}"),
+    )
+    .await
+}
+
+// pub async fn gsfs_get_chkblob(
+//     client: &Client,
+//     endpoint: &str,
+//     chkblob_hash: &str,
+// ) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>> {
+//     gsfs_get(client, endpoint, format!("/chkblob/{chkblob_hash}")).await
+// }

crates/bwmapserver/src/pumpers.rs

@@ -1,4 +1,7 @@
-use crate::gsfs::gsfs_put_mapblob;
+use crate::{
+    gsfs::{gsfs_put_chkblob, gsfs_put_mapblob},
+    util::calculate_hash_of_object,
+};
 use anyhow::Result;
 use async_stream::stream;
 use backblaze::api::{b2_authorize_account, b2_get_upload_url, b2_upload_file};
@@ -33,28 +36,61 @@ pub async fn start_gsfs_pumper(client: reqwest::Client) -> Result<()> {
                     }
                 }
 
-                let Ok(sha256) = entry.file_name().into_string() else {
+                let Ok(mapblob_hash) = entry.file_name().into_string() else {
                     error!("could not stringify filename: {:?}", entry.file_name());
                     continue;
                 };
 
-                info!("attempting to upload file to gsfs: {sha256}");
-                match gsfs_put_mapblob(&client, &endpoint, entry.path(), &sha256).await {
+                info!("attempting to upload mapblob to gsfs: {mapblob_hash}");
+                match gsfs_put_mapblob(&client, &endpoint, entry.path(), &mapblob_hash).await {
                     Ok(_) => {}
                     Err(err) => {
-                        error!("failed to put file to gsfs: {err}, sha256: {sha256}");
+                        error!(
+                            "failed to put mapblob to gsfs: {err}, mapblob_hash: {mapblob_hash}"
+                        );
                         continue;
                     }
                 }
 
+                let chkblob_hash = {
+                    let path = entry.path().to_path_buf();
+                    let chkblob = match tokio::task::spawn_blocking(move || {
+                        bwmpq::get_chk_from_mpq_filename(&path)
+                    })
+                    .await
+                    {
+                        Ok(Ok(chkblob)) => chkblob,
+                        Ok(Err(error)) => {
+                            error!("failed to parse mpq: {error}");
+                            continue;
+                        }
+                        Err(error) => {
+                            error!("failed to join with task: {error}");
+                            continue;
+                        }
+                    };
+
+                    let chkblob_hash = calculate_hash_of_object(chkblob);
+
+                    info!("attempting to upload chkblob to gsfs: {chkblob_hash}");
+                    match gsfs_put_chkblob(&client, &endpoint, entry.path(), &chkblob_hash).await {
+                        Ok(_) => {}
+                        Err(err) => {
+                            error!("failed to put chkblob to gsfs: {err}, mapblob_hash: {mapblob_hash}, chkblob_hash: {chkblob_hash}");
+                            continue;
+                        }
+                    }
+
+                    chkblob_hash
+                };
+
                 if let Err(e) = tokio::fs::remove_file(entry.path()).await {
                     error!("failed to remove file: {e}");
                     continue;
                 }
 
                 info!(
-                    "Successfully uploaded file to gsfs: {}",
-                    entry.path().display()
+                    "Successfully uploaded file to gsfs: /scmscx.com/mapblob/{mapblob_hash}, /scmscx.com/chkblob/{chkblob_hash}",
                 );
             }
         }

crates/bwmapserver/src/util.rs

@@ -23,6 +23,13 @@ pub(crate) fn sanitize_sc_string(s: &str) -> String {
     }
 }
 
+pub fn calculate_hash_of_object(object: impl AsRef<[u8]>) -> String {
+    use sha2::Digest;
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(&object);
+    format!("{:x}", hasher.finalize())
+}
+
 // pub(crate) fn sanitize_sc_scenario_string(s: &str) -> String {
 //     // split string by left or right marks