From 50a79fef4e3e64b3421065be9d6aa4e16b4cbe70 Mon Sep 17 00:00:00 2001 From: zzlk <2418897+zzlk@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:15:56 -0800 Subject: [PATCH] upload chkblobs to gsfs as well as mapblobs --- crates/bwmapserver/src/gsfs.rs | 72 +++++++++++++++++++++++++------ crates/bwmapserver/src/pumpers.rs | 50 ++++++++++++++++++--- crates/bwmapserver/src/util.rs | 7 +++ 3 files changed, 108 insertions(+), 21 deletions(-) diff --git a/crates/bwmapserver/src/gsfs.rs b/crates/bwmapserver/src/gsfs.rs index 1de0026..35518de 100644 --- a/crates/bwmapserver/src/gsfs.rs +++ b/crates/bwmapserver/src/gsfs.rs @@ -33,19 +33,17 @@ async fn read_file_as_stream( }) } -pub async fn gsfs_put_mapblob( +async fn gsfs_put( client: &Client, endpoint: &str, - path: impl AsRef + 'static, - mapblob_hash: &str, + src: impl Stream> + Sync + Send + 'static, + dst: impl AsRef + 'static, ) -> Result<()> { + anyhow::ensure!(dst.as_ref().starts_with("/"), "dst must start with /"); + let response = client - .put(format!( - "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}" - )) - .body(Body::wrap_stream( - read_file_as_stream(path, 1024 * 1024).await?, - )) + .put(format!("{endpoint}/api/fs/scmscx.com{}", dst.as_ref())) + .body(Body::wrap_stream(src)) .send() .await?; @@ -60,15 +58,15 @@ pub async fn gsfs_put_mapblob( Ok(()) } -pub async fn gsfs_get_mapblob( +async fn gsfs_get( client: &Client, endpoint: &str, - mapblob_hash: &str, + path: impl AsRef + 'static, ) -> Result>> { + anyhow::ensure!(path.as_ref().starts_with("/"), "path must start with /"); + let response = client - .get(format!( - "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}" - )) + .get(format!("{endpoint}/api/fs/scmscx.com{}", path.as_ref())) .send() .await?; @@ -82,3 +80,49 @@ pub async fn gsfs_get_mapblob( Ok(response.bytes_stream()) } + +pub async fn gsfs_put_mapblob( + client: &Client, + endpoint: &str, + path: impl AsRef + 'static, + mapblob_hash: &str, +) -> Result<()> { + gsfs_put( + client, + endpoint, + read_file_as_stream(path, 1024 * 1024).await?, + format!("/mapblob/{mapblob_hash}"), + ) + .await +} + +pub async fn gsfs_get_mapblob( + client: &Client, + endpoint: &str, + mapblob_hash: &str, +) -> Result>> { + gsfs_get(client, endpoint, format!("/mapblob/{mapblob_hash}")).await +} + +pub async fn gsfs_put_chkblob( + client: &Client, + endpoint: &str, + path: impl AsRef + 'static, + chkblob_hash: &str, +) -> Result<()> { + gsfs_put( + client, + endpoint, + read_file_as_stream(path, 1024 * 1024).await?, + format!("/chkblob/{chkblob_hash}"), + ) + .await +} + +// pub async fn gsfs_get_chkblob( +// client: &Client, +// endpoint: &str, +// chkblob_hash: &str, +// ) -> Result>> { +// gsfs_get(client, endpoint, format!("/chkblob/{chkblob_hash}")).await +// } diff --git a/crates/bwmapserver/src/pumpers.rs b/crates/bwmapserver/src/pumpers.rs index 9cc27e2..3b2e7de 100644 --- a/crates/bwmapserver/src/pumpers.rs +++ b/crates/bwmapserver/src/pumpers.rs @@ -1,4 +1,7 @@ -use crate::gsfs::gsfs_put_mapblob; +use crate::{ + gsfs::{gsfs_put_chkblob, gsfs_put_mapblob}, + util::calculate_hash_of_object, +}; use anyhow::Result; use async_stream::stream; use backblaze::api::{b2_authorize_account, b2_get_upload_url, b2_upload_file}; @@ -33,28 +36,61 @@ pub async fn start_gsfs_pumper(client: reqwest::Client) -> Result<()> { } } - let Ok(sha256) = entry.file_name().into_string() else { + let Ok(mapblob_hash) = entry.file_name().into_string() else { error!("could not stringify filename: {:?}", entry.file_name()); continue; }; - info!("attempting to upload file to gsfs: {sha256}"); - match gsfs_put_mapblob(&client, &endpoint, entry.path(), &sha256).await { + info!("attempting to upload mapblob to gsfs: {mapblob_hash}"); + match gsfs_put_mapblob(&client, &endpoint, entry.path(), &mapblob_hash).await { Ok(_) => {} Err(err) => { - error!("failed to put file to gsfs: {err}, sha256: {sha256}"); + error!( + "failed to put mapblob to gsfs: {err}, mapblob_hash: {mapblob_hash}" + ); continue; } } + let chkblob_hash = { + let path = entry.path().to_path_buf(); + let chkblob = match tokio::task::spawn_blocking(move || { + bwmpq::get_chk_from_mpq_filename(&path) + }) + .await + { + Ok(Ok(chkblob)) => chkblob, + Ok(Err(error)) => { + error!("failed to parse mpq: {error}"); + continue; + } + Err(error) => { + error!("failed to join with task: {error}"); + continue; + } + }; + + let chkblob_hash = calculate_hash_of_object(chkblob); + + info!("attempting to upload chkblob to gsfs: {chkblob_hash}"); + match gsfs_put_chkblob(&client, &endpoint, entry.path(), &chkblob_hash).await { + Ok(_) => {} + Err(err) => { + error!("failed to put chkblob to gsfs: {err}, mapblob_hash: {mapblob_hash}, chkblob_hash: {chkblob_hash}"); + continue; + } + } + + chkblob_hash + }; + if let Err(e) = tokio::fs::remove_file(entry.path()).await { error!("failed to remove file: {e}"); continue; } info!( - "Successfully uploaded file to gsfs: {}", - entry.path().display() + "Successfully uploaded file to gsfs: /scmscx.com/mapblob/{mapblob_hash}, /scmscx.com/chkblob/{chkblob_hash}", ); } } diff --git a/crates/bwmapserver/src/util.rs b/crates/bwmapserver/src/util.rs index ab34dee..9fdc7ed 100644 --- a/crates/bwmapserver/src/util.rs +++ b/crates/bwmapserver/src/util.rs @@ -23,6 +23,13 @@ pub(crate) fn sanitize_sc_string(s: &str) -> String { } } +pub fn calculate_hash_of_object(object: impl AsRef<[u8]>) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(&object); + format!("{:x}", hasher.finalize()) +} + // pub(crate) fn sanitize_sc_scenario_string(s: &str) -> String { // // split string by left or right marks