From 25b68b97ca264b5a644b85fa7129d7cfe9d060da Mon Sep 17 00:00:00 2001 From: zzlk <2418897+zzlk@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:09:12 -0800 Subject: [PATCH 1/2] fix new rust clippy warnings --- crates/bwcommon/src/denorm.rs | 6 ++-- crates/bwcommon/src/id.rs | 44 ++++++++++++------------ crates/bwmap/src/util.rs | 6 ++-- crates/bwminimaprender/src/minimap.rs | 2 +- crates/stormlib-bindings/src/bindings.rs | 1 + 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/crates/bwcommon/src/denorm.rs b/crates/bwcommon/src/denorm.rs index 9756a0c..247e8d9 100644 --- a/crates/bwcommon/src/denorm.rs +++ b/crates/bwcommon/src/denorm.rs @@ -51,7 +51,7 @@ pub fn calculate_perceptual_hashes(minimap: &Vec) -> Result<(Vec, Vec>() .chunks_exact(8) - .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x)) + .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x)) .collect(); anyhow::ensure!(ph8x8.len() == 8 * 8 / 8); @@ -60,7 +60,7 @@ pub fn calculate_perceptual_hashes(minimap: &Vec) -> Result<(Vec, Vec>() .chunks_exact(8) - .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x)) + .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x)) .collect(); anyhow::ensure!(ph16x16.len() == 16 * 16 / 8); @@ -69,7 +69,7 @@ pub fn calculate_perceptual_hashes(minimap: &Vec) -> Result<(Vec, Vec>() .chunks_exact(8) - .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x)) + .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x)) .collect(); anyhow::ensure!(ph32x32.len() == 32 * 32 / 8); diff --git a/crates/bwcommon/src/id.rs b/crates/bwcommon/src/id.rs index 978b80c..1e549b9 100644 --- a/crates/bwcommon/src/id.rs +++ b/crates/bwcommon/src/id.rs @@ -107,17 +107,17 @@ fn encode_base32(bytes: &[u8; 5]) -> String { #[rustfmt::skip] let encoded = String::from_iter([ - (bytes[0] & 0b11111000) >> 3, - (bytes[0] & 0b00000111) << 2 | - (bytes[1] & 0b11000000) >> 6, - (bytes[1] & 0b00111110) >> 1, - (bytes[1] & 0b00000001) << 4 | - (bytes[2] & 0b11110000) >> 4, - (bytes[2] & 0b00001111) << 1 | - (bytes[3] & 0b10000000) >> 7, - (bytes[3] & 0b01111100) >> 2, - (bytes[3] & 0b00000011) << 3 | - (bytes[4] & 0b11100000) >> 5, + ((bytes[0] & 0b11111000) >> 3), + ((bytes[0] & 0b00000111) << 2) | + ((bytes[1] & 0b11000000) >> 6), + ((bytes[1] & 0b00111110) >> 1), + ((bytes[1] & 0b00000001) << 4) | + ((bytes[2] & 0b11110000) >> 4), + ((bytes[2] & 0b00001111) << 1) | + ((bytes[3] & 0b10000000) >> 7), + ((bytes[3] & 0b01111100) >> 2), + ((bytes[3] & 0b00000011) << 3) | + ((bytes[4] & 0b11100000) >> 5), (bytes[4] & 0b00011111), ].into_iter().map(|x| CHARACTER_MAP[x as usize])); @@ -170,41 +170,41 @@ fn decode_base32(string: &str) -> Result<[u8; 5]> { let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[0] |= chunk << 3 & 0b11111000; + output[0] |= (chunk << 3) & 0b11111000; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[0] |= chunk >> 2 & 0b00000111; - output[1] |= chunk << 6 & 0b11000000; + output[0] |= (chunk >> 2) & 0b00000111; + output[1] |= (chunk << 6) & 0b11000000; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[1] |= chunk << 1 & 0b00111110; + output[1] |= (chunk << 1) & 0b00111110; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[1] |= chunk >> 4 & 0b00000001; - output[2] |= chunk << 4 & 0b11110000; + output[1] |= (chunk >> 4) & 0b00000001; + output[2] |= (chunk << 4) & 0b11110000; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[2] |= chunk >> 1 & 0b00001111; - output[3] |= chunk << 7 & 0b10000000; + output[2] |= (chunk >> 1) & 0b00001111; + output[3] |= (chunk << 7) & 0b10000000; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[3] |= chunk << 2 & 0b01111100; + output[3] |= (chunk << 2) & 0b01111100; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) .ok_or(anyhow!("invalid character"))?; - output[3] |= chunk >> 3 & 0b00000011; - output[4] |= chunk << 5 & 0b11100000; + output[3] |= (chunk >> 3) & 0b00000011; + output[4] |= (chunk << 5) & 0b11100000; let chunk = *INVERSE_CHARACTER_MAP .get(&iter.next().unwrap()) diff --git a/crates/bwmap/src/util.rs b/crates/bwmap/src/util.rs index 8e42372..37fa967 100644 --- a/crates/bwmap/src/util.rs +++ b/crates/bwmap/src/util.rs @@ -261,17 +261,17 @@ impl<'a> CursorSlicer<'a> { let lower_16_bits = 0_u16; let upper_16_bits = 0_u16; self.current_offset += 0; - upper_16_bits << 8 | lower_16_bits + (upper_16_bits << 8) | lower_16_bits } else if self.s.len() <= self.current_offset + 1 { let lower_16_bits = self.s[self.current_offset] as u16; let upper_16_bits = 0_u16; self.current_offset += 1; - upper_16_bits << 8 | lower_16_bits + (upper_16_bits << 8) | lower_16_bits } else { let lower_16_bits = self.s[self.current_offset] as u16; let upper_16_bits = self.s[self.current_offset + 1] as u16; self.current_offset += 2; - upper_16_bits << 8 | lower_16_bits + (upper_16_bits << 8) | lower_16_bits } } diff --git a/crates/bwminimaprender/src/minimap.rs b/crates/bwminimaprender/src/minimap.rs index 10bf53e..040ecf9 100644 --- a/crates/bwminimaprender/src/minimap.rs +++ b/crates/bwminimaprender/src/minimap.rs @@ -126,7 +126,7 @@ pub fn calculate_perceptual_hash(png: &[u8]) -> Result<[u8; 16 * 16 / 8]> { .map(|x| if *x < ph16x16_avg { 0 } else { 1 }) .collect::>() .chunks_exact(8) - .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x)) + .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x)) .collect(); anyhow::Ok(ph16x16.as_slice().try_into()?) diff --git a/crates/stormlib-bindings/src/bindings.rs b/crates/stormlib-bindings/src/bindings.rs index f43e34b..642a311 100644 --- a/crates/stormlib-bindings/src/bindings.rs +++ b/crates/stormlib-bindings/src/bindings.rs @@ -4,5 +4,6 @@ #![allow(improper_ctypes)] #![allow(dead_code)] #![allow(deref_nullptr)] +#![allow(clippy::ptr_offset_with_cast)] include!(concat!(env!("OUT_DIR"), "/bindings.rs")); From 46255af48c49eb925abad4ad2baa4bb8a8d6ce50 Mon Sep 17 00:00:00 2001 From: zzlk <2418897+zzlk@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:17:13 -0800 Subject: [PATCH 2/2] upload chkblobs to gsfs as well as mapblobs --- crates/bwmapserver/src/gsfs.rs | 72 +++++++++++++++++++++++++------ crates/bwmapserver/src/pumpers.rs | 50 ++++++++++++++++++--- crates/bwmapserver/src/util.rs | 7 +++ 3 files changed, 108 insertions(+), 21 deletions(-) diff --git a/crates/bwmapserver/src/gsfs.rs b/crates/bwmapserver/src/gsfs.rs index 1de0026..35518de 100644 --- a/crates/bwmapserver/src/gsfs.rs +++ b/crates/bwmapserver/src/gsfs.rs @@ -33,19 +33,17 @@ async fn read_file_as_stream( }) } -pub async fn gsfs_put_mapblob( +async fn gsfs_put( client: &Client, endpoint: &str, - path: impl AsRef + 'static, - mapblob_hash: &str, + src: impl Stream> + Sync + Send + 'static, + dst: impl AsRef + 'static, ) -> Result<()> { + anyhow::ensure!(dst.as_ref().starts_with("/"), "dst must start with /"); + let response = client - .put(format!( - "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}" - )) - .body(Body::wrap_stream( - read_file_as_stream(path, 1024 * 1024).await?, - )) + .put(format!("{endpoint}/api/fs/scmscx.com{}", dst.as_ref())) + .body(Body::wrap_stream(src)) .send() .await?; @@ -60,15 +58,15 @@ pub async fn gsfs_put_mapblob( Ok(()) } -pub async fn gsfs_get_mapblob( +async fn gsfs_get( client: &Client, endpoint: &str, - mapblob_hash: &str, + path: impl AsRef + 'static, ) -> Result>> { + anyhow::ensure!(path.as_ref().starts_with("/"), "path must start with /"); + let response = client - .get(format!( - "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}" - )) + .get(format!("{endpoint}/api/fs/scmscx.com{}", path.as_ref())) .send() .await?; @@ -82,3 +80,49 @@ pub async fn gsfs_get_mapblob( Ok(response.bytes_stream()) } + +pub async fn gsfs_put_mapblob( + client: &Client, + endpoint: &str, + path: impl AsRef + 'static, + mapblob_hash: &str, +) -> Result<()> { + gsfs_put( + client, + endpoint, + read_file_as_stream(path, 1024 * 1024).await?, + format!("/mapblob/{mapblob_hash}"), + ) + .await +} + +pub async fn gsfs_get_mapblob( + client: &Client, + endpoint: &str, + mapblob_hash: &str, +) -> Result>> { + gsfs_get(client, endpoint, format!("/mapblob/{mapblob_hash}")).await +} + +pub async fn gsfs_put_chkblob( + client: &Client, + endpoint: &str, + path: impl AsRef + 'static, + chkblob_hash: &str, +) -> Result<()> { + gsfs_put( + client, + endpoint, + read_file_as_stream(path, 1024 * 1024).await?, + format!("/chkblob/{chkblob_hash}"), + ) + .await +} + +// pub async fn gsfs_get_chkblob( +// client: &Client, +// endpoint: &str, +// chkblob_hash: &str, +// ) -> Result>> { +// gsfs_get(client, endpoint, format!("/chkblob/{chkblob_hash}")).await +// } diff --git a/crates/bwmapserver/src/pumpers.rs b/crates/bwmapserver/src/pumpers.rs index 9cc27e2..3b2e7de 100644 --- a/crates/bwmapserver/src/pumpers.rs +++ b/crates/bwmapserver/src/pumpers.rs @@ -1,4 +1,7 @@ -use crate::gsfs::gsfs_put_mapblob; +use crate::{ + gsfs::{gsfs_put_chkblob, gsfs_put_mapblob}, + util::calculate_hash_of_object, +}; use anyhow::Result; use async_stream::stream; use backblaze::api::{b2_authorize_account, b2_get_upload_url, b2_upload_file}; @@ -33,28 +36,61 @@ pub async fn start_gsfs_pumper(client: reqwest::Client) -> Result<()> { } } - let Ok(sha256) = entry.file_name().into_string() else { + let Ok(mapblob_hash) = entry.file_name().into_string() else { error!("could not stringify filename: {:?}", entry.file_name()); continue; }; - info!("attempting to upload file to gsfs: {sha256}"); - match gsfs_put_mapblob(&client, &endpoint, entry.path(), &sha256).await { + info!("attempting to upload mapblob to gsfs: {mapblob_hash}"); + match gsfs_put_mapblob(&client, &endpoint, entry.path(), &mapblob_hash).await { Ok(_) => {} Err(err) => { - error!("failed to put file to gsfs: {err}, sha256: {sha256}"); + error!( + "failed to put mapblob to gsfs: {err}, mapblob_hash: {mapblob_hash}" + ); continue; } } + let chkblob_hash = { + let path = entry.path().to_path_buf(); + let chkblob = match tokio::task::spawn_blocking(move || { + bwmpq::get_chk_from_mpq_filename(&path) + }) + .await + { + Ok(Ok(chkblob)) => chkblob, + Ok(Err(error)) => { + error!("failed to parse mpq: {error}"); + continue; + } + Err(error) => { + error!("failed to join with task: {error}"); + continue; + } + }; + + let chkblob_hash = calculate_hash_of_object(chkblob); + + info!("attempting to upload chkblob to gsfs: {chkblob_hash}"); + match gsfs_put_chkblob(&client, &endpoint, entry.path(), &chkblob_hash).await { + Ok(_) => {} + Err(err) => { + error!("failed to put chkblob to gsfs: {err}, mapblob_hash: {mapblob_hash}, chkblob_hash: {chkblob_hash}"); + continue; + } + } + + chkblob_hash + }; + if let Err(e) = tokio::fs::remove_file(entry.path()).await { error!("failed to remove file: {e}"); continue; } info!( - "Successfully uploaded file to gsfs: {}", - entry.path().display() + "Successfully uploaded file to gsfs: /scmscx.com/mapblob/{mapblob_hash}, /scmscx.com/chkblob/{chkblob_hash}", ); } } diff --git a/crates/bwmapserver/src/util.rs b/crates/bwmapserver/src/util.rs index ab34dee..9fdc7ed 100644 --- a/crates/bwmapserver/src/util.rs +++ b/crates/bwmapserver/src/util.rs @@ -23,6 +23,13 @@ pub(crate) fn sanitize_sc_string(s: &str) -> String { } } +pub fn calculate_hash_of_object(object: impl AsRef<[u8]>) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(&object); + format!("{:x}", hasher.finalize()) +} + // pub(crate) fn sanitize_sc_scenario_string(s: &str) -> String { // // split string by left or right marks