upload chkblobs to gsfs as well as mapblobs

crates/bwcommon/src/denorm.rs

@@ -51,7 +51,7 @@ pub fn calculate_perceptual_hashes(minimap: &Vec<u8>) -> Result<(Vec<u8>, Vec<u8
         .map(|x| if *x < ph8x8_avg { 0 } else { 1 })
         .collect::<Vec<u8>>()
         .chunks_exact(8)
-        .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x))
+        .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x))
         .collect();
     anyhow::ensure!(ph8x8.len() == 8 * 8 / 8);
 
@@ -60,7 +60,7 @@ pub fn calculate_perceptual_hashes(minimap: &Vec<u8>) -> Result<(Vec<u8>, Vec<u8
         .map(|x| if *x < ph16x16_avg { 0 } else { 1 })
         .collect::<Vec<u8>>()
         .chunks_exact(8)
-        .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x))
+        .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x))
         .collect();
     anyhow::ensure!(ph16x16.len() == 16 * 16 / 8);
 
@@ -69,7 +69,7 @@ pub fn calculate_perceptual_hashes(minimap: &Vec<u8>) -> Result<(Vec<u8>, Vec<u8
         .map(|x| if *x < ph32x32_avg { 0 } else { 1 })
         .collect::<Vec<u8>>()
         .chunks_exact(8)
-        .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x))
+        .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x))
         .collect();
     anyhow::ensure!(ph32x32.len() == 32 * 32 / 8);
 

crates/bwcommon/src/id.rs

@@ -107,17 +107,17 @@ fn encode_base32(bytes: &[u8; 5]) -> String {
 
     #[rustfmt::skip]
     let encoded = String::from_iter([
-        (bytes[0] & 0b11111000) >> 3,
-        (bytes[0] & 0b00000111) << 2 |
-        (bytes[1] & 0b11000000) >> 6,
-        (bytes[1] & 0b00111110) >> 1,
-        (bytes[1] & 0b00000001) << 4 |
-        (bytes[2] & 0b11110000) >> 4,
-        (bytes[2] & 0b00001111) << 1 |
-        (bytes[3] & 0b10000000) >> 7,
-        (bytes[3] & 0b01111100) >> 2,
-        (bytes[3] & 0b00000011) << 3 |
-        (bytes[4] & 0b11100000) >> 5,
+        ((bytes[0] & 0b11111000) >> 3),
+        ((bytes[0] & 0b00000111) << 2) |
+        ((bytes[1] & 0b11000000) >> 6),
+        ((bytes[1] & 0b00111110) >> 1),
+        ((bytes[1] & 0b00000001) << 4) |
+        ((bytes[2] & 0b11110000) >> 4),
+        ((bytes[2] & 0b00001111) << 1) |
+        ((bytes[3] & 0b10000000) >> 7),
+        ((bytes[3] & 0b01111100) >> 2),
+        ((bytes[3] & 0b00000011) << 3) |
+        ((bytes[4] & 0b11100000) >> 5),
         (bytes[4] & 0b00011111),
     ].into_iter().map(|x| CHARACTER_MAP[x as usize]));
 
@@ -170,41 +170,41 @@ fn decode_base32(string: &str) -> Result<[u8; 5]> {
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[0] |= chunk << 3 & 0b11111000;
+    output[0] |= (chunk << 3) & 0b11111000;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[0] |= chunk >> 2 & 0b00000111;
-    output[1] |= chunk << 6 & 0b11000000;
+    output[0] |= (chunk >> 2) & 0b00000111;
+    output[1] |= (chunk << 6) & 0b11000000;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[1] |= chunk << 1 & 0b00111110;
+    output[1] |= (chunk << 1) & 0b00111110;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[1] |= chunk >> 4 & 0b00000001;
-    output[2] |= chunk << 4 & 0b11110000;
+    output[1] |= (chunk >> 4) & 0b00000001;
+    output[2] |= (chunk << 4) & 0b11110000;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[2] |= chunk >> 1 & 0b00001111;
-    output[3] |= chunk << 7 & 0b10000000;
+    output[2] |= (chunk >> 1) & 0b00001111;
+    output[3] |= (chunk << 7) & 0b10000000;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[3] |= chunk << 2 & 0b01111100;
+    output[3] |= (chunk << 2) & 0b01111100;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())
         .ok_or(anyhow!("invalid character"))?;
-    output[3] |= chunk >> 3 & 0b00000011;
-    output[4] |= chunk << 5 & 0b11100000;
+    output[3] |= (chunk >> 3) & 0b00000011;
+    output[4] |= (chunk << 5) & 0b11100000;
 
     let chunk = *INVERSE_CHARACTER_MAP
         .get(&iter.next().unwrap())

crates/bwmap/src/util.rs

@@ -261,17 +261,17 @@ impl<'a> CursorSlicer<'a> {
             let lower_16_bits = 0_u16;
             let upper_16_bits = 0_u16;
             self.current_offset += 0;
-            upper_16_bits << 8 | lower_16_bits
+            (upper_16_bits << 8) | lower_16_bits
         } else if self.s.len() <= self.current_offset + 1 {
             let lower_16_bits = self.s[self.current_offset] as u16;
             let upper_16_bits = 0_u16;
             self.current_offset += 1;
-            upper_16_bits << 8 | lower_16_bits
+            (upper_16_bits << 8) | lower_16_bits
         } else {
             let lower_16_bits = self.s[self.current_offset] as u16;
             let upper_16_bits = self.s[self.current_offset + 1] as u16;
             self.current_offset += 2;
-            upper_16_bits << 8 | lower_16_bits
+            (upper_16_bits << 8) | lower_16_bits
         }
     }
 

crates/bwmapserver/src/gsfs.rs

@@ -33,19 +33,17 @@ async fn read_file_as_stream(
     })
 }
 
-pub async fn gsfs_put_mapblob(
+async fn gsfs_put(
     client: &Client,
     endpoint: &str,
-    path: impl AsRef<Path> + 'static,
-    mapblob_hash: &str,
+    src: impl Stream<Item = Result<Bytes, std::io::Error>> + Sync + Send + 'static,
+    dst: impl AsRef<str> + 'static,
 ) -> Result<()> {
+    anyhow::ensure!(dst.as_ref().starts_with("/"), "dst must start with /");
+
     let response = client
-        .put(format!(
-            "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}"
-        ))
-        .body(Body::wrap_stream(
-            read_file_as_stream(path, 1024 * 1024).await?,
-        ))
+        .put(format!("{endpoint}/api/fs/scmscx.com{}", dst.as_ref()))
+        .body(Body::wrap_stream(src))
         .send()
         .await?;
 
@@ -60,15 +58,15 @@ pub async fn gsfs_put_mapblob(
     Ok(())
 }
 
-pub async fn gsfs_get_mapblob(
+async fn gsfs_get(
     client: &Client,
     endpoint: &str,
-    mapblob_hash: &str,
+    path: impl AsRef<str> + 'static,
 ) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>> {
+    anyhow::ensure!(path.as_ref().starts_with("/"), "path must start with /");
+
     let response = client
-        .get(format!(
-            "{endpoint}/api/fs/scmscx.com/mapblob/{mapblob_hash}"
-        ))
+        .get(format!("{endpoint}/api/fs/scmscx.com{}", path.as_ref()))
         .send()
         .await?;
 
@@ -82,3 +80,49 @@ pub async fn gsfs_get_mapblob(
 
     Ok(response.bytes_stream())
 }
+
+pub async fn gsfs_put_mapblob(
+    client: &Client,
+    endpoint: &str,
+    path: impl AsRef<Path> + 'static,
+    mapblob_hash: &str,
+) -> Result<()> {
+    gsfs_put(
+        client,
+        endpoint,
+        read_file_as_stream(path, 1024 * 1024).await?,
+        format!("/mapblob/{mapblob_hash}"),
+    )
+    .await
+}
+
+pub async fn gsfs_get_mapblob(
+    client: &Client,
+    endpoint: &str,
+    mapblob_hash: &str,
+) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>> {
+    gsfs_get(client, endpoint, format!("/mapblob/{mapblob_hash}")).await
+}
+
+pub async fn gsfs_put_chkblob(
+    client: &Client,
+    endpoint: &str,
+    path: impl AsRef<Path> + 'static,
+    chkblob_hash: &str,
+) -> Result<()> {
+    gsfs_put(
+        client,
+        endpoint,
+        read_file_as_stream(path, 1024 * 1024).await?,
+        format!("/chkblob/{chkblob_hash}"),
+    )
+    .await
+}
+
+// pub async fn gsfs_get_chkblob(
+//     client: &Client,
+//     endpoint: &str,
+//     chkblob_hash: &str,
+// ) -> Result<impl Stream<Item = Result<Bytes, reqwest::Error>>> {
+//     gsfs_get(client, endpoint, format!("/chkblob/{chkblob_hash}")).await
+// }

crates/bwmapserver/src/pumpers.rs

@@ -1,4 +1,7 @@
-use crate::gsfs::gsfs_put_mapblob;
+use crate::{
+    gsfs::{gsfs_put_chkblob, gsfs_put_mapblob},
+    util::calculate_hash_of_object,
+};
 use anyhow::Result;
 use async_stream::stream;
 use backblaze::api::{b2_authorize_account, b2_get_upload_url, b2_upload_file};
@@ -33,28 +36,61 @@ pub async fn start_gsfs_pumper(client: reqwest::Client) -> Result<()> {
                     }
                 }
 
-                let Ok(sha256) = entry.file_name().into_string() else {
+                let Ok(mapblob_hash) = entry.file_name().into_string() else {
                     error!("could not stringify filename: {:?}", entry.file_name());
                     continue;
                 };
 
-                info!("attempting to upload file to gsfs: {sha256}");
-                match gsfs_put_mapblob(&client, &endpoint, entry.path(), &sha256).await {
+                info!("attempting to upload mapblob to gsfs: {mapblob_hash}");
+                match gsfs_put_mapblob(&client, &endpoint, entry.path(), &mapblob_hash).await {
                     Ok(_) => {}
                     Err(err) => {
-                        error!("failed to put file to gsfs: {err}, sha256: {sha256}");
+                        error!(
+                            "failed to put mapblob to gsfs: {err}, mapblob_hash: {mapblob_hash}"
+                        );
                         continue;
                     }
                 }
 
+                let chkblob_hash = {
+                    let path = entry.path().to_path_buf();
+                    let chkblob = match tokio::task::spawn_blocking(move || {
+                        bwmpq::get_chk_from_mpq_filename(&path)
+                    })
+                    .await
+                    {
+                        Ok(Ok(chkblob)) => chkblob,
+                        Ok(Err(error)) => {
+                            error!("failed to parse mpq: {error}");
+                            continue;
+                        }
+                        Err(error) => {
+                            error!("failed to join with task: {error}");
+                            continue;
+                        }
+                    };
+
+                    let chkblob_hash = calculate_hash_of_object(chkblob);
+
+                    info!("attempting to upload chkblob to gsfs: {chkblob_hash}");
+                    match gsfs_put_chkblob(&client, &endpoint, entry.path(), &chkblob_hash).await {
+                        Ok(_) => {}
+                        Err(err) => {
+                            error!("failed to put chkblob to gsfs: {err}, mapblob_hash: {mapblob_hash}, chkblob_hash: {chkblob_hash}");
+                            continue;
+                        }
+                    }
+
+                    chkblob_hash
+                };
+
                 if let Err(e) = tokio::fs::remove_file(entry.path()).await {
                     error!("failed to remove file: {e}");
                     continue;
                 }
 
                 info!(
-                    "Successfully uploaded file to gsfs: {}",
-                    entry.path().display()
+                    "Successfully uploaded file to gsfs: /scmscx.com/mapblob/{mapblob_hash}, /scmscx.com/chkblob/{chkblob_hash}",
                 );
             }
         }

crates/bwmapserver/src/util.rs

@@ -23,6 +23,13 @@ pub(crate) fn sanitize_sc_string(s: &str) -> String {
     }
 }
 
+pub fn calculate_hash_of_object(object: impl AsRef<[u8]>) -> String {
+    use sha2::Digest;
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(&object);
+    format!("{:x}", hasher.finalize())
+}
+
 // pub(crate) fn sanitize_sc_scenario_string(s: &str) -> String {
 //     // split string by left or right marks
 

crates/bwminimaprender/src/minimap.rs

@@ -126,7 +126,7 @@ pub fn calculate_perceptual_hash(png: &[u8]) -> Result<[u8; 16 * 16 / 8]> {
         .map(|x| if *x < ph16x16_avg { 0 } else { 1 })
         .collect::<Vec<u8>>()
         .chunks_exact(8)
-        .map(|x| x.iter().fold(0u8, |acc, x| acc << 1 | *x))
+        .map(|x| x.iter().fold(0u8, |acc, x| (acc << 1) | *x))
         .collect();
 
     anyhow::Ok(ph16x16.as_slice().try_into()?)

crates/stormlib-bindings/src/bindings.rs

@@ -4,5 +4,6 @@
 #![allow(improper_ctypes)]
 #![allow(dead_code)]
 #![allow(deref_nullptr)]
+#![allow(clippy::ptr_offset_with_cast)]
 
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));