Actually be lazy

Kimahriman · Jan 7, 2024 · 1939dfa · 1939dfa
1 parent e0fc13b
commit 1939dfa
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/crates/hdfs-native/src/hdfs/datanode.rs b/crates/hdfs-native/src/hdfs/datanode.rs
@@ -853,6 +853,11 @@ impl StripedBlockWriter {
             .zip(self.block_writers.iter_mut())
             .enumerate()
         {
+            // Don't create the blocks on the data nodes until there's actually data for it
+            if data.is_empty() {
+                continue;
+            }
+
             if writer.is_none() {
                 let mut cloned = self.block.clone();
                 cloned.b.block_id += index as u64;