Handle DataNode failures in replicated writes (#164)

Kimahriman · Dec 5, 2024 · f72ab92 · f72ab92
1 parent c7d40bd
commit f72ab92
Show file tree

Hide file tree

Showing 8 changed files with 599 additions and 302 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/rust/minidfs/src/main/java/main/Main.java b/rust/minidfs/src/main/java/main/Main.java
@@ -106,7 +106,7 @@ public static void main(String args[]) throws Exception {
         } else {
             MiniDFSNNTopology nnTopology = generateTopology(flags, hdfsConf);
 
-            int numDataNodes = 1;
+            int numDataNodes = 3;
             if (flags.contains("ec")) {
                 // Enough for the largest EC policy
                 numDataNodes = 14;

diff --git a/rust/src/file.rs b/rust/src/file.rs
@@ -208,9 +208,7 @@ impl FileWriter {
             // Not appending to an existing block, just create a new one
             // If there's an existing block writer, close it first
             let extended_block = if let Some(block_writer) = self.block_writer.take() {
-                let extended_block = block_writer.get_extended_block();
-                block_writer.close().await?;
-                Some(extended_block)
+                Some(block_writer.close().await?)
             } else {
                 None
             };
@@ -265,9 +263,7 @@ impl FileWriter {
     pub async fn close(&mut self) -> Result<()> {
         if !self.closed {
             let extended_block = if let Some(block_writer) = self.block_writer.take() {
-                let extended_block = block_writer.get_extended_block();
-                block_writer.close().await?;
-                Some(extended_block)
+                Some(block_writer.close().await?)
             } else {
                 None
             };