🙋 Implement feed auditing (#55)

* Implemented audit; working on a bad-data test

* Appease cargo fmt

* Add a test to ensure that audit catches corrupt data

* Add a proper return type for feed.audit

src/audit.rs

@@ -0,0 +1,20 @@
+/// The audit report for a feed, created by the `.audit()` method.
+#[derive(Debug, PartialEq, Clone)]
+pub struct Audit {
+  /// The number of valid blocks identified
+  pub valid_blocks: usize,
+  /// The number of invalid blocks identified
+  pub invalid_blocks: usize,
+}
+
+impl Audit {
+  /// Access the `valid_blocks` field from the proof.
+  pub fn valid_blocks(&self) -> usize {
+    self.valid_blocks
+  }
+
+  /// Access the `invalid_blocks` field from the proof.
+  pub fn invalid_blocks(&self) -> usize {
+    self.invalid_blocks
+  }
+}

src/bitfield/mod.rs

@@ -165,10 +165,11 @@ impl Bitfield {
 
     self.iterator.seek(start);
 
-    while self.iterator.index() < max_len && self
-      .index
-      .set_byte(self.iterator.index(), byte)
-      .is_changed()
+    while self.iterator.index() < max_len
+      && self
+        .index
+        .set_byte(self.iterator.index(), byte)
+        .is_changed()
     {
       if self.iterator.is_left() {
         let index: usize = self.index.get_byte(self.iterator.sibling()).into();

src/feed.rs

@@ -4,6 +4,7 @@ use crate::feed_builder::FeedBuilder;
 use crate::replicate::{Message, Peer};
 pub use crate::storage::{Node, NodeTrait, Storage, Store};
 
+use crate::audit::Audit;
 use crate::bitfield::Bitfield;
 use crate::crypto::{generate_keypair, sign, verify, Hash, Merkle};
 use crate::proof::Proof;
@@ -514,6 +515,31 @@ where
     Ok(extra_nodes)
   }
 
+  /// Audit all data in the feed. Checks that all current data matches
+  /// the hashes in the merkle tree, and clears the bitfield if not.
+  /// The tuple returns is (valid_blocks, invalid_blocks)
+  pub fn audit(&mut self) -> Result<Audit> {
+    let mut valid_blocks = 0;
+    let mut invalid_blocks = 0;
+    for index in 0..self.length {
+      if self.bitfield.get(index) {
+        let node = self.storage.get_node(2 * index)?;
+        let data = self.storage.get_data(index)?;
+        let data_hash = Hash::from_leaf(&data);
+        if node.hash == data_hash.as_bytes() {
+          valid_blocks += 1;
+        } else {
+          invalid_blocks += 1;
+          self.bitfield.set(index, false);
+        }
+      }
+    }
+    Ok(Audit {
+      valid_blocks: valid_blocks,
+      invalid_blocks: invalid_blocks,
+    })
+  }
+
   /// (unimplemented) Provide a range of data to download.
   pub fn download(&mut self, _range: Range<usize>) -> Result<()> {
     unimplemented!();

src/lib.rs

@@ -42,6 +42,7 @@ extern crate tree_index;
 pub mod bitfield;
 pub mod prelude;
 
+mod audit;
 mod crypto;
 mod event;
 mod feed;

src/storage/node.rs

@@ -1,5 +1,5 @@
-use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use crate::Result;
+use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use flat_tree;
 use merkle_tree_stream::Node as NodeTrait;
 use pretty_hash::fmt as pretty_fmt;

tests/feed.rs

@@ -11,7 +11,10 @@ use common::create_feed;
 use hypercore::{
   generate_keypair, Feed, NodeTrait, PublicKey, SecretKey, Storage,
 };
+use std::env::temp_dir;
 use std::fmt::Debug;
+use std::fs;
+use std::io::Write;
 
 #[test]
 fn create_with_key() {
@@ -183,3 +186,69 @@ fn copy_keys(
     _ => panic!("<tests/common>: Could not access secret key"),
   }
 }
+
+#[test]
+fn audit() {
+  let mut feed = create_feed(50).unwrap();
+  feed.append(b"hello").unwrap();
+  feed.append(b"world").unwrap();
+  match feed.audit() {
+    Ok(audit_report) => {
+      assert_eq!(audit_report.valid_blocks, 2);
+      assert_eq!(audit_report.invalid_blocks, 0);
+    }
+    Err(e) => {
+      panic!(e);
+    }
+  }
+}
+
+#[test]
+fn audit_bad_data() {
+  let mut dir = temp_dir();
+  dir.push("audit_bad_data");
+  let storage = Storage::new_disk(&dir).unwrap();
+  let mut feed = Feed::with_storage(storage).unwrap();
+  feed.append(b"hello").unwrap();
+  feed.append(b"world").unwrap();
+  let datapath = dir.join("data");
+  let mut hypercore_data = fs::OpenOptions::new()
+    .write(true)
+    .open(datapath)
+    .expect("Unable to open the hypercore's data file!");
+  hypercore_data
+    .write("yello".as_bytes())
+    .expect("Unable to corrupt the hypercore data file!");
+
+  match feed.audit() {
+    Ok(audit_report) => {
+      assert_eq!(audit_report.valid_blocks, 1);
+      assert_eq!(audit_report.invalid_blocks, 1);
+      // Ensure that audit has cleared up the invalid block
+      match feed.audit() {
+        Ok(audit_report) => {
+          assert_eq!(
+            audit_report.valid_blocks, 1,
+            "Audit did not clean up the invalid block!"
+          );
+          assert_eq!(
+            audit_report.invalid_blocks, 0,
+            "Audit did not clean up the invalid block!"
+          );
+          fs::remove_dir_all(dir)
+            .expect("Should be able to remove our temporary directory");
+        }
+        Err(e) => {
+          fs::remove_dir_all(dir)
+            .expect("Should be able to remove our temporary directory");
+          panic!(e);
+        }
+      }
+    }
+    Err(e) => {
+      fs::remove_dir_all(dir)
+        .expect("Should be able to remove our temporary directory");
+      panic!(e);
+    }
+  }
+}