merge queue: embarking main (8734fd9) and #8106 together

zebra-scan/src/storage.rs

@@ -104,6 +104,8 @@ impl Storage {
     /// Add the sapling results for `height` to the storage. The results can be any map of
     /// [`TransactionIndex`] to [`SaplingScannedResult`].
     ///
+    /// Also adds empty progress tracking entries every `INSERT_CONTROL_INTERVAL` blocks if needed.
+    ///
     /// # Performance / Hangs
     ///
     /// This method can block while writing database files, so it must be inside spawn_blocking()
@@ -120,7 +122,15 @@ impl Storage {
 
         // Every `INSERT_CONTROL_INTERVAL` we add a new entry to the scanner database for each key
         // so we can track progress made in the last interval even if no transaction was yet found.
-        let is_control_time = height.0 % INSERT_CONTROL_INTERVAL == 0 && sapling_results.is_empty();
+        let needs_control_entry =
+            height.0 % INSERT_CONTROL_INTERVAL == 0 && sapling_results.is_empty();
+
+        // Add scanner progress tracking entry for key.
+        // Defensive programming: add the tracking entry first, so that we don't accidentally
+        // overwrite real results with it. (This is currently prevented by the empty check.)
+        if needs_control_entry {
+            batch.insert_sapling_height(self, sapling_key, height);
+        }
 
         for (index, sapling_result) in sapling_results {
             let index = SaplingScannedDatabaseIndex {
@@ -136,11 +146,6 @@ impl Storage {
             batch.insert_sapling_result(self, entry);
         }
 
-        // Add tracking entry for key.
-        if is_control_time {
-            batch.insert_sapling_height(self, sapling_key, height);
-        }
-
         self.write_batch(batch);
     }
 

zebra-scan/src/storage/db/sapling.rs

@@ -107,27 +107,25 @@ impl Storage {
             Option<SaplingScannedResult>,
         )> = self.db.zs_last_key_value(&sapling_tx_ids);
 
-        loop {
-            let Some((mut last_stored_record_index, _result)) = last_stored_record else {
-                return keys;
-            };
-
+        while let Some((last_stored_record_index, _result)) = last_stored_record {
             let sapling_key = last_stored_record_index.sapling_key.clone();
             let height = last_stored_record_index.tx_loc.height;
 
             let prev_height = keys.insert(sapling_key.clone(), height);
             assert_eq!(
                 prev_height, None,
-                "unexpected duplicate key: keys must only be inserted once\
+                "unexpected duplicate key: keys must only be inserted once \
                  last_stored_record_index: {last_stored_record_index:?}",
             );
 
             // Skip all the results until the next key.
-            last_stored_record_index = SaplingScannedDatabaseIndex::min_for_key(&sapling_key);
-            last_stored_record = self
-                .db
-                .zs_prev_key_value_strictly_before(&sapling_tx_ids, &last_stored_record_index);
+            last_stored_record = self.db.zs_prev_key_value_strictly_before(
+                &sapling_tx_ids,
+                &SaplingScannedDatabaseIndex::min_for_key(&sapling_key),
+            );
         }
+
+        keys
     }
 
     /// Returns the Sapling indexes and results in the supplied range.

zebra-scan/src/storage/db/tests.rs

@@ -1,6 +1,6 @@
 //! General scanner database tests.
 
-use std::sync::Arc;
+use std::{collections::BTreeMap, sync::Arc};
 
 use zebra_chain::{
     block::{Block, Height},
@@ -10,7 +10,7 @@ use zebra_chain::{
 use zebra_state::TransactionIndex;
 
 use crate::{
-    storage::Storage,
+    storage::{Storage, INSERT_CONTROL_INTERVAL},
     tests::{FAKE_SAPLING_VIEWING_KEY, ZECPAGES_SAPLING_VIEWING_KEY},
     Config,
 };
@@ -32,7 +32,15 @@ pub fn add_fake_keys(storage: &mut Storage) {
 }
 
 /// Add fake results to `storage` for testing purposes.
-pub fn add_fake_results(storage: &mut Storage, network: Network, height: Height) {
+///
+/// If `add_progress_marker` is `true`, adds a progress marker.
+/// If it is `false`, adds the transaction hashes from `height`.
+pub fn add_fake_results(
+    storage: &mut Storage,
+    network: Network,
+    height: Height,
+    add_progress_marker: bool,
+) {
     let blocks = match network {
         Mainnet => &*zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS,
         Testnet => &*zebra_test::vectors::CONTINUOUS_TESTNET_BLOCKS,
@@ -44,15 +52,25 @@ pub fn add_fake_results(storage: &mut Storage, network: Network, height: Height)
         .zcash_deserialize_into()
         .expect("test data deserializes");
 
-    // Fake results from the first few blocks
-    storage.add_sapling_results(
-        &ZECPAGES_SAPLING_VIEWING_KEY.to_string(),
-        height,
-        block
-            .transactions
-            .iter()
-            .enumerate()
-            .map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into()))
-            .collect(),
-    );
+    if add_progress_marker {
+        // Fake a progress marker.
+        let next_progress_height = height.0.next_multiple_of(INSERT_CONTROL_INTERVAL);
+        storage.add_sapling_results(
+            &FAKE_SAPLING_VIEWING_KEY.to_string(),
+            Height(next_progress_height),
+            BTreeMap::new(),
+        );
+    } else {
+        // Fake results from the block at `height`.
+        storage.add_sapling_results(
+            &ZECPAGES_SAPLING_VIEWING_KEY.to_string(),
+            height,
+            block
+                .transactions
+                .iter()
+                .enumerate()
+                .map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into()))
+                .collect(),
+        );
+    }
 }

zebra-scan/src/storage/db/tests/snapshot.rs

@@ -89,7 +89,8 @@ fn test_database_format_with_network(network: Network) {
     //
     // We limit the number of blocks, because we create 2 snapshots per block, one for each network.
     for height in 0..=2 {
-        super::add_fake_results(&mut storage, network, Height(height));
+        super::add_fake_results(&mut storage, network, Height(height), true);
+        super::add_fake_results(&mut storage, network, Height(height), false);
 
         let mut settings = insta::Settings::clone_current();
         settings.set_snapshot_suffix(format!("{net_suffix}_{height}"));
@@ -98,8 +99,6 @@ fn test_database_format_with_network(network: Network) {
         settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
         settings.bind(|| snapshot_typed_result_data(&storage));
     }
-
-    // TODO: add an empty marker result after PR #8080 merges
 }
 
 /// Snapshot the data in each column family, using `cargo insta` and RON serialization.

zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_0.snap

@@ -3,5 +3,6 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
 expression: sapling_results
 ---
 {
+  Height(0): [],
   Height(999999): [],
 }

zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_1.snap

@@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
 expression: sapling_results
 ---
 {
+  Height(0): [],
+  Height(1000): [],
   Height(999999): [],
 }

zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@mainnet_2.snap

@@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
 expression: sapling_results
 ---
 {
+  Height(0): [],
+  Height(1000): [],
   Height(999999): [],
 }

zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_0.snap

@@ -3,5 +3,6 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
 expression: sapling_results
 ---
 {
+  Height(0): [],
   Height(999999): [],
 }

zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_1.snap

@@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
 expression: sapling_results
 ---
 {
+  Height(0): [],
+  Height(1000): [],
   Height(999999): [],
 }

zebra-scan/src/storage/db/tests/snapshots/sapling_key_1_results@testnet_2.snap

@@ -3,5 +3,7 @@ source: zebra-scan/src/storage/db/tests/snapshot.rs
 expression: sapling_results
 ---
 {
+  Height(0): [],
+  Height(1000): [],
   Height(999999): [],
 }

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap

@@ -11,6 +11,10 @@ expression: cf_data
     k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
     v: "",
   ),
+  KV(
+    k: "7a78766965777366616b650000000000",
+    v: "",
+  ),
   KV(
     k: "7a78766965777366616b650f423f0000",
     v: "",

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap

@@ -15,6 +15,14 @@ expression: cf_data
     k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
     v: "",
   ),
+  KV(
+    k: "7a78766965777366616b650000000000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650003e80000",
+    v: "",
+  ),
   KV(
     k: "7a78766965777366616b650f423f0000",
     v: "",

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap

@@ -19,6 +19,14 @@ expression: cf_data
     k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
     v: "",
   ),
+  KV(
+    k: "7a78766965777366616b650000000000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650003e80000",
+    v: "",
+  ),
   KV(
     k: "7a78766965777366616b650f423f0000",
     v: "",

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap

@@ -11,6 +11,10 @@ expression: cf_data
     k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
     v: "",
   ),
+  KV(
+    k: "7a78766965777366616b650000000000",
+    v: "",
+  ),
   KV(
     k: "7a78766965777366616b650f423f0000",
     v: "",

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap

@@ -15,6 +15,14 @@ expression: cf_data
     k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
     v: "",
   ),
+  KV(
+    k: "7a78766965777366616b650000000000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650003e80000",
+    v: "",
+  ),
   KV(
     k: "7a78766965777366616b650f423f0000",
     v: "",

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap

@@ -19,6 +19,14 @@ expression: cf_data
     k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
     v: "",
   ),
+  KV(
+    k: "7a78766965777366616b650000000000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650003e80000",
+    v: "",
+  ),
   KV(
     k: "7a78766965777366616b650f423f0000",
     v: "",