Store L2=>Witness in ledgerdb (#830)

* Remove mutex from Witness

* Store L2=>Witness in ledgerdb

* Revert "Remove mutex from Witness"

This reverts commit 6740398.

* Revert WitnessesByNumber table from ledgerdb schema

* Add method to get_l2_witness from ledgerdb

* Load witness from ledgerdb

crates/prover/src/runner.rs

@@ -332,16 +332,16 @@ where
 
             let mut soft_confirmations: VecDeque<Vec<SignedSoftConfirmationBatch>> =
                 VecDeque::new();
-            let mut state_transition_witnesses: VecDeque<Vec<Stf::Witness>> = VecDeque::new();
+            let mut commitments_l2_heights = vec![];
             let mut da_block_headers_of_soft_confirmations: VecDeque<
                 Vec<<<Da as DaService>::Spec as DaSpec>::BlockHeader>,
             > = VecDeque::new();
 
             let mut traversed_l1_tuples = vec![];
 
             for sequencer_commitment in sequencer_commitments.clone().into_iter() {
+                let mut commitment_l2_heights = vec![];
                 let mut sof_soft_confirmations_to_push = vec![];
-                let mut state_transition_witnesses_to_push = vec![];
                 let mut da_block_headers_to_push: Vec<
                     <<Da as DaService>::Spec as DaSpec>::BlockHeader,
                 > = vec![];
@@ -434,6 +434,7 @@ where
                     let mut signed_soft_confirmation: SignedSoftConfirmationBatch =
                         soft_batch.clone().into();
 
+                    commitment_l2_heights.push(l2_height);
                     sof_soft_confirmations_to_push.push(signed_soft_confirmation.clone());
 
                     // The filtered block of soft batch, which is the block at the da_slot_height of soft batch
@@ -469,7 +470,8 @@ where
                         &mut signed_soft_confirmation,
                     );
 
-                    state_transition_witnesses_to_push.push(slot_result.witness);
+                    self.ledger_db
+                        .set_l2_witness(l2_height, &slot_result.witness)?;
 
                     for receipt in slot_result.batch_receipts {
                         data_to_commit.add_batch(receipt);
@@ -521,15 +523,28 @@ where
                     l2_height += 1;
                 }
 
+                commitments_l2_heights.push(commitment_l2_heights);
                 soft_confirmations.push_back(sof_soft_confirmations_to_push);
-                state_transition_witnesses.push_back(state_transition_witnesses_to_push);
                 da_block_headers_of_soft_confirmations.push_back(da_block_headers_to_push);
             }
 
             info!("Sending for proving");
 
             let hash = da_block_header_of_commitments.hash();
 
+            let mut state_transition_witnesses = VecDeque::new();
+            for l2_heights in commitments_l2_heights {
+                let mut witnesses = vec![];
+                for l2_height in l2_heights {
+                    let witness = self
+                        .ledger_db
+                        .get_l2_witness::<Stf::Witness>(l2_height)?
+                        .expect("A witness must be present");
+                    witnesses.push(witness);
+                }
+                state_transition_witnesses.push_back(witnesses);
+            }
+
             let transition_data: StateTransitionData<Stf::StateRoot, Stf::Witness, Da::Spec> =
                 StateTransitionData {
                     initial_state_root,

crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs

@@ -1,6 +1,7 @@
 use std::path::Path;
 use std::sync::{Arc, Mutex};
 
+use serde::de::DeserializeOwned;
 use serde::Serialize;
 use sov_rollup_interface::da::{DaSpec, SequencerCommitment};
 use sov_rollup_interface::services::da::SlotData;
@@ -12,7 +13,7 @@ use tracing::instrument;
 use crate::rocks_db_config::gen_rocksdb_options;
 use crate::schema::tables::{
     BatchByHash, BatchByNumber, CommitmentsByNumber, EventByKey, EventByNumber, L2RangeByL1Height,
-    LastSequencerCommitmentSent, ProofBySlotNumber, ProverLastScannedSlot, SlotByHash,
+    L2Witness, LastSequencerCommitmentSent, ProofBySlotNumber, ProverLastScannedSlot, SlotByHash,
     SlotByNumber, SoftBatchByHash, SoftBatchByNumber, SoftConfirmationStatus, TxByHash, TxByNumber,
     VerifiedProofsBySlotNumber, LEDGER_TABLES,
 };
@@ -564,6 +565,37 @@ impl LedgerDB {
         Ok(())
     }
 
+    /// Get the witness by L2 height
+    #[instrument(level = "trace", skip_all, err)]
+    pub fn get_l2_witness<Witness: DeserializeOwned>(
+        &self,
+        l2_height: u64,
+    ) -> anyhow::Result<Option<Witness>> {
+        let buf = self.db.get::<L2Witness>(&BatchNumber(l2_height))?;
+        if let Some(buf) = buf {
+            let witness = bincode::deserialize(&buf)?;
+            Ok(Some(witness))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Set the witness by L2 height
+    #[instrument(level = "trace", skip_all, err, ret)]
+    pub fn set_l2_witness<Witness: Serialize>(
+        &self,
+        l2_height: u64,
+        witness: &Witness,
+    ) -> anyhow::Result<()> {
+        let buf = bincode::serialize(witness)?;
+        let mut schema_batch = SchemaBatch::new();
+        schema_batch.put::<L2Witness>(&BatchNumber(l2_height), &buf)?;
+
+        self.db.write_schemas(schema_batch)?;
+
+        Ok(())
+    }
+
     /// Gets the commitments in the da slot with given height if any
     /// Adds the new coming commitment info
     #[instrument(level = "trace", skip(self, commitment), err, ret)]

crates/sovereign-sdk/full-node/db/sov-db/src/schema/tables.rs

@@ -56,6 +56,7 @@ pub const LEDGER_TABLES: &[&str] = &[
     SoftBatchByNumber::table_name(),
     SoftBatchByHash::table_name(),
     L2RangeByL1Height::table_name(),
+    L2Witness::table_name(),
     LastSequencerCommitmentSent::table_name(),
     ProverLastScannedSlot::table_name(),
     BatchByHash::table_name(),
@@ -249,6 +250,11 @@ define_table_with_default_codec!(
     (L2RangeByL1Height) SlotNumber => L2HeightRange
 );
 
+define_table_with_default_codec!(
+    /// The primary source of witness by L2 height
+    (L2Witness) BatchNumber => Vec<u8>
+);
+
 define_table_with_seek_key_codec!(
     /// Sequencer uses this table to store the last commitment it sent
     (LastSequencerCommitmentSent) () => SlotNumber