From 502b08c2336dc5c6496531d0927ce29b3aee216c Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 27 Jan 2025 15:11:13 +0000
Subject: [PATCH 01/38] add stable map for blocks

---
 rs/ledger_suite/icrc1/ledger/src/lib.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 17a38358ac4..68ff4b7c480 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -499,6 +499,7 @@ const UPGRADES_MEMORY_ID: MemoryId = MemoryId::new(0);
 const ALLOWANCES_MEMORY_ID: MemoryId = MemoryId::new(1);
 const ALLOWANCES_EXPIRATIONS_MEMORY_ID: MemoryId = MemoryId::new(2);
 const BALANCES_MEMORY_ID: MemoryId = MemoryId::new(3);
+const BLOCKS_MEMORY_ID: MemoryId = MemoryId::new(4);
 
 thread_local! {
     static MEMORY_MANAGER: RefCell<MemoryManager<DefaultMemoryImpl>> = RefCell::new(
@@ -524,6 +525,10 @@ thread_local! {
     // account -> tokens - map storing ledger balances.
     pub static BALANCES_MEMORY: RefCell<StableBTreeMap<Account, Tokens, VirtualMemory<DefaultMemoryImpl>>> =
         MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(BALANCES_MEMORY_ID))));
+
+    // block_index -> block
+    pub static BLOCKS_MEMORY: RefCell<StableBTreeMap<u64, Vec<u8>, VirtualMemory<DefaultMemoryImpl>>> =
+        MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(BLOCKS_MEMORY_ID))));
 }
 
 #[derive(Copy, Clone, Serialize, Deserialize, Debug)]

From 3c20ea41d034621fc534591c241cdcc202fcab14 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 30 Jan 2025 16:03:00 +0000
Subject: [PATCH 02/38] replace blocks vector with HeapBlockData

---
 .../ledger_canister_core/src/blockchain.rs    | 97 +++++++++++++++----
 .../common/ledger_canister_core/src/ledger.rs | 14 ++-
 rs/ledger_suite/icp/ledger/src/lib.rs         | 11 ++-
 rs/ledger_suite/icp/ledger/src/main.rs        | 48 +++++++--
 rs/ledger_suite/icrc1/ledger/src/lib.rs       | 11 ++-
 rs/ledger_suite/icrc1/ledger/src/main.rs      |  4 +-
 6 files changed, 147 insertions(+), 38 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 0fe6e4e4a8c..3779df88425 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -4,18 +4,69 @@ use crate::{
 };
 use serde::{Deserialize, Serialize};
 use std::collections::VecDeque;
-use std::convert::TryFrom;
 use std::sync::{Arc, RwLock};
 
 use ic_ledger_core::block::{BlockIndex, BlockType, EncodedBlock};
 use ic_ledger_core::timestamp::TimeStamp;
 use ic_ledger_hash_of::HashOf;
+use std::ops::Range;
+
+pub trait BlockData {
+    fn add_block(&mut self, index: u64, block: EncodedBlock) -> Result<(), String>;
+    fn get_blocks(&self, range: Range<usize>) -> Vec<EncodedBlock>;
+    fn get_block(&self, index: u64) -> Option<EncodedBlock>;
+    fn remove_blocks(&mut self, num_blocks: u64);
+    fn len(&self) -> u64;
+    fn is_empty(&self) -> bool;
+    fn last(&self) -> Option<EncodedBlock>;
+}
+
+#[derive(Debug, Deserialize, Serialize, Default)]
+#[serde(transparent)]
+pub struct HeapBlockData {
+    blocks: Vec<EncodedBlock>,
+}
+
+impl BlockData for HeapBlockData {
+    fn add_block(&mut self, _index: u64, block: EncodedBlock) -> Result<(), String> {
+        self.blocks.push(block);
+        Ok(())
+    }
+
+    fn get_blocks(&self, range: Range<usize>) -> Vec<EncodedBlock> {
+        self.blocks[range].to_vec()
+    }
+
+    fn get_block(&self, index: u64) -> Option<EncodedBlock> {
+        self.blocks.get(index as usize).cloned()
+    }
+
+    fn remove_blocks(&mut self, num_blocks: u64) {
+        self.blocks = self.blocks.split_off(num_blocks as usize);
+    }
+
+    fn len(&self) -> u64 {
+        self.blocks.len() as u64
+    }
+
+    fn is_empty(&self) -> bool {
+        self.blocks.is_empty()
+    }
+
+    fn last(&self) -> Option<EncodedBlock> {
+        self.blocks.last().cloned()
+    }
+}
 
 /// Stores a chain of transactions with their metadata
 #[derive(Debug, Deserialize, Serialize)]
-#[serde(bound = "")]
-pub struct Blockchain<Rt: Runtime, Wasm: ArchiveCanisterWasm> {
-    pub blocks: Vec<EncodedBlock>,
+#[serde(bound = "BD: Serialize, for<'a> BD: Deserialize<'a>")]
+pub struct Blockchain<Rt: Runtime, Wasm: ArchiveCanisterWasm, BD>
+where
+    BD: BlockData + Serialize + Default,
+    for<'a> BD: Deserialize<'a>,
+{
+    blocks: BD,
     pub last_hash: Option<HashOf<EncodedBlock>>,
 
     /// The timestamp of the most recent block. Must be monotonically
@@ -32,10 +83,14 @@ pub struct Blockchain<Rt: Runtime, Wasm: ArchiveCanisterWasm> {
     pub num_archived_blocks: u64,
 }
 
-impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Default for Blockchain<Rt, Wasm> {
+impl<Rt: Runtime, Wasm: ArchiveCanisterWasm, BD> Default for Blockchain<Rt, Wasm, BD>
+where
+    BD: BlockData + Serialize + Default,
+    for<'a> BD: Deserialize<'a>,
+{
     fn default() -> Self {
         Self {
-            blocks: vec![],
+            blocks: Default::default(),
             last_hash: None,
             last_timestamp: TimeStamp::from_nanos_since_unix_epoch(0),
             archive: Arc::new(RwLock::new(None)),
@@ -44,7 +99,11 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Default for Blockchain<Rt, Wasm> {
     }
 }
 
-impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
+impl<Rt: Runtime, Wasm: ArchiveCanisterWasm, BD> Blockchain<Rt, Wasm, BD>
+where
+    BD: BlockData + Serialize + Default,
+    for<'a> BD: Deserialize<'a>,
+{
     pub fn new_with_archive(archive_options: ArchiveOptions) -> Self {
         Self {
             archive: Arc::new(RwLock::new(Some(Archive::new(archive_options)))),
@@ -68,20 +127,20 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
         self.last_timestamp = block.timestamp();
         let encoded_block = block.encode();
         self.last_hash = Some(B::block_hash(&encoded_block));
-        self.blocks.push(encoded_block);
+        self.blocks.add_block(self.chain_length(), encoded_block)?;
         Ok(self.chain_length().checked_sub(1).unwrap())
     }
 
-    pub fn get(&self, height: BlockIndex) -> Option<&EncodedBlock> {
+    pub fn get(&self, height: BlockIndex) -> Option<EncodedBlock> {
         if height < self.num_archived_blocks() {
             None
         } else {
             self.blocks
-                .get(usize::try_from(height - self.num_archived_blocks()).unwrap())
+                .get_block(height.saturating_sub(self.num_archived_blocks()))
         }
     }
 
-    pub fn last(&self) -> Option<&EncodedBlock> {
+    pub fn last(&self) -> Option<EncodedBlock> {
         self.blocks.last()
     }
 
@@ -90,7 +149,7 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
     }
 
     pub fn num_unarchived_blocks(&self) -> u64 {
-        self.blocks.len() as u64
+        self.blocks.len()
     }
 
     /// The range of block indices that are not archived yet.
@@ -103,7 +162,7 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
     /// # Panic
     ///
     /// This function panics if the specified range is not a subset of locally available blocks.
-    pub fn block_slice(&self, local_blocks: std::ops::Range<u64>) -> &[EncodedBlock] {
+    pub fn block_slice(&self, local_blocks: std::ops::Range<u64>) -> Vec<EncodedBlock> {
         use crate::range_utils::{is_subrange, offset};
 
         assert!(
@@ -113,7 +172,8 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
             self.local_block_range()
         );
 
-        &self.blocks[offset(&local_blocks, self.num_archived_blocks)]
+        self.blocks
+            .get_blocks(offset(&local_blocks, self.num_archived_blocks))
     }
 
     pub fn chain_length(&self) -> BlockIndex {
@@ -123,14 +183,14 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
     pub fn remove_archived_blocks(&mut self, len: usize) {
         // redundant since split_off would panic, but here we can give a more
         // descriptive message
-        if len > self.blocks.len() {
+        if len as u64 > self.blocks.len() {
             panic!(
                 "Asked to remove more blocks than present. Present: {}, to remove: {}",
                 self.blocks.len(),
                 len
             );
         }
-        self.blocks = self.blocks.split_off(len);
+        self.blocks.remove_blocks(len as u64);
         self.num_archived_blocks += len as u64;
     }
 
@@ -151,7 +211,10 @@ impl<Rt: Runtime, Wasm: ArchiveCanisterWasm> Blockchain<Rt, Wasm> {
         }
 
         let blocks_to_archive: VecDeque<EncodedBlock> =
-            VecDeque::from(self.blocks[0..num_blocks_to_archive.min(num_blocks_before)].to_vec());
+            VecDeque::from(self.blocks.get_blocks(Range {
+                start: 0,
+                end: num_blocks_to_archive.min(num_blocks_before),
+            }));
 
         println!(
             "get_blocks_for_archiving(): trigger_threshold: {}, num_blocks: {}, blocks before archiving: {}, blocks to archive: {}",
diff --git a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs
index 30e9d972a98..dab4ae0558a 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs
@@ -1,4 +1,9 @@
-use crate::{archive::ArchiveCanisterWasm, blockchain::Blockchain, range_utils, runtime::Runtime};
+use crate::{
+    archive::ArchiveCanisterWasm,
+    blockchain::{BlockData, Blockchain},
+    range_utils,
+    runtime::Runtime,
+};
 use ic_base_types::CanisterId;
 use ic_canister_log::{log, Sink};
 use ic_ledger_core::approvals::{
@@ -145,6 +150,7 @@ pub trait LedgerData: LedgerContext {
     type Transaction: LedgerTransaction<AccountId = Self::AccountId, Tokens = Self::Tokens>
         + Ord
         + Clone;
+    type BlockData: BlockData + Serialize + Default + for<'a> Deserialize<'a>;
 
     // Purge configuration
 
@@ -175,8 +181,10 @@ pub trait LedgerData: LedgerContext {
 
     // Ledger data structures
 
-    fn blockchain(&self) -> &Blockchain<Self::Runtime, Self::ArchiveWasm>;
-    fn blockchain_mut(&mut self) -> &mut Blockchain<Self::Runtime, Self::ArchiveWasm>;
+    fn blockchain(&self) -> &Blockchain<Self::Runtime, Self::ArchiveWasm, Self::BlockData>;
+    fn blockchain_mut(
+        &mut self,
+    ) -> &mut Blockchain<Self::Runtime, Self::ArchiveWasm, Self::BlockData>;
 
     fn transactions_by_hash(&self) -> &BTreeMap<HashOf<Self::Transaction>, BlockIndex>;
     fn transactions_by_hash_mut(&mut self) -> &mut BTreeMap<HashOf<Self::Transaction>, BlockIndex>;
diff --git a/rs/ledger_suite/icp/ledger/src/lib.rs b/rs/ledger_suite/icp/ledger/src/lib.rs
index d401537dd74..080f1cfaf2c 100644
--- a/rs/ledger_suite/icp/ledger/src/lib.rs
+++ b/rs/ledger_suite/icp/ledger/src/lib.rs
@@ -1,7 +1,7 @@
 use dfn_core::api::{now, trap_with};
 use ic_base_types::{CanisterId, PrincipalId};
 use ic_ledger_canister_core::archive::ArchiveCanisterWasm;
-use ic_ledger_canister_core::blockchain::Blockchain;
+use ic_ledger_canister_core::blockchain::{Blockchain, HeapBlockData};
 use ic_ledger_canister_core::ledger::{
     self as core_ledger, LedgerContext, LedgerData, TransactionInfo,
 };
@@ -181,7 +181,7 @@ pub struct Ledger {
     approvals: LedgerAllowances,
     #[serde(default)]
     stable_approvals: AllowanceTable<StableAllowancesData>,
-    pub blockchain: Blockchain<dfn_runtime::DfnRuntime, IcpLedgerArchiveWasm>,
+    pub blockchain: Blockchain<dfn_runtime::DfnRuntime, IcpLedgerArchiveWasm, HeapBlockData>,
     // A cap on the maximum number of accounts.
     pub maximum_number_of_accounts: usize,
     // When maximum number of accounts is exceeded, a specified number of
@@ -262,6 +262,7 @@ impl LedgerData for Ledger {
     type ArchiveWasm = IcpLedgerArchiveWasm;
     type Transaction = Transaction;
     type Block = Block;
+    type BlockData = HeapBlockData;
 
     fn transaction_window(&self) -> Duration {
         self.transaction_window
@@ -291,11 +292,13 @@ impl LedgerData for Ledger {
         &self.token_symbol
     }
 
-    fn blockchain(&self) -> &Blockchain<Self::Runtime, Self::ArchiveWasm> {
+    fn blockchain(&self) -> &Blockchain<Self::Runtime, Self::ArchiveWasm, Self::BlockData> {
         &self.blockchain
     }
 
-    fn blockchain_mut(&mut self) -> &mut Blockchain<Self::Runtime, Self::ArchiveWasm> {
+    fn blockchain_mut(
+        &mut self,
+    ) -> &mut Blockchain<Self::Runtime, Self::ArchiveWasm, Self::BlockData> {
         &mut self.blockchain
     }
 
diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index 433c750b487..1792263cb54 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -35,8 +35,8 @@ use icp_ledger::{
     max_blocks_per_request, protobuf, tokens_into_proto, AccountBalanceArgs, AccountIdBlob,
     AccountIdentifier, AccountIdentifierByteBuf, ArchiveInfo, ArchivedBlocksRange,
     ArchivedEncodedBlocksRange, Archives, BinaryAccountBalanceArgs, Block, BlockArg, BlockRes,
-    CandidBlock, Decimals, FeatureFlags, GetBlocksArgs, InitArgs, IterBlocksArgs,
-    LedgerCanisterPayload, Memo, Name, Operation, PaymentError, QueryBlocksResponse,
+    CandidBlock, Decimals, FeatureFlags, GetBlocksArgs, GetBlocksRes, InitArgs, IterBlocksArgs,
+    IterBlocksRes, LedgerCanisterPayload, Memo, Name, Operation, PaymentError, QueryBlocksResponse,
     QueryEncodedBlocksResponse, SendArgs, Subaccount, Symbol, TipOfChainRes, TotalSupplyArgs,
     Transaction, TransferArgs, TransferError, TransferFee, TransferFeeArgs, MEMO_SIZE_BYTES,
 };
@@ -585,7 +585,7 @@ fn block(block_index: BlockIndex) -> Option<Result<EncodedBlock, CanisterId>> {
             "[ledger] Checking the ledger for block [{}]",
             block_index
         ));
-        state.blockchain.get(block_index).cloned().map(Ok)
+        state.blockchain.get(block_index).map(Ok)
     }
 }
 
@@ -1315,9 +1315,19 @@ fn icrc1_total_supply_candid() {
 #[export_name = "canister_query iter_blocks_pb"]
 fn iter_blocks_() {
     over(protobuf, |IterBlocksArgs { start, length }| {
-        let blocks = &LEDGER.read().unwrap().blockchain.blocks;
         let length = std::cmp::min(length, max_blocks_per_request(&caller()));
-        icp_ledger::iter_blocks(blocks, start, length)
+        let blocks_len = LEDGER.read().unwrap().blockchain.num_unarchived_blocks() as usize;
+        let start = std::cmp::min(start, blocks_len);
+        let end = std::cmp::min(start + length, blocks_len);
+        let blocks = LEDGER
+            .read()
+            .unwrap()
+            .blockchain
+            .block_slice(std::ops::Range {
+                start: start as u64,
+                end: end as u64,
+            });
+        IterBlocksRes(blocks)
     });
 }
 
@@ -1329,7 +1339,29 @@ fn get_blocks_() {
         let length = std::cmp::min(length, max_blocks_per_request(&caller()) as u64);
         let blockchain = &LEDGER.read().unwrap().blockchain;
         let start_offset = blockchain.num_archived_blocks();
-        icp_ledger::get_blocks(&blockchain.blocks, start_offset, start, length as usize)
+        let blocks_len = LEDGER.read().unwrap().blockchain.num_unarchived_blocks() as usize;
+        let range_from_offset = start;
+        let range_from = start_offset;
+        // Inclusive end of the range of *requested* blocks
+        let requested_range_to = range_from as usize + length as usize - 1;
+        // Inclusive end of the range of *available* blocks
+        let range_to = range_from_offset as usize + blocks_len - 1;
+        // Example: If the Node stores 10 blocks beginning at BlockIndex 100, i.e.
+        // [100 .. 109] then requesting blocks at BlockIndex < 100 or BlockIndex
+        // > 109 is an error
+        if range_from < range_from_offset || requested_range_to > range_to {
+            return GetBlocksRes(Err(format!("Requested blocks outside the range stored in the archive node. Requested [{} .. {}]. Available [{} .. {}].",
+            range_from, requested_range_to, range_from_offset, range_to)));
+        }
+        // Example: If the node stores blocks [100 .. 109] then BLOCK_HEIGHT_OFFSET
+        // is 100 and the Block with BlockIndex 100 is at index 0
+        let offset = (range_from - range_from_offset) as usize;
+        GetBlocksRes(Ok(LEDGER.read().unwrap().blockchain.block_slice(
+            std::ops::Range {
+                start: offset as u64,
+                end: offset as u64 + length as u64,
+            },
+        )))
     });
 }
 
@@ -1475,7 +1507,7 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder<Vec<u8>>) -> std::i
     )?;
     w.encode_gauge(
         "ledger_blocks",
-        ledger.blockchain.blocks.len() as f64,
+        ledger.blockchain.num_unarchived_blocks() as f64,
         "Total number of blocks stored in the main memory.",
     )?;
     // This value can go down -- the number is increased before archiving, and if
@@ -1489,7 +1521,7 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder<Vec<u8>>) -> std::i
     // order to be able to accurately calculate the total block rate.
     w.encode_gauge(
         "ledger_total_blocks",
-        ledger.blockchain.num_archived_blocks.saturating_add(ledger.blockchain.blocks.len() as u64) as f64,
+        ledger.blockchain.num_archived_blocks.saturating_add(ledger.blockchain.num_unarchived_blocks()) as f64,
         "Total number of blocks stored in the main memory, plus total number of blocks sent to the archive.",
     )?;
     if is_ready() {
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 68ff4b7c480..6df4ab7128e 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -21,7 +21,7 @@ pub use ic_ledger_canister_core::archive::ArchiveOptions;
 use ic_ledger_canister_core::runtime::Runtime;
 use ic_ledger_canister_core::{
     archive::ArchiveCanisterWasm,
-    blockchain::Blockchain,
+    blockchain::{Blockchain, HeapBlockData},
     ledger::{
         apply_transaction_no_trimming, block_locations, LedgerContext, LedgerData, TransactionInfo,
     },
@@ -562,7 +562,7 @@ pub struct Ledger {
     approvals: LedgerAllowances<Tokens>,
     #[serde(default)]
     stable_approvals: AllowanceTable<StableAllowancesData>,
-    blockchain: Blockchain<CdkRuntime, Icrc1ArchiveWasm>,
+    blockchain: Blockchain<CdkRuntime, Icrc1ArchiveWasm, HeapBlockData>,
 
     minting_account: Account,
     fee_collector: Option<FeeCollector<Account>>,
@@ -773,6 +773,7 @@ impl LedgerData for Ledger {
     type ArchiveWasm = Icrc1ArchiveWasm;
     type Transaction = Transaction<Tokens>;
     type Block = Block<Tokens>;
+    type BlockData = HeapBlockData;
 
     fn transaction_window(&self) -> Duration {
         TRANSACTION_WINDOW
@@ -802,11 +803,13 @@ impl LedgerData for Ledger {
         &self.token_symbol
     }
 
-    fn blockchain(&self) -> &Blockchain<Self::Runtime, Self::ArchiveWasm> {
+    fn blockchain(&self) -> &Blockchain<Self::Runtime, Self::ArchiveWasm, Self::BlockData> {
         &self.blockchain
     }
 
-    fn blockchain_mut(&mut self) -> &mut Blockchain<Self::Runtime, Self::ArchiveWasm> {
+    fn blockchain_mut(
+        &mut self,
+    ) -> &mut Blockchain<Self::Runtime, Self::ArchiveWasm, Self::BlockData> {
         &mut self.blockchain
     }
 
diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs
index 154819b4249..457623103fa 100644
--- a/rs/ledger_suite/icrc1/ledger/src/main.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/main.rs
@@ -390,7 +390,7 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder<Vec<u8>>) -> std::i
         )?;
         w.encode_gauge(
             "ledger_transactions",
-            ledger.blockchain().blocks.len() as f64,
+            ledger.blockchain().num_unarchived_blocks() as f64,
             "Total number of transactions stored in the main memory.",
         )?;
         w.encode_gauge(
@@ -402,7 +402,7 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder<Vec<u8>>) -> std::i
         // in order to be able to accurately calculate the total transaction rate.
         w.encode_gauge(
             "ledger_total_transactions",
-            ledger.blockchain().num_archived_blocks.saturating_add(ledger.blockchain().blocks.len() as u64) as f64,
+            ledger.blockchain().num_archived_blocks.saturating_add(ledger.blockchain().num_unarchived_blocks()) as f64,
             "Total number of transactions stored in the main memory, plus total number of transactions sent to the archive.",
         )?;
         if is_ready() {

From 4b70f04850bc5f0819420811a2d07e497b99ecab Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 31 Jan 2025 10:44:41 +0000
Subject: [PATCH 03/38] impl stable block data

---
 .../ledger_canister_core/src/blockchain.rs    | 23 ++++---
 .../ledger_canister_core/src/range_utils.rs   |  7 +-
 rs/ledger_suite/icrc1/ledger/src/lib.rs       | 66 ++++++++++++++++++-
 3 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 3779df88425..02da36afcaf 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -12,8 +12,8 @@ use ic_ledger_hash_of::HashOf;
 use std::ops::Range;
 
 pub trait BlockData {
-    fn add_block(&mut self, index: u64, block: EncodedBlock) -> Result<(), String>;
-    fn get_blocks(&self, range: Range<usize>) -> Vec<EncodedBlock>;
+    fn add_block(&mut self, block: EncodedBlock);
+    fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock>;
     fn get_block(&self, index: u64) -> Option<EncodedBlock>;
     fn remove_blocks(&mut self, num_blocks: u64);
     fn len(&self) -> u64;
@@ -28,12 +28,15 @@ pub struct HeapBlockData {
 }
 
 impl BlockData for HeapBlockData {
-    fn add_block(&mut self, _index: u64, block: EncodedBlock) -> Result<(), String> {
+    fn add_block(&mut self, block: EncodedBlock) {
         self.blocks.push(block);
-        Ok(())
     }
 
-    fn get_blocks(&self, range: Range<usize>) -> Vec<EncodedBlock> {
+    fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock> {
+        let range = Range {
+            start: range.start as usize,
+            end: range.end as usize,
+        };
         self.blocks[range].to_vec()
     }
 
@@ -127,7 +130,7 @@ where
         self.last_timestamp = block.timestamp();
         let encoded_block = block.encode();
         self.last_hash = Some(B::block_hash(&encoded_block));
-        self.blocks.add_block(self.chain_length(), encoded_block)?;
+        self.blocks.add_block(encoded_block);
         Ok(self.chain_length().checked_sub(1).unwrap())
     }
 
@@ -154,7 +157,7 @@ where
 
     /// The range of block indices that are not archived yet.
     pub fn local_block_range(&self) -> std::ops::Range<u64> {
-        self.num_archived_blocks..self.num_archived_blocks + self.blocks.len() as u64
+        self.num_archived_blocks..self.num_archived_blocks + self.blocks.len()
     }
 
     /// Returns the slice of blocks stored locally.
@@ -204,16 +207,16 @@ where
         // archiving will trigger when there are 2000 blocks in the ledger and
         // the 1000 oldest bocks will be archived, leaving the remaining 1000
         // blocks in place.
-        let num_blocks_before = self.num_unarchived_blocks() as usize;
+        let num_blocks_before = self.num_unarchived_blocks();
 
-        if num_blocks_before < trigger_threshold {
+        if num_blocks_before < trigger_threshold as u64 {
             return VecDeque::new();
         }
 
         let blocks_to_archive: VecDeque<EncodedBlock> =
             VecDeque::from(self.blocks.get_blocks(Range {
                 start: 0,
-                end: num_blocks_to_archive.min(num_blocks_before),
+                end: (num_blocks_to_archive as u64).min(num_blocks_before),
             }));
 
         println!(
diff --git a/rs/ledger_suite/common/ledger_canister_core/src/range_utils.rs b/rs/ledger_suite/common/ledger_canister_core/src/range_utils.rs
index da6ebec3cff..f8935971ee7 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/range_utils.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/range_utils.rs
@@ -69,12 +69,9 @@ pub fn as_indices(r: &Range<u64>) -> Range<usize> {
 }
 
 /// Converts the specified range into a range of indices relative to the specified offset.
-pub fn offset(r: &Range<u64>, offset: u64) -> Range<usize> {
+pub fn offset(r: &Range<u64>, offset: u64) -> Range<u64> {
     debug_assert!(offset <= r.start);
     let start = r.start.saturating_sub(offset);
     let end = start + range_len(r);
-    Range {
-        start: start as usize,
-        end: end as usize,
-    }
+    Range { start, end }
 }
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index ac504bb7651..c07818dc163 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -16,9 +16,9 @@ use ic_certification::{
 };
 use ic_icrc1::blocks::encoded_block_to_generic_block;
 use ic_icrc1::{Block, LedgerAllowances, LedgerBalances, Transaction};
-use ic_ledger_canister_core::archive::Archive;
 pub use ic_ledger_canister_core::archive::ArchiveOptions;
 use ic_ledger_canister_core::runtime::Runtime;
+use ic_ledger_canister_core::{archive::Archive, blockchain::BlockData};
 use ic_ledger_canister_core::{
     archive::ArchiveCanisterWasm,
     blockchain::{Blockchain, HeapBlockData},
@@ -56,7 +56,7 @@ use serde_bytes::ByteBuf;
 use std::borrow::Cow;
 use std::cell::RefCell;
 use std::collections::{BTreeMap, VecDeque};
-use std::ops::DerefMut;
+use std::ops::{DerefMut, Range};
 use std::time::Duration;
 
 const TRANSACTION_WINDOW: Duration = Duration::from_secs(24 * 60 * 60);
@@ -1294,3 +1294,65 @@ impl BalancesStore for StableBalances {
         }
     }
 }
+
+#[derive(Debug, Deserialize, Serialize, Default)]
+#[serde(transparent)]
+pub struct StableBlockData {
+    blocks: Vec<EncodedBlock>,
+}
+
+impl BlockData for StableBlockData {
+    fn add_block(&mut self, block: EncodedBlock) {
+        BLOCKS_MEMORY.with_borrow_mut(|blocks| {
+            let index = blocks.last_key_value().unwrap_or((0, vec![])).0;
+            let _ = blocks.insert(index, block.into_vec());
+        });
+    }
+
+    fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock> {
+        BLOCKS_MEMORY.with_borrow(|blocks| {
+            let mut result = vec![];
+            let first_index = blocks.first_key_value().unwrap_or((0, vec![])).0;
+            let range_offset = range_utils::offset(&range, first_index);
+            for block in blocks.range(range_offset) {
+                result.push(EncodedBlock::from_vec(block.1));
+            }
+            result
+        })
+    }
+
+    fn get_block(&self, index: u64) -> Option<EncodedBlock> {
+        BLOCKS_MEMORY.with_borrow(|blocks| {
+            let first_index = blocks.first_key_value().unwrap_or((0, vec![])).0;
+            blocks
+                .get(&index.saturating_sub(first_index))
+                .map(EncodedBlock::from_vec)
+        })
+    }
+
+    fn remove_blocks(&mut self, num_blocks: u64) {
+        BLOCKS_MEMORY.with_borrow_mut(|blocks| {
+            let mut removed = 0;
+            while !blocks.is_empty() && removed < num_blocks {
+                blocks.pop_first();
+                removed += 1;
+            }
+        });
+    }
+
+    fn len(&self) -> u64 {
+        BLOCKS_MEMORY.with_borrow(|blocks| blocks.len())
+    }
+
+    fn is_empty(&self) -> bool {
+        BLOCKS_MEMORY.with_borrow(|blocks| blocks.is_empty())
+    }
+
+    fn last(&self) -> Option<EncodedBlock> {
+        BLOCKS_MEMORY.with_borrow(|blocks| {
+            blocks
+                .last_key_value()
+                .map(|kv| EncodedBlock::from_vec(kv.1))
+        })
+    }
+}

From 15c6c27acd35a0f9fc57d5bb999cb2092ca6e1c0 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 31 Jan 2025 14:06:12 +0000
Subject: [PATCH 04/38] some tests

---
 .../ledger_canister_core/src/blockchain.rs    |  9 ++++
 rs/ledger_suite/icrc1/ledger/src/lib.rs       | 46 +++++++++++++++----
 rs/ledger_suite/icrc1/ledger/src/main.rs      | 13 +++++-
 rs/ledger_suite/icrc1/ledger/tests/tests.rs   | 27 ++++++++++-
 4 files changed, 82 insertions(+), 13 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 02da36afcaf..3b687d53562 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -19,6 +19,7 @@ pub trait BlockData {
     fn len(&self) -> u64;
     fn is_empty(&self) -> bool;
     fn last(&self) -> Option<EncodedBlock>;
+    fn migrate_one_block(&mut self, num_archived_blocks: u64) -> bool;
 }
 
 #[derive(Debug, Deserialize, Serialize, Default)]
@@ -59,6 +60,10 @@ impl BlockData for HeapBlockData {
     fn last(&self) -> Option<EncodedBlock> {
         self.blocks.last().cloned()
     }
+
+    fn migrate_one_block(&mut self, _num_archived_blocks: u64) -> bool {
+        panic!("HeapBlockData cannot perform migration!");
+    }
 }
 
 /// Stores a chain of transactions with their metadata
@@ -229,4 +234,8 @@ where
 
         blocks_to_archive
     }
+
+    pub fn migrate_one_block(&mut self) -> bool {
+        self.blocks.migrate_one_block(self.num_archived_blocks)
+    }
 }
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index c07818dc163..7ed075d619b 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -21,7 +21,7 @@ use ic_ledger_canister_core::runtime::Runtime;
 use ic_ledger_canister_core::{archive::Archive, blockchain::BlockData};
 use ic_ledger_canister_core::{
     archive::ArchiveCanisterWasm,
-    blockchain::{Blockchain, HeapBlockData},
+    blockchain::Blockchain,
     ledger::{apply_transaction, block_locations, LedgerContext, LedgerData, TransactionInfo},
     range_utils,
 };
@@ -82,14 +82,15 @@ pub type Tokens = ic_icrc1_tokens_u256::U256;
 ///   * 0 - the whole ledger state is stored on the heap.
 ///   * 1 - the allowances are stored in stable structures.
 ///   * 2 - the balances are stored in stable structures.
+///   * 3 - the blocks are stored in stable structures.
 // TODO: When moving to version 3 consider adding `#[serde(default, skip_serializing)]`
 // to `balances` and `approvals` fields of the `Ledger` struct.
 // Since `balances` don't use a default, this can only be done with an incompatible change.
 #[cfg(not(feature = "next-ledger-version"))]
-pub const LEDGER_VERSION: u64 = 2;
+pub const LEDGER_VERSION: u64 = 3;
 
 #[cfg(feature = "next-ledger-version")]
-pub const LEDGER_VERSION: u64 = 3;
+pub const LEDGER_VERSION: u64 = 4;
 
 #[derive(Clone, Debug)]
 pub struct Icrc1ArchiveWasm;
@@ -534,6 +535,7 @@ pub enum LedgerField {
     Allowances,
     AllowancesExpirations,
     Balances,
+    Blocks,
 }
 
 #[derive(Copy, Clone, Serialize, Deserialize, Debug)]
@@ -560,7 +562,7 @@ pub struct Ledger {
     approvals: LedgerAllowances<Tokens>,
     #[serde(default)]
     stable_approvals: AllowanceTable<StableAllowancesData>,
-    blockchain: Blockchain<CdkRuntime, Icrc1ArchiveWasm, HeapBlockData>,
+    blockchain: Blockchain<CdkRuntime, Icrc1ArchiveWasm, StableBlockData>,
 
     minting_account: Account,
     fee_collector: Option<FeeCollector<Account>>,
@@ -724,6 +726,10 @@ impl Ledger {
         }
     }
 
+    pub fn migrate_one_block(&mut self) -> bool {
+        self.blockchain.migrate_one_block()
+    }
+
     pub fn clear_arrivals(&mut self) {
         self.approvals.allowances_data.clear_arrivals();
     }
@@ -769,7 +775,7 @@ impl LedgerData for Ledger {
     type ArchiveWasm = Icrc1ArchiveWasm;
     type Transaction = Transaction<Tokens>;
     type Block = Block<Tokens>;
-    type BlockData = HeapBlockData;
+    type BlockData = StableBlockData;
 
     fn transaction_window(&self) -> Duration {
         TRANSACTION_WINDOW
@@ -1304,8 +1310,11 @@ pub struct StableBlockData {
 impl BlockData for StableBlockData {
     fn add_block(&mut self, block: EncodedBlock) {
         BLOCKS_MEMORY.with_borrow_mut(|blocks| {
-            let index = blocks.last_key_value().unwrap_or((0, vec![])).0;
-            let _ = blocks.insert(index, block.into_vec());
+            let index = match blocks.last_key_value() {
+                None => 0u64,
+                Some(kv) => kv.0 + 1,
+            };
+            assert!(blocks.insert(index, block.into_vec()).is_none());
         });
     }
 
@@ -1313,8 +1322,9 @@ impl BlockData for StableBlockData {
         BLOCKS_MEMORY.with_borrow(|blocks| {
             let mut result = vec![];
             let first_index = blocks.first_key_value().unwrap_or((0, vec![])).0;
-            let range_offset = range_utils::offset(&range, first_index);
-            for block in blocks.range(range_offset) {
+            let start = range.start + first_index;
+            let end = range.end + first_index;
+            for block in blocks.range(start..end) {
                 result.push(EncodedBlock::from_vec(block.1));
             }
             result
@@ -1325,7 +1335,7 @@ impl BlockData for StableBlockData {
         BLOCKS_MEMORY.with_borrow(|blocks| {
             let first_index = blocks.first_key_value().unwrap_or((0, vec![])).0;
             blocks
-                .get(&index.saturating_sub(first_index))
+                .get(&(index + first_index))
                 .map(EncodedBlock::from_vec)
         })
     }
@@ -1355,4 +1365,20 @@ impl BlockData for StableBlockData {
                 .map(|kv| EncodedBlock::from_vec(kv.1))
         })
     }
+
+    fn migrate_one_block(&mut self, num_archived_blocks: u64) -> bool {
+        let migrated_len = self.len();
+        if migrated_len < self.blocks.len() as u64 {
+            BLOCKS_MEMORY.with_borrow_mut(|blocks| {
+                blocks.insert(
+                    num_archived_blocks + migrated_len,
+                    self.blocks[migrated_len as usize].clone().into_vec(),
+                )
+            });
+            true
+        } else {
+            self.blocks.clear();
+            false
+        }
+    }
 }
diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs
index 94624a0f59c..0410de715df 100644
--- a/rs/ledger_suite/icrc1/ledger/src/main.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/main.rs
@@ -238,6 +238,9 @@ fn post_upgrade(args: Option<LedgerArgument>) {
 
     PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed);
 
+    if upgrade_from_version < 3 {
+        set_ledger_state(LedgerState::Migrating(LedgerField::Blocks));
+    }
     if upgrade_from_version < 2 {
         set_ledger_state(LedgerState::Migrating(LedgerField::Balances));
         log_message(format!("Upgrading from version {upgrade_from_version} which does not store balances in stable structures, clearing stable balances data.").as_str());
@@ -272,6 +275,7 @@ fn migrate_next_part(instruction_limit: u64) {
     let mut migrated_allowances = 0;
     let mut migrated_expirations = 0;
     let mut migrated_balances = 0;
+    let mut migrated_blocks = 0;
 
     log_message("Migrating part of the ledger state.");
 
@@ -301,6 +305,13 @@ fn migrate_next_part(instruction_limit: u64) {
                 LedgerField::Balances => {
                     if ledger.migrate_one_balance() {
                         migrated_balances += 1;
+                    } else {
+                        set_ledger_state(LedgerState::Migrating(LedgerField::Blocks));
+                    }
+                }
+                LedgerField::Blocks => {
+                    if ledger.migrate_one_block() {
+                        migrated_blocks += 1;
                     } else {
                         set_ledger_state(LedgerState::Ready);
                     }
@@ -308,7 +319,7 @@ fn migrate_next_part(instruction_limit: u64) {
             }
         }
         let instructions_migration = instruction_counter() - instructions_migration_start;
-        let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations} balances: {migrated_balances}. Migration step instructions: {instructions_migration}, total instructions used in message: {}." ,
+        let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations} balances: {migrated_balances} blocks: {migrated_blocks}. Migration step instructions: {instructions_migration}, total instructions used in message: {}." ,
             instruction_counter());
         if !is_ready() {
             log_message(
diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index c2571a84ccb..449e1f3f6a4 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -248,6 +248,20 @@ fn encode_init_args_with_small_sized_archive(
     }
 }
 
+fn encode_init_args_with_large_archive_trigger_threshold(
+    args: ic_ledger_suite_state_machine_tests::InitArgs,
+) -> LedgerArgument {
+    match encode_init_args(args) {
+        LedgerArgument::Init(mut init_args) => {
+            init_args.archive_options.trigger_threshold = 1000;
+            LedgerArgument::Init(init_args)
+        }
+        LedgerArgument::Upgrade(_) => {
+            panic!("BUG: Expected Init argument")
+        }
+    }
+}
+
 fn encode_upgrade_args() -> LedgerArgument {
     LedgerArgument::Upgrade(None)
 }
@@ -510,7 +524,7 @@ fn test_block_transformation() {
 
 #[test]
 fn icrc1_test_upgrade_serialization_from_mainnet() {
-    icrc1_test_upgrade_serialization(ledger_mainnet_wasm(), false);
+    icrc1_test_upgrade_serialization(ledger_mainnet_wasm(), true);
 }
 
 #[test]
@@ -541,6 +555,15 @@ fn icrc1_test_upgrade_serialization(ledger_mainnet_wasm: Vec<u8>, mainnet_on_pre
     );
 }
 
+#[test]
+fn icrc1_test_multi_step_migration_from_mainnet() {
+    ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration(
+        ledger_mainnet_wasm(),
+        ledger_wasm_lowupgradeinstructionlimits(),
+        encode_init_args_with_large_archive_trigger_threshold,
+    );
+}
+
 #[test]
 fn icrc1_test_multi_step_migration_from_v3() {
     ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration(
@@ -575,7 +598,7 @@ fn icrc1_test_downgrade_from_incompatible_version() {
         ledger_wasm_nextledgerversion(),
         ledger_wasm(),
         encode_init_args,
-        true,
+        false,
     );
 }
 

From 5fbdb573911eda1853cc645a329cf5a0f00095f1 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 3 Feb 2025 10:51:41 +0000
Subject: [PATCH 05/38] readd index in add_block

---
 .../ledger_canister_core/src/blockchain.rs    |  6 +++---
 rs/ledger_suite/icrc1/ledger/src/lib.rs       | 20 +++++++------------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 3b687d53562..05dd7563e39 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -12,7 +12,7 @@ use ic_ledger_hash_of::HashOf;
 use std::ops::Range;
 
 pub trait BlockData {
-    fn add_block(&mut self, block: EncodedBlock);
+    fn add_block(&mut self, index: u64, block: EncodedBlock);
     fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock>;
     fn get_block(&self, index: u64) -> Option<EncodedBlock>;
     fn remove_blocks(&mut self, num_blocks: u64);
@@ -29,7 +29,7 @@ pub struct HeapBlockData {
 }
 
 impl BlockData for HeapBlockData {
-    fn add_block(&mut self, block: EncodedBlock) {
+    fn add_block(&mut self, _index: u64, block: EncodedBlock) {
         self.blocks.push(block);
     }
 
@@ -135,7 +135,7 @@ where
         self.last_timestamp = block.timestamp();
         let encoded_block = block.encode();
         self.last_hash = Some(B::block_hash(&encoded_block));
-        self.blocks.add_block(encoded_block);
+        self.blocks.add_block(self.chain_length(), encoded_block);
         Ok(self.chain_length().checked_sub(1).unwrap())
     }
 
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 7ed075d619b..96ec98af313 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -1308,12 +1308,8 @@ pub struct StableBlockData {
 }
 
 impl BlockData for StableBlockData {
-    fn add_block(&mut self, block: EncodedBlock) {
+    fn add_block(&mut self, index: u64, block: EncodedBlock) {
         BLOCKS_MEMORY.with_borrow_mut(|blocks| {
-            let index = match blocks.last_key_value() {
-                None => 0u64,
-                Some(kv) => kv.0 + 1,
-            };
             assert!(blocks.insert(index, block.into_vec()).is_none());
         });
     }
@@ -1367,14 +1363,12 @@ impl BlockData for StableBlockData {
     }
 
     fn migrate_one_block(&mut self, num_archived_blocks: u64) -> bool {
-        let migrated_len = self.len();
-        if migrated_len < self.blocks.len() as u64 {
-            BLOCKS_MEMORY.with_borrow_mut(|blocks| {
-                blocks.insert(
-                    num_archived_blocks + migrated_len,
-                    self.blocks[migrated_len as usize].clone().into_vec(),
-                )
-            });
+        let num_migrated = self.len();
+        if num_migrated < self.blocks.len() as u64 {
+            self.add_block(
+                num_archived_blocks + num_migrated,
+                self.blocks[num_migrated as usize].clone(),
+            );
             true
         } else {
             self.blocks.clear();

From 023809703e8450a4198e0d89a86d0ceca597f038 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 3 Feb 2025 12:43:32 +0000
Subject: [PATCH 06/38] fix icp ledger tests

---
 .../ledger_canister_core/src/blockchain.rs    |  2 +-
 rs/ledger_suite/icp/ledger/src/main.rs        | 28 ++---------
 rs/ledger_suite/icp/ledger/src/tests.rs       |  8 +--
 rs/ledger_suite/icp/src/lib.rs                | 50 +++++++++++++++----
 4 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 05dd7563e39..fe4eaf4c877 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -74,7 +74,7 @@ where
     BD: BlockData + Serialize + Default,
     for<'a> BD: Deserialize<'a>,
 {
-    blocks: BD,
+    pub blocks: BD,
     pub last_hash: Option<HashOf<EncodedBlock>>,
 
     /// The timestamp of the most recent block. Must be monotonically
diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index 3e5a79551d0..cc4a224ea8e 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -35,8 +35,8 @@ use icp_ledger::{
     max_blocks_per_request, protobuf, tokens_into_proto, AccountBalanceArgs, AccountIdBlob,
     AccountIdentifier, AccountIdentifierByteBuf, ArchiveInfo, ArchivedBlocksRange,
     ArchivedEncodedBlocksRange, Archives, BinaryAccountBalanceArgs, Block, BlockArg, BlockRes,
-    CandidBlock, Decimals, FeatureFlags, GetBlocksArgs, GetBlocksRes, InitArgs, IterBlocksArgs,
-    IterBlocksRes, LedgerCanisterPayload, Memo, Name, Operation, PaymentError, QueryBlocksResponse,
+    CandidBlock, Decimals, FeatureFlags, GetBlocksArgs, InitArgs, IterBlocksArgs, IterBlocksRes,
+    LedgerCanisterPayload, Memo, Name, Operation, PaymentError, QueryBlocksResponse,
     QueryEncodedBlocksResponse, SendArgs, Subaccount, Symbol, TipOfChainRes, TotalSupplyArgs,
     Transaction, TransferArgs, TransferError, TransferFee, TransferFeeArgs, MEMO_SIZE_BYTES,
 };
@@ -1345,29 +1345,7 @@ fn get_blocks_() {
         let length = std::cmp::min(length, max_blocks_per_request(&caller()) as u64);
         let blockchain = &LEDGER.read().unwrap().blockchain;
         let start_offset = blockchain.num_archived_blocks();
-        let blocks_len = LEDGER.read().unwrap().blockchain.num_unarchived_blocks() as usize;
-        let range_from_offset = start;
-        let range_from = start_offset;
-        // Inclusive end of the range of *requested* blocks
-        let requested_range_to = range_from as usize + length as usize - 1;
-        // Inclusive end of the range of *available* blocks
-        let range_to = range_from_offset as usize + blocks_len - 1;
-        // Example: If the Node stores 10 blocks beginning at BlockIndex 100, i.e.
-        // [100 .. 109] then requesting blocks at BlockIndex < 100 or BlockIndex
-        // > 109 is an error
-        if range_from < range_from_offset || requested_range_to > range_to {
-            return GetBlocksRes(Err(format!("Requested blocks outside the range stored in the archive node. Requested [{} .. {}]. Available [{} .. {}].",
-            range_from, requested_range_to, range_from_offset, range_to)));
-        }
-        // Example: If the node stores blocks [100 .. 109] then BLOCK_HEIGHT_OFFSET
-        // is 100 and the Block with BlockIndex 100 is at index 0
-        let offset = (range_from - range_from_offset) as usize;
-        GetBlocksRes(Ok(LEDGER.read().unwrap().blockchain.block_slice(
-            std::ops::Range {
-                start: offset as u64,
-                end: offset as u64 + length as u64,
-            },
-        )))
+        icp_ledger::get_blocks_ledger(&blockchain.blocks, start_offset, start, length as usize)
     });
 }
 
diff --git a/rs/ledger_suite/icp/ledger/src/tests.rs b/rs/ledger_suite/icp/ledger/src/tests.rs
index a3bf66c0566..336df0a21dc 100644
--- a/rs/ledger_suite/icp/ledger/src/tests.rs
+++ b/rs/ledger_suite/icp/ledger/src/tests.rs
@@ -259,8 +259,8 @@ fn serialize() {
         state_decoded.blockchain.last_hash
     );
     assert_eq!(
-        state.blockchain.blocks.len(),
-        state_decoded.blockchain.blocks.len()
+        state.blockchain.num_unarchived_blocks(),
+        state_decoded.blockchain.num_unarchived_blocks()
     );
     assert_eq!(state.balances.store, state_decoded.balances.store);
 }
@@ -562,13 +562,13 @@ fn get_blocks_returns_correct_blocks() {
 
     let blocks = &state.blockchain.blocks;
 
-    let first_blocks = icp_ledger::get_blocks(blocks, 0, 1, 5).0.unwrap();
+    let first_blocks = icp_ledger::get_blocks_ledger(blocks, 0, 1, 5).0.unwrap();
     for i in 0..first_blocks.len() {
         let block = Block::decode(first_blocks.get(i).unwrap().clone()).unwrap();
         assert_eq!(block.transaction.memo.0, i as u64);
     }
 
-    let last_blocks = icp_ledger::get_blocks(blocks, 0, 6, 5).0.unwrap();
+    let last_blocks = icp_ledger::get_blocks_ledger(blocks, 0, 6, 5).0.unwrap();
     for i in 0..last_blocks.len() {
         let block = Block::decode(last_blocks.get(i).unwrap().clone()).unwrap();
         assert_eq!(block.transaction.memo.0, 5 + i as u64);
diff --git a/rs/ledger_suite/icp/src/lib.rs b/rs/ledger_suite/icp/src/lib.rs
index 90c381de74f..9cca7245368 100644
--- a/rs/ledger_suite/icp/src/lib.rs
+++ b/rs/ledger_suite/icp/src/lib.rs
@@ -3,6 +3,7 @@ use dfn_protobuf::ProtoBuf;
 use ic_base_types::{CanisterId, PrincipalId};
 use ic_crypto_sha2::Sha256;
 pub use ic_ledger_canister_core::archive::ArchiveOptions;
+use ic_ledger_canister_core::blockchain::BlockData;
 use ic_ledger_canister_core::ledger::{LedgerContext, LedgerTransaction, TxApplyError};
 use ic_ledger_core::{
     approvals::{AllowanceTable, HeapAllowancesData},
@@ -19,6 +20,7 @@ use serde_bytes::ByteBuf;
 use std::collections::{HashMap, HashSet};
 use std::convert::TryFrom;
 use std::fmt;
+use std::ops::Range;
 use std::time::Duration;
 use std::{borrow::Cow, collections::BTreeMap};
 use strum_macros::IntoStaticStr;
@@ -1132,32 +1134,60 @@ pub struct IterBlocksRes(pub Vec<EncodedBlock>);
 pub struct BlockArg(pub BlockIndex);
 pub struct BlockRes(pub Option<Result<EncodedBlock, CanisterId>>);
 
-// A helper function for ledger/get_blocks and archive_node/get_blocks endpoints
-pub fn get_blocks(
-    blocks: &[EncodedBlock],
+fn get_block_indices(
+    blocks_len: usize,
     range_from_offset: BlockIndex,
     range_from: BlockIndex,
     length: usize,
-) -> GetBlocksRes {
+) -> Result<Range<usize>, String> {
     // Inclusive end of the range of *requested* blocks
     let requested_range_to = range_from as usize + length - 1;
     // Inclusive end of the range of *available* blocks
-    let range_to = range_from_offset as usize + blocks.len() - 1;
+    let range_to = range_from_offset as usize + blocks_len - 1;
     // Example: If the Node stores 10 blocks beginning at BlockIndex 100, i.e.
     // [100 .. 109] then requesting blocks at BlockIndex < 100 or BlockIndex
     // > 109 is an error
     if range_from < range_from_offset || requested_range_to > range_to {
-        return GetBlocksRes(Err(format!("Requested blocks outside the range stored in the archive node. Requested [{} .. {}]. Available [{} .. {}].",
-            range_from, requested_range_to, range_from_offset, range_to)));
+        return Err(format!("Requested blocks outside the range stored in the archive node. Requested [{} .. {}]. Available [{} .. {}].",
+            range_from, requested_range_to, range_from_offset, range_to));
     }
     // Example: If the node stores blocks [100 .. 109] then BLOCK_HEIGHT_OFFSET
     // is 100 and the Block with BlockIndex 100 is at index 0
     let offset = (range_from - range_from_offset) as usize;
-    GetBlocksRes(Ok(blocks[offset..offset + length].to_vec()))
+    Ok(offset..offset + length)
+}
+
+// A helper function for archive_node/get_blocks endpoint
+pub fn get_blocks(
+    blocks: &[EncodedBlock],
+    range_from_offset: BlockIndex,
+    range_from: BlockIndex,
+    length: usize,
+) -> GetBlocksRes {
+    match get_block_indices(blocks.len(), range_from_offset, range_from, length) {
+        Ok(range) => GetBlocksRes(Ok(blocks[range].to_vec())),
+        Err(e) => GetBlocksRes(Err(e)),
+    }
+}
+
+// A helper function for ledger/get_blocks endpoint
+pub fn get_blocks_ledger<BD: BlockData>(
+    blocks: &BD,
+    range_from_offset: BlockIndex,
+    range_from: BlockIndex,
+    length: usize,
+) -> GetBlocksRes {
+    match get_block_indices(blocks.len() as usize, range_from_offset, range_from, length) {
+        Ok(range) => {
+            let start = range.start as u64;
+            let end = range.end as u64;
+            GetBlocksRes(Ok(blocks.get_blocks(start..end)))
+        }
+        Err(e) => GetBlocksRes(Err(e)),
+    }
 }
 
-// A helper function for ledger/iter_blocks and archive_node/iter_blocks
-// endpoints
+// A helper function for archive_node/iter_blocks endpoint
 pub fn iter_blocks(blocks: &[EncodedBlock], offset: usize, length: usize) -> IterBlocksRes {
     let start = std::cmp::min(offset, blocks.len());
     let end = std::cmp::min(start + length, blocks.len());

From cf07916379fb794968fa6ee2582f6977115f847e Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 3 Feb 2025 14:26:44 +0000
Subject: [PATCH 07/38] fix icp iter blocks

---
 rs/ledger_suite/icp/ledger/src/main.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index cc4a224ea8e..489736fd5d6 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -16,6 +16,7 @@ use ic_ledger_canister_core::ledger::LedgerContext;
 use ic_ledger_canister_core::runtime::heap_memory_size_bytes;
 use ic_ledger_canister_core::{
     archive::{Archive, ArchiveOptions},
+    blockchain::BlockData,
     ledger::{
         apply_transaction, archive_blocks, block_locations, find_block_in_archive, LedgerAccess,
         TransferError as CoreTransferError,
@@ -1329,7 +1330,8 @@ fn iter_blocks_() {
             .read()
             .unwrap()
             .blockchain
-            .block_slice(std::ops::Range {
+            .blocks
+            .get_blocks(std::ops::Range {
                 start: start as u64,
                 end: end as u64,
             });

From e1d9e7d03edffc11dc1d241c3827d24284eb8dd8 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 3 Feb 2025 14:30:04 +0000
Subject: [PATCH 08/38] increase 256 ledger wasm size limit

---
 publish/canisters/BUILD.bazel | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/publish/canisters/BUILD.bazel b/publish/canisters/BUILD.bazel
index e353b17af02..4099ebe4800 100644
--- a/publish/canisters/BUILD.bazel
+++ b/publish/canisters/BUILD.bazel
@@ -62,7 +62,8 @@ CANISTERS_MAX_SIZE_COMPRESSED_E5_BYTES = {
     # we are setting the check to 7 to leave some space for growth
     # but enough to get an alert in case of a spike in size.
     "ic-icrc1-ledger.wasm.gz": "7",
-    "ic-icrc1-ledger-u256.wasm.gz": "7",
+    # The size is currently at 704585 bytes.
+    "ic-icrc1-ledger-u256.wasm.gz": "8",
     # Size when constraint addded: 841_234 bytes
     "ledger-canister.wasm.gz": "9",
     # Size when constraint addded: 906_940 bytes

From e9b66a34fbcdfd769c24a7948ecbc3bdc85f098d Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Tue, 4 Feb 2025 13:02:17 +0000
Subject: [PATCH 09/38] verify blocks

---
 rs/ledger_suite/icp/ledger/tests/tests.rs   | 66 ++++++++++++++++++++-
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 19 ++++--
 rs/ledger_suite/tests/sm-tests/src/lib.rs   |  7 ++-
 3 files changed, 85 insertions(+), 7 deletions(-)

diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs
index 5880d38248d..614015dab6a 100644
--- a/rs/ledger_suite/icp/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icp/ledger/tests/tests.rs
@@ -5,7 +5,7 @@ use dfn_protobuf::ProtoBuf;
 use ic_agent::identity::Identity;
 use ic_base_types::{CanisterId, PrincipalId};
 use ic_icrc1_test_utils::minter_identity;
-use ic_ledger_core::block::BlockIndex;
+use ic_ledger_core::block::{BlockIndex, EncodedBlock};
 use ic_ledger_core::{block::BlockType, Tokens};
 use ic_ledger_suite_state_machine_tests::{
     balance_of, default_approve_args, default_transfer_from_args, expect_icrc2_disabled,
@@ -1266,12 +1266,75 @@ fn test_upgrade_serialization(ledger_wasm_mainnet: Vec<u8>) {
     );
 }
 
+fn get_all_blocks(state_machine: &StateMachine, ledger_id: CanisterId) -> Vec<EncodedBlock> {
+    let p1 = PrincipalId::new_user_test_id(1);
+    let blocks_res = query_blocks(&state_machine, p1.0, ledger_id, 0, u32::MAX.into());
+    let mut result = vec![];
+    println!(
+        "chain length: {}, local length: {}, num_archives: {}",
+        blocks_res.chain_length,
+        blocks_res.blocks.len(),
+        blocks_res.archived_blocks.len(),
+    );
+    for archived in blocks_res.archived_blocks {
+        println!(
+            "get blocks from archive, start: {}, length: {}",
+            archived.start, archived.length
+        );
+        let get_blocks_args = Encode!(&GetBlocksArgs {
+            start: archived.start,
+            length: archived.length,
+        })
+        .unwrap();
+        let archived_blocks = Decode!(
+            &state_machine
+                .query(
+                    CanisterId::unchecked_from_principal(archived.callback.canister_id.into()),
+                    "get_blocks",
+                    get_blocks_args.clone()
+                )
+                .unwrap()
+                .bytes(),
+            GetBlocksResult
+        )
+        .unwrap()
+        .unwrap();
+        result.extend(archived_blocks.blocks);
+    }
+
+    result.extend(blocks_res.blocks);
+    assert_eq!(result.len(), blocks_res.chain_length as usize);
+    let mut prev_hash = None;
+    let mut i = 0;
+
+    for block in &result {
+        println!("for loop iter {i}");
+        i += 1;
+        let block: Block = block.clone().try_into().unwrap();
+        println!("{:?}", block.parent_hash);
+        println!("{:?}", prev_hash);
+        println!("");
+        assert_eq!(block.parent_hash, prev_hash);
+        let encoded_block = block.encode();
+        prev_hash = Some(icp_ledger::Block::block_hash(&encoded_block));
+    }
+
+    result
+        .into_iter()
+        .map(|b| {
+            let block: Block = b.try_into().unwrap();
+            block.encode()
+        })
+        .collect()
+}
+
 #[test]
 fn test_multi_step_migration_from_mainnet() {
     ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration(
         ledger_wasm_mainnet(),
         ledger_wasm_low_instruction_limits(),
         encode_init_args,
+        get_all_blocks,
     );
 }
 
@@ -1281,6 +1344,7 @@ fn test_multi_step_migration_from_v2() {
         ledger_wasm_mainnet_v2(),
         ledger_wasm_low_instruction_limits(),
         encode_init_args,
+        get_all_blocks,
     );
 }
 
diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index 449e1f3f6a4..bf7967c0bd1 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -8,15 +8,15 @@ use ic_icrc1_ledger::{
 };
 use ic_icrc1_test_utils::minter_identity;
 use ic_ledger_canister_core::archive::ArchiveOptions;
-use ic_ledger_core::block::{BlockIndex, BlockType};
+use ic_ledger_core::block::{BlockIndex, BlockType, EncodedBlock};
 use ic_ledger_hash_of::{HashOf, HASH_LENGTH};
 use ic_ledger_suite_state_machine_tests::fee_collector::BlockRetrieval;
 use ic_ledger_suite_state_machine_tests::in_memory_ledger::verify_ledger_state;
 use ic_ledger_suite_state_machine_tests::{
-    send_approval, send_transfer_from, AllowanceProvider, ARCHIVE_TRIGGER_THRESHOLD, BLOB_META_KEY,
-    BLOB_META_VALUE, DECIMAL_PLACES, FEE, INT_META_KEY, INT_META_VALUE, MINTER, NAT_META_KEY,
-    NAT_META_VALUE, NUM_BLOCKS_TO_ARCHIVE, TEXT_META_KEY, TEXT_META_VALUE, TOKEN_NAME,
-    TOKEN_SYMBOL,
+    get_all_ledger_and_archive_blocks, send_approval, send_transfer_from, AllowanceProvider,
+    ARCHIVE_TRIGGER_THRESHOLD, BLOB_META_KEY, BLOB_META_VALUE, DECIMAL_PLACES, FEE, INT_META_KEY,
+    INT_META_VALUE, MINTER, NAT_META_KEY, NAT_META_VALUE, NUM_BLOCKS_TO_ARCHIVE, TEXT_META_KEY,
+    TEXT_META_VALUE, TOKEN_NAME, TOKEN_SYMBOL,
 };
 use ic_state_machine_tests::StateMachine;
 use icrc_ledger_types::icrc::generic_metadata_value::MetadataValue;
@@ -555,12 +555,18 @@ fn icrc1_test_upgrade_serialization(ledger_mainnet_wasm: Vec<u8>, mainnet_on_pre
     );
 }
 
+fn get_all_blocks(state_machine: &StateMachine, ledger_id: CanisterId) -> Vec<EncodedBlock> {
+    let blocks = get_all_ledger_and_archive_blocks::<Tokens>(state_machine, ledger_id, None, None);
+    blocks.into_iter().map(|b| b.encode()).collect()
+}
+
 #[test]
 fn icrc1_test_multi_step_migration_from_mainnet() {
     ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration(
         ledger_mainnet_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
         encode_init_args_with_large_archive_trigger_threshold,
+        get_all_blocks,
     );
 }
 
@@ -570,6 +576,7 @@ fn icrc1_test_multi_step_migration_from_v3() {
         ledger_mainnet_v3_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
         encode_init_args,
+        get_all_blocks,
     );
 }
 
@@ -579,6 +586,7 @@ fn icrc1_test_multi_step_migration_from_v2() {
         ledger_mainnet_v2_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
         encode_init_args,
+        get_all_blocks,
     );
 }
 
@@ -588,6 +596,7 @@ fn icrc1_test_multi_step_migration_from_v2_noledgerversion() {
         ledger_mainnet_v2_noledgerversion_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
         encode_init_args,
+        get_all_blocks,
     );
 }
 
diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs
index 5a84a1166d1..184ca9180f5 100644
--- a/rs/ledger_suite/tests/sm-tests/src/lib.rs
+++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs
@@ -12,7 +12,7 @@ use ic_icrc1::{endpoints::StandardRecord, hash::Hash, Block, Operation, Transact
 use ic_icrc1_ledger::FeatureFlags;
 use ic_icrc1_test_utils::{valid_transactions_strategy, ArgWithCaller, LedgerEndpointArg};
 use ic_ledger_canister_core::archive::ArchiveOptions;
-use ic_ledger_core::block::{BlockIndex, BlockType};
+use ic_ledger_core::block::{BlockIndex, BlockType, EncodedBlock};
 use ic_ledger_core::timestamp::TimeStamp;
 use ic_ledger_core::tokens::TokensType;
 use ic_ledger_core::Tokens;
@@ -2504,6 +2504,7 @@ pub fn icrc1_test_multi_step_migration<T>(
     ledger_wasm_mainnet: Vec<u8>,
     ledger_wasm_current_lowinstructionlimits: Vec<u8>,
     encode_init_args: fn(InitArgs) -> T,
+    get_all_blocks: fn(&StateMachine, CanisterId) -> Vec<EncodedBlock>,
 ) where
     T: CandidType,
 {
@@ -2580,6 +2581,8 @@ pub fn icrc1_test_multi_step_migration<T>(
     let test_upgrade = |ledger_wasm: Vec<u8>,
                         balances: BTreeMap<&Account, Nat>,
                         min_migration_steps: u64| {
+        let blocks_before = get_all_blocks(&env, canister_id);
+
         env.upgrade_canister(
             canister_id,
             ledger_wasm,
@@ -2589,6 +2592,8 @@ pub fn icrc1_test_multi_step_migration<T>(
 
         wait_ledger_ready(&env, canister_id, 20);
 
+        assert_eq!(blocks_before, get_all_blocks(&env, canister_id));
+
         let stable_upgrade_migration_steps =
             parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps");
         assert!(stable_upgrade_migration_steps >= min_migration_steps);

From db6a4dd3e87fc5b73c97cbdcb2e2970d393a00c7 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Tue, 4 Feb 2025 16:37:19 +0000
Subject: [PATCH 10/38] use encoded blocks for calculating hash

---
 rs/ledger_suite/icp/ledger/tests/tests.rs | 39 +++++------------------
 1 file changed, 8 insertions(+), 31 deletions(-)

diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs
index 614015dab6a..473434d8787 100644
--- a/rs/ledger_suite/icp/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icp/ledger/tests/tests.rs
@@ -1268,19 +1268,9 @@ fn test_upgrade_serialization(ledger_wasm_mainnet: Vec<u8>) {
 
 fn get_all_blocks(state_machine: &StateMachine, ledger_id: CanisterId) -> Vec<EncodedBlock> {
     let p1 = PrincipalId::new_user_test_id(1);
-    let blocks_res = query_blocks(&state_machine, p1.0, ledger_id, 0, u32::MAX.into());
+    let blocks_res = query_encoded_blocks(state_machine, p1.0, ledger_id, 0, u32::MAX.into());
     let mut result = vec![];
-    println!(
-        "chain length: {}, local length: {}, num_archives: {}",
-        blocks_res.chain_length,
-        blocks_res.blocks.len(),
-        blocks_res.archived_blocks.len(),
-    );
     for archived in blocks_res.archived_blocks {
-        println!(
-            "get blocks from archive, start: {}, length: {}",
-            archived.start, archived.length
-        );
         let get_blocks_args = Encode!(&GetBlocksArgs {
             start: archived.start,
             length: archived.length,
@@ -1290,42 +1280,29 @@ fn get_all_blocks(state_machine: &StateMachine, ledger_id: CanisterId) -> Vec<En
             &state_machine
                 .query(
                     CanisterId::unchecked_from_principal(archived.callback.canister_id.into()),
-                    "get_blocks",
+                    "get_encoded_blocks",
                     get_blocks_args.clone()
                 )
                 .unwrap()
                 .bytes(),
-            GetBlocksResult
+            GetEncodedBlocksResult
         )
         .unwrap()
         .unwrap();
-        result.extend(archived_blocks.blocks);
+        result.extend(archived_blocks);
     }
 
     result.extend(blocks_res.blocks);
     assert_eq!(result.len(), blocks_res.chain_length as usize);
+
     let mut prev_hash = None;
-    let mut i = 0;
-
-    for block in &result {
-        println!("for loop iter {i}");
-        i += 1;
-        let block: Block = block.clone().try_into().unwrap();
-        println!("{:?}", block.parent_hash);
-        println!("{:?}", prev_hash);
-        println!("");
+    for encoded_block in &result {
+        let block = Block::decode(encoded_block.clone()).expect("failed to decode block");
         assert_eq!(block.parent_hash, prev_hash);
-        let encoded_block = block.encode();
-        prev_hash = Some(icp_ledger::Block::block_hash(&encoded_block));
+        prev_hash = Some(Block::block_hash(encoded_block));
     }
 
     result
-        .into_iter()
-        .map(|b| {
-            let block: Block = b.try_into().unwrap();
-            block.encode()
-        })
-        .collect()
 }
 
 #[test]

From e53fff7a8d4c69cb5c7899bd8afa3a5a295b8c32 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Tue, 4 Feb 2025 17:01:01 +0000
Subject: [PATCH 11/38] disable block endpoints while migrating

---
 rs/ledger_suite/icrc1/ledger/src/main.rs    |  2 ++
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 32 +++++++++++++++------
 rs/ledger_suite/tests/sm-tests/src/lib.rs   |  2 +-
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs
index 0410de715df..b015ecc3ad9 100644
--- a/rs/ledger_suite/icrc1/ledger/src/main.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/main.rs
@@ -803,6 +803,7 @@ fn get_transactions(req: GetTransactionsRequest) -> GetTransactionsResponse {
 #[query]
 #[candid_method(query)]
 fn get_blocks(req: GetBlocksRequest) -> GetBlocksResponse {
+    panic_if_not_ready();
     let (start, length) = req
         .as_start_and_length()
         .unwrap_or_else(|msg| ic_cdk::api::trap(&msg));
@@ -972,6 +973,7 @@ fn icrc3_supported_block_types() -> Vec<icrc_ledger_types::icrc3::blocks::Suppor
 #[query]
 #[candid_method(query)]
 fn icrc3_get_blocks(args: Vec<GetBlocksRequest>) -> GetBlocksResult {
+    panic_if_not_ready();
     Access::with_ledger(|ledger| ledger.icrc3_get_blocks(args))
 }
 
diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index bf7967c0bd1..26e1e925f8b 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -611,23 +611,37 @@ fn icrc1_test_downgrade_from_incompatible_version() {
     );
 }
 
+#[test]
+fn icrc1_test_stable_migration_endpoints_disabled_from_mainnet() {
+    test_stable_migration_endpoints_disabled(ledger_mainnet_wasm());
+}
+
 #[test]
 fn icrc1_test_stable_migration_endpoints_disabled_from_v3() {
-    ic_ledger_suite_state_machine_tests::icrc1_test_stable_migration_endpoints_disabled(
-        ledger_mainnet_v3_wasm(),
-        ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args,
-        vec![],
-    );
+    test_stable_migration_endpoints_disabled(ledger_mainnet_v3_wasm());
 }
 
 #[test]
 fn icrc1_test_stable_migration_endpoints_disabled_from_v2() {
+    test_stable_migration_endpoints_disabled(ledger_mainnet_v2_wasm());
+}
+
+fn test_stable_migration_endpoints_disabled(ledger_wasm_mainnet: Vec<u8>) {
+    let get_blocks_arg = Encode!(&GetBlocksRequest {
+        start: Nat::from(0u64),
+        length: Nat::from(1u64),
+    })
+    .unwrap();
+    let args: Vec<GetBlocksRequest> = vec![];
+    let icrc3_get_blocks_arg = Encode!(&args).unwrap();
     ic_ledger_suite_state_machine_tests::icrc1_test_stable_migration_endpoints_disabled(
-        ledger_mainnet_v2_wasm(),
+        ledger_wasm_mainnet,
         ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args,
-        vec![],
+        encode_init_args_with_large_archive_trigger_threshold,
+        vec![
+            ("get_blocks", get_blocks_arg),
+            ("icrc3_get_blocks", icrc3_get_blocks_arg),
+        ],
     );
 }
 
diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs
index 184ca9180f5..56bed38c3b5 100644
--- a/rs/ledger_suite/tests/sm-tests/src/lib.rs
+++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs
@@ -2825,7 +2825,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled<T>(
         test_endpoint(endpoint_name, args, true);
     }
 
-    wait_ledger_ready(&env, canister_id, 20);
+    wait_ledger_ready(&env, canister_id, 50);
 
     test_endpoint("icrc1_transfer", Encode!(&transfer_args).unwrap(), false);
     test_endpoint("icrc2_approve", Encode!(&approve_args).unwrap(), false);

From f13c8ecc2444d915b026899986b5762a2f5e18e0 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 09:58:13 +0000
Subject: [PATCH 12/38] disable get_transactions while migrating, add test

---
 rs/ledger_suite/icrc1/ledger/src/main.rs    | 1 +
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs
index b015ecc3ad9..762d4fd98f2 100644
--- a/rs/ledger_suite/icrc1/ledger/src/main.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/main.rs
@@ -793,6 +793,7 @@ fn supported_standards() -> Vec<StandardRecord> {
 #[query]
 #[candid_method(query)]
 fn get_transactions(req: GetTransactionsRequest) -> GetTransactionsResponse {
+    panic_if_not_ready();
     let (start, length) = req
         .as_start_and_length()
         .unwrap_or_else(|msg| ic_cdk::api::trap(&msg));
diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index 26e1e925f8b..31e5228dcd6 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -639,7 +639,8 @@ fn test_stable_migration_endpoints_disabled(ledger_wasm_mainnet: Vec<u8>) {
         ledger_wasm_lowupgradeinstructionlimits(),
         encode_init_args_with_large_archive_trigger_threshold,
         vec![
-            ("get_blocks", get_blocks_arg),
+            ("get_blocks", get_blocks_arg.clone()),
+            ("get_transactions", get_blocks_arg),
             ("icrc3_get_blocks", icrc3_get_blocks_arg),
         ],
     );

From 34216499b79d35668e57a3a917b2c1a5d5b1386a Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 10:03:23 +0000
Subject: [PATCH 13/38] fix upgrade downgrade test

---
 rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs b/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs
index db8191ea320..d44725258db 100644
--- a/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs
+++ b/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs
@@ -77,12 +77,20 @@ fn should_upgrade_and_downgrade_ledger_canister_suite() {
     )
     .unwrap();
 
-    env.upgrade_canister(
+    match env.upgrade_canister(
         ledger_id,
         ledger_mainnet_wasm(),
         Encode!(&ledger_upgrade_arg).unwrap(),
-    )
-    .unwrap();
+    ) {
+        Ok(_) => {
+            panic!("Upgrade to mainnet should fail!")
+        }
+        Err(e) => {
+            assert!(e
+                .description()
+                .contains("Trying to downgrade from incompatible version"))
+        }
+    };
 }
 
 fn default_archive_options() -> ArchiveOptions {

From 244ed8091593daa007a645ed78e7e2ad1cb91827 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 12:07:42 +0000
Subject: [PATCH 14/38] add comments

---
 .../ledger_canister_core/src/blockchain.rs       | 16 ++++++++++++++++
 rs/ledger_suite/icp/ledger/tests/tests.rs        |  3 +++
 2 files changed, 19 insertions(+)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index fe4eaf4c877..2f880a18a7c 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -11,9 +11,25 @@ use ic_ledger_core::timestamp::TimeStamp;
 use ic_ledger_hash_of::HashOf;
 use std::ops::Range;
 
+// There is a discrepancy in the way the trait uses indices for
+// adding and getting blocks (see `add_block` and `get_blocks`).
+// This is due to the fact that `HeapBlockData` doesn't store
+// block indices. Once `HeapBlockData` is removed, the getters
+// can be switched to global indices and `Blockchain` code can
+// be simplifies - it currently needs to offset indices passed
+// to getters.
 pub trait BlockData {
+    // The `index` should take into account archived blocks.
+    // I.e. if there are 10 archived blocks and we add 11th block
+    // to the ledger, it should be added with index 10.
     fn add_block(&mut self, index: u64, block: EncodedBlock);
+    // The `range` should be 0 based - independently of the number
+    // of archived blocks. I.e. `get_blocks(0..1)` should always return
+    // the first block stored in the ledger.
     fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock>;
+    // The `index` should be 0 based - independently of the number
+    // of archived blocks. I.e. `get_block(0)` should always return
+    // the first block stored in the ledger.
     fn get_block(&self, index: u64) -> Option<EncodedBlock>;
     fn remove_blocks(&mut self, num_blocks: u64);
     fn len(&self) -> u64;
diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs
index 473434d8787..883629cdf75 100644
--- a/rs/ledger_suite/icp/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icp/ledger/tests/tests.rs
@@ -1266,6 +1266,9 @@ fn test_upgrade_serialization(ledger_wasm_mainnet: Vec<u8>) {
     );
 }
 
+// This function should only be used in small tests (<2000 blocks).
+// It only makes one query to ledger and archives and fails if it is not able
+// to get all blocks this way.
 fn get_all_blocks(state_machine: &StateMachine, ledger_id: CanisterId) -> Vec<EncodedBlock> {
     let p1 = PrincipalId::new_user_test_id(1);
     let blocks_res = query_encoded_blocks(state_machine, p1.0, ledger_id, 0, u32::MAX.into());

From 73654ab74ffbc251e87e07a76b463fbd355583ac Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 12:18:10 +0000
Subject: [PATCH 15/38] remove comment

---
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 2f880a18a7c..eefc5042b79 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -205,8 +205,6 @@ where
     }
 
     pub fn remove_archived_blocks(&mut self, len: usize) {
-        // redundant since split_off would panic, but here we can give a more
-        // descriptive message
         if len as u64 > self.blocks.len() {
             panic!(
                 "Asked to remove more blocks than present. Present: {}, to remove: {}",

From e6495b1d37c55669edf62ac6a1a1b57ddc62d5ba Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 15:40:47 +0000
Subject: [PATCH 16/38] add tests from mainnet

---
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 36 +++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index 31e5228dcd6..afe48fe65bf 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -646,6 +646,15 @@ fn test_stable_migration_endpoints_disabled(ledger_wasm_mainnet: Vec<u8>) {
     );
 }
 
+#[test]
+fn icrc1_test_incomplete_migration_from_mainnet() {
+    ic_ledger_suite_state_machine_tests::test_incomplete_migration(
+        ledger_mainnet_wasm(),
+        ledger_wasm_lowupgradeinstructionlimits(),
+        encode_init_args_with_large_archive_trigger_threshold,
+    );
+}
+
 #[test]
 fn icrc1_test_incomplete_migration_from_v3() {
     ic_ledger_suite_state_machine_tests::test_incomplete_migration(
@@ -673,6 +682,15 @@ fn icrc1_test_incomplete_migration_from_v2_noledgerversion() {
     );
 }
 
+#[test]
+fn icrc1_test_incomplete_migration_to_current_from_mainnet() {
+    ic_ledger_suite_state_machine_tests::test_incomplete_migration_to_current(
+        ledger_mainnet_wasm(),
+        ledger_wasm_lowupgradeinstructionlimits(),
+        encode_init_args_with_large_archive_trigger_threshold,
+    );
+}
+
 #[test]
 fn icrc1_test_incomplete_migration_to_current_from_v3() {
     ic_ledger_suite_state_machine_tests::test_incomplete_migration_to_current(
@@ -700,6 +718,15 @@ fn icrc1_test_incomplete_migration_to_current_from_v2_noledgerversion() {
     );
 }
 
+#[test]
+fn icrc1_test_migration_resumes_from_frozen_from_mainnet() {
+    ic_ledger_suite_state_machine_tests::test_migration_resumes_from_frozen(
+        ledger_mainnet_wasm(),
+        ledger_wasm_lowupgradeinstructionlimits(),
+        encode_init_args,
+    );
+}
+
 #[test]
 fn icrc1_test_migration_resumes_from_frozen_from_v3() {
     ic_ledger_suite_state_machine_tests::test_migration_resumes_from_frozen(
@@ -718,6 +745,15 @@ fn icrc1_test_migration_resumes_from_frozen_from_v2() {
     );
 }
 
+#[test]
+fn icrc1_test_metrics_while_migrating_from_mainnet() {
+    ic_ledger_suite_state_machine_tests::test_metrics_while_migrating(
+        ledger_mainnet_wasm(),
+        ledger_wasm_lowupgradeinstructionlimits(),
+        encode_init_args_with_large_archive_trigger_threshold,
+    );
+}
+
 #[test]
 fn icrc1_test_metrics_while_migrating_from_v3() {
     ic_ledger_suite_state_machine_tests::test_metrics_while_migrating(

From 7bdb2bf27544a3c37292bac07a750e0acc91fd0c Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 17:00:59 +0000
Subject: [PATCH 17/38] update canbench results

---
 .../ledger/canbench_results/canbench_u256.yml | 36 +++++++++----------
 .../ledger/canbench_results/canbench_u64.yml  | 34 +++++++++---------
 2 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml
index 27a1b10cf2e..aa0bf52825f 100644
--- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml
+++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml
@@ -1,54 +1,54 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 58832813772
-      heap_increase: 271
+      instructions: 67536723756
+      heap_increase: 262
       stable_memory_increase: 256
     scopes:
       icrc1_transfer:
-        instructions: 13635001695
-        heap_increase: 43
+        instructions: 16505736562
+        heap_increase: 31
         stable_memory_increase: 0
       icrc2_approve:
-        instructions: 20413485760
-        heap_increase: 41
+        instructions: 23302128840
+        heap_increase: 28
         stable_memory_increase: 128
       icrc2_transfer_from:
-        instructions: 24042619927
-        heap_increase: 5
+        instructions: 26980407644
+        heap_increase: 3
         stable_memory_increase: 0
       icrc3_get_blocks:
-        instructions: 7877258
+        instructions: 8557135
         heap_increase: 0
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 354777092
-        heap_increase: 53
+        instructions: 353759761
+        heap_increase: 71
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 149556765
+        instructions: 149186080
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 504335921
-        heap_increase: 182
+        instructions: 502947905
+        heap_increase: 200
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8684182
+      instructions: 8683947
       heap_increase: 258
       stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8605997
+        instructions: 8605927
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 75778
+        instructions: 75628
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 8683493
+        instructions: 8683258
         heap_increase: 258
         stable_memory_increase: 128
 version: 0.1.8
diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
index fd17caa5c7c..8e647a0291b 100644
--- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
+++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
@@ -1,54 +1,54 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 56837535933
-      heap_increase: 271
+      instructions: 65554136279
+      heap_increase: 264
       stable_memory_increase: 256
     scopes:
       icrc1_transfer:
-        instructions: 13071791984
-        heap_increase: 42
+        instructions: 15857556107
+        heap_increase: 30
         stable_memory_increase: 0
       icrc2_approve:
-        instructions: 19627513485
+        instructions: 22630062632
         heap_increase: 29
         stable_memory_increase: 128
       icrc2_transfer_from:
-        instructions: 23404082941
-        heap_increase: 18
+        instructions: 26322782282
+        heap_increase: 3
         stable_memory_increase: 0
       icrc3_get_blocks:
-        instructions: 7540214
+        instructions: 8191444
         heap_increase: 0
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 353134588
-        heap_increase: 53
+        instructions: 354450730
+        heap_increase: 73
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 149315334
+        instructions: 149186269
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 502451986
-        heap_increase: 182
+        instructions: 503639063
+        heap_increase: 202
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8683414
+      instructions: 8684960
       heap_increase: 258
       stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8604304
+        instructions: 8605982
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 76703
+        instructions: 76586
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 8682725
+        instructions: 8684271
         heap_increase: 258
         stable_memory_increase: 128
 version: 0.1.8

From 919f4b160ead2c00b3557303ceaa7a9f4076e894 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 17:24:01 +0000
Subject: [PATCH 18/38] checked_sub

---
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index eefc5042b79..b67800d4eb5 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -160,7 +160,7 @@ where
             None
         } else {
             self.blocks
-                .get_block(height.saturating_sub(self.num_archived_blocks()))
+                .get_block(height.checked_sub(self.num_archived_blocks()).unwrap())
         }
     }
 

From eea231e8212a0c09ba9701585c0712cd00e3d0a5 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 5 Feb 2025 17:31:50 +0000
Subject: [PATCH 19/38] simplify range

---
 .../common/ledger_canister_core/src/blockchain.rs        | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index b67800d4eb5..df6138ecefb 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -232,11 +232,10 @@ where
             return VecDeque::new();
         }
 
-        let blocks_to_archive: VecDeque<EncodedBlock> =
-            VecDeque::from(self.blocks.get_blocks(Range {
-                start: 0,
-                end: (num_blocks_to_archive as u64).min(num_blocks_before),
-            }));
+        let blocks_to_archive: VecDeque<EncodedBlock> = VecDeque::from(
+            self.blocks
+                .get_blocks(0..(num_blocks_to_archive as u64).min(num_blocks_before)),
+        );
 
         println!(
             "get_blocks_for_archiving(): trigger_threshold: {}, num_blocks: {}, blocks before archiving: {}, blocks to archive: {}",

From a2385ef73b640d5742ed6050363105e81766e418 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 6 Feb 2025 11:36:39 +0000
Subject: [PATCH 20/38] remove todo

---
 rs/ledger_suite/icrc1/ledger/src/lib.rs | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 96ec98af313..a3ccbe9b76f 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -83,9 +83,6 @@ pub type Tokens = ic_icrc1_tokens_u256::U256;
 ///   * 1 - the allowances are stored in stable structures.
 ///   * 2 - the balances are stored in stable structures.
 ///   * 3 - the blocks are stored in stable structures.
-// TODO: When moving to version 3 consider adding `#[serde(default, skip_serializing)]`
-// to `balances` and `approvals` fields of the `Ledger` struct.
-// Since `balances` don't use a default, this can only be done with an incompatible change.
 #[cfg(not(feature = "next-ledger-version"))]
 pub const LEDGER_VERSION: u64 = 3;
 

From 133879cb926a0dea82a295fc1ead51090412dc16 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 6 Feb 2025 12:53:24 +0000
Subject: [PATCH 21/38] comment

---
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index df6138ecefb..ea9af5034ea 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -12,7 +12,8 @@ use ic_ledger_hash_of::HashOf;
 use std::ops::Range;
 
 // There is a discrepancy in the way the trait uses indices for
-// adding and getting blocks (see `add_block` and `get_blocks`).
+// adding and getting blocks - `add_block` uses global indices
+// while `get_blocks` uses 0-based indices (local).
 // This is due to the fact that `HeapBlockData` doesn't store
 // block indices. Once `HeapBlockData` is removed, the getters
 // can be switched to global indices and `Blockchain` code can

From f0ae72c610a9cc85493017e46a9a4e8b35bc0109 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 6 Feb 2025 17:00:05 +0000
Subject: [PATCH 22/38] small refactor

---
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index afe48fe65bf..c4e296326ad 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -248,12 +248,12 @@ fn encode_init_args_with_small_sized_archive(
     }
 }
 
-fn encode_init_args_with_large_archive_trigger_threshold(
+fn encode_init_args_no_archiving(
     args: ic_ledger_suite_state_machine_tests::InitArgs,
 ) -> LedgerArgument {
     match encode_init_args(args) {
         LedgerArgument::Init(mut init_args) => {
-            init_args.archive_options.trigger_threshold = 1000;
+            init_args.archive_options.trigger_threshold = 1_000_000_000_000;
             LedgerArgument::Init(init_args)
         }
         LedgerArgument::Upgrade(_) => {
@@ -565,7 +565,7 @@ fn icrc1_test_multi_step_migration_from_mainnet() {
     ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration(
         ledger_mainnet_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args_with_large_archive_trigger_threshold,
+        encode_init_args_no_archiving,
         get_all_blocks,
     );
 }
@@ -637,7 +637,7 @@ fn test_stable_migration_endpoints_disabled(ledger_wasm_mainnet: Vec<u8>) {
     ic_ledger_suite_state_machine_tests::icrc1_test_stable_migration_endpoints_disabled(
         ledger_wasm_mainnet,
         ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args_with_large_archive_trigger_threshold,
+        encode_init_args_no_archiving,
         vec![
             ("get_blocks", get_blocks_arg.clone()),
             ("get_transactions", get_blocks_arg),
@@ -651,7 +651,7 @@ fn icrc1_test_incomplete_migration_from_mainnet() {
     ic_ledger_suite_state_machine_tests::test_incomplete_migration(
         ledger_mainnet_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args_with_large_archive_trigger_threshold,
+        encode_init_args_no_archiving,
     );
 }
 
@@ -687,7 +687,7 @@ fn icrc1_test_incomplete_migration_to_current_from_mainnet() {
     ic_ledger_suite_state_machine_tests::test_incomplete_migration_to_current(
         ledger_mainnet_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args_with_large_archive_trigger_threshold,
+        encode_init_args_no_archiving,
     );
 }
 
@@ -750,7 +750,7 @@ fn icrc1_test_metrics_while_migrating_from_mainnet() {
     ic_ledger_suite_state_machine_tests::test_metrics_while_migrating(
         ledger_mainnet_wasm(),
         ledger_wasm_lowupgradeinstructionlimits(),
-        encode_init_args_with_large_archive_trigger_threshold,
+        encode_init_args_no_archiving,
     );
 }
 

From 5b16bd548a0a63023e32cc5e6d56e4e70b020b8b Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 6 Feb 2025 21:57:14 +0000
Subject: [PATCH 23/38] build fix

---
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index c4e296326ad..b39c3684fe2 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -253,7 +253,7 @@ fn encode_init_args_no_archiving(
 ) -> LedgerArgument {
     match encode_init_args(args) {
         LedgerArgument::Init(mut init_args) => {
-            init_args.archive_options.trigger_threshold = 1_000_000_000_000;
+            init_args.archive_options.trigger_threshold = 1_000_000_000;
             LedgerArgument::Init(init_args)
         }
         LedgerArgument::Upgrade(_) => {

From f6e54e1053dff21973ddcd7dce97e78f5090727c Mon Sep 17 00:00:00 2001
From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com>
Date: Fri, 7 Feb 2025 11:51:07 +0100
Subject: [PATCH 24/38] Update
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Mathias Björkqvist <mathias.bjorkqvist@dfinity.org>
---
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index ea9af5034ea..72ea6bf67af 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -17,7 +17,7 @@ use std::ops::Range;
 // This is due to the fact that `HeapBlockData` doesn't store
 // block indices. Once `HeapBlockData` is removed, the getters
 // can be switched to global indices and `Blockchain` code can
-// be simplifies - it currently needs to offset indices passed
+// be simplified - it currently needs to offset indices passed
 // to getters.
 pub trait BlockData {
     // The `index` should take into account archived blocks.

From 3ff3a43f9d343f6979cd565ae07a19da183ab46d Mon Sep 17 00:00:00 2001
From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com>
Date: Fri, 7 Feb 2025 11:51:30 +0100
Subject: [PATCH 25/38] Update
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Mathias Björkqvist <mathias.bjorkqvist@dfinity.org>
---
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 72ea6bf67af..b6fe86245d1 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -225,7 +225,7 @@ where
         // Upon reaching the `trigger_threshold` we will archive
         // `num_blocks_to_archive`. For example, when set to (2000, 1000)
         // archiving will trigger when there are 2000 blocks in the ledger and
-        // the 1000 oldest bocks will be archived, leaving the remaining 1000
+        // the 1000 oldest blocks will be archived, leaving the remaining 1000
         // blocks in place.
         let num_blocks_before = self.num_unarchived_blocks();
 

From 993e5460da1cda1d24418e434018018012e81360 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 7 Feb 2025 11:37:25 +0000
Subject: [PATCH 26/38] clear stable blocks data before migration

---
 rs/ledger_suite/icrc1/ledger/src/lib.rs  | 6 ++++++
 rs/ledger_suite/icrc1/ledger/src/main.rs | 7 +++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index a3ccbe9b76f..b48da6ea325 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -1158,6 +1158,12 @@ pub fn clear_stable_balances_data() {
     });
 }
 
+pub fn clear_stable_blocks_data() {
+    BLOCKS_MEMORY.with_borrow_mut(|blocks| {
+        blocks.clear_new();
+    });
+}
+
 pub fn balances_len() -> u64 {
     BALANCES_MEMORY.with_borrow(|balances| balances.len())
 }
diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs
index 762d4fd98f2..2b2e0ca8476 100644
--- a/rs/ledger_suite/icrc1/ledger/src/main.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/main.rs
@@ -16,8 +16,9 @@ use ic_icrc1::{
     Operation, Transaction,
 };
 use ic_icrc1_ledger::{
-    balances_len, clear_stable_allowance_data, clear_stable_balances_data, is_ready, ledger_state,
-    panic_if_not_ready, set_ledger_state, LEDGER_VERSION, UPGRADES_MEMORY,
+    balances_len, clear_stable_allowance_data, clear_stable_balances_data,
+    clear_stable_blocks_data, is_ready, ledger_state, panic_if_not_ready, set_ledger_state,
+    LEDGER_VERSION, UPGRADES_MEMORY,
 };
 use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState};
 use ic_ledger_canister_core::ledger::{
@@ -240,6 +241,8 @@ fn post_upgrade(args: Option<LedgerArgument>) {
 
     if upgrade_from_version < 3 {
         set_ledger_state(LedgerState::Migrating(LedgerField::Blocks));
+        log_message(format!("Upgrading from version {upgrade_from_version} which does not store blocks in stable structures, clearing stable blocks data.").as_str());
+        clear_stable_blocks_data();
     }
     if upgrade_from_version < 2 {
         set_ledger_state(LedgerState::Migrating(LedgerField::Balances));

From d03a97d25c6c4179376c04352893a469f7f70ff8 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 7 Feb 2025 12:10:13 +0000
Subject: [PATCH 27/38] rename block_slice to get_blocks

---
 rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs | 4 ++--
 rs/ledger_suite/icp/ledger/src/main.rs                        | 4 ++--
 rs/ledger_suite/icrc1/ledger/src/lib.rs                       | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index b6fe86245d1..90d09f6917b 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -182,12 +182,12 @@ where
         self.num_archived_blocks..self.num_archived_blocks + self.blocks.len()
     }
 
-    /// Returns the slice of blocks stored locally.
+    /// Returns the blocks stored locally.
     ///
     /// # Panic
     ///
     /// This function panics if the specified range is not a subset of locally available blocks.
-    pub fn block_slice(&self, local_blocks: std::ops::Range<u64>) -> Vec<EncodedBlock> {
+    pub fn get_blocks(&self, local_blocks: std::ops::Range<u64>) -> Vec<EncodedBlock> {
         use crate::range_utils::{is_subrange, offset};
 
         assert!(
diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index 489736fd5d6..4fc9fbc5827 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -1366,7 +1366,7 @@ fn query_blocks(GetBlocksArgs { start, length }: GetBlocksArgs) -> QueryBlocksRe
 
     let blocks: Vec<CandidBlock> = ledger
         .blockchain
-        .block_slice(local_blocks.clone())
+        .get_blocks(local_blocks.clone())
         .iter()
         .map(|enc_block| {
             CandidBlock::from(
@@ -1584,7 +1584,7 @@ fn query_encoded_blocks(
     let local_blocks =
         range_utils::take(&locations.local_blocks, max_blocks_per_request(&caller()));
 
-    let blocks = ledger.blockchain.block_slice(local_blocks.clone()).to_vec();
+    let blocks = ledger.blockchain.get_blocks(local_blocks.clone()).to_vec();
 
     let archived_blocks = locations
         .archived_blocks
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index b48da6ea325..608a5f39fae 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -981,7 +981,7 @@ impl Ledger {
 
         let local_blocks: Vec<B> = self
             .blockchain
-            .block_slice(local_blocks_range)
+            .get_blocks(local_blocks_range)
             .iter()
             .map(decode)
             .collect();

From 64673b00ef742ab8cce3ec702312b538c50cbec1 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 7 Feb 2025 12:39:10 +0000
Subject: [PATCH 28/38] remove oldest blocks

---
 .../common/ledger_canister_core/src/blockchain.rs          | 7 ++++---
 rs/ledger_suite/icrc1/ledger/src/lib.rs                    | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 90d09f6917b..cd13ee08c79 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -32,7 +32,8 @@ pub trait BlockData {
     // of archived blocks. I.e. `get_block(0)` should always return
     // the first block stored in the ledger.
     fn get_block(&self, index: u64) -> Option<EncodedBlock>;
-    fn remove_blocks(&mut self, num_blocks: u64);
+    /// Removes `num_blocks` with the smallest index.
+    fn remove_oldest_blocks(&mut self, num_blocks: u64);
     fn len(&self) -> u64;
     fn is_empty(&self) -> bool;
     fn last(&self) -> Option<EncodedBlock>;
@@ -62,7 +63,7 @@ impl BlockData for HeapBlockData {
         self.blocks.get(index as usize).cloned()
     }
 
-    fn remove_blocks(&mut self, num_blocks: u64) {
+    fn remove_oldest_blocks(&mut self, num_blocks: u64) {
         self.blocks = self.blocks.split_off(num_blocks as usize);
     }
 
@@ -213,7 +214,7 @@ where
                 len
             );
         }
-        self.blocks.remove_blocks(len as u64);
+        self.blocks.remove_oldest_blocks(len as u64);
         self.num_archived_blocks += len as u64;
     }
 
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 608a5f39fae..6928d67fb68 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -1339,7 +1339,7 @@ impl BlockData for StableBlockData {
         })
     }
 
-    fn remove_blocks(&mut self, num_blocks: u64) {
+    fn remove_oldest_blocks(&mut self, num_blocks: u64) {
         BLOCKS_MEMORY.with_borrow_mut(|blocks| {
             let mut removed = 0;
             while !blocks.is_empty() && removed < num_blocks {

From 0a2fe7ec93f27a6b4f82e806bda0631f857c2f35 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 7 Feb 2025 13:01:41 +0000
Subject: [PATCH 29/38] disable certificate getters while migrating

---
 rs/ledger_suite/icrc1/ledger/src/main.rs    | 2 ++
 rs/ledger_suite/icrc1/ledger/tests/tests.rs | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs
index 2b2e0ca8476..31ce5b260a7 100644
--- a/rs/ledger_suite/icrc1/ledger/src/main.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/main.rs
@@ -817,6 +817,7 @@ fn get_blocks(req: GetBlocksRequest) -> GetBlocksResponse {
 #[query]
 #[candid_method(query)]
 fn get_data_certificate() -> DataCertificate {
+    panic_if_not_ready();
     let hash_tree = Access::with_ledger(|ledger| ledger.construct_hash_tree());
     let mut tree_buf = vec![];
     ciborium::ser::into_writer(&hash_tree, &mut tree_buf).unwrap();
@@ -935,6 +936,7 @@ fn icrc3_get_archives(args: GetArchivesArgs) -> GetArchivesResult {
 #[query]
 #[candid_method(query)]
 fn icrc3_get_tip_certificate() -> Option<ICRC3DataCertificate> {
+    panic_if_not_ready();
     let certificate = ByteBuf::from(ic_cdk::api::data_certificate()?);
     let hash_tree = Access::with_ledger(|ledger| ledger.construct_hash_tree());
     let mut tree_buf = vec![];
diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
index b39c3684fe2..a5c72dfc0a5 100644
--- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs
+++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs
@@ -642,6 +642,8 @@ fn test_stable_migration_endpoints_disabled(ledger_wasm_mainnet: Vec<u8>) {
             ("get_blocks", get_blocks_arg.clone()),
             ("get_transactions", get_blocks_arg),
             ("icrc3_get_blocks", icrc3_get_blocks_arg),
+            ("get_data_certificate", Encode!().unwrap()),
+            ("icrc3_get_tip_certificate", Encode!().unwrap()),
         ],
     );
 }

From 1205982494bc0a21f9b5414eb65cd17387e43c75 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Fri, 7 Feb 2025 14:03:02 +0000
Subject: [PATCH 30/38] small refactor

---
 rs/ledger_suite/icp/ledger/src/main.rs | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index 4fc9fbc5827..469725eabcb 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -1322,19 +1322,16 @@ fn icrc1_total_supply_candid() {
 #[export_name = "canister_query iter_blocks_pb"]
 fn iter_blocks_() {
     over(protobuf, |IterBlocksArgs { start, length }| {
-        let length = std::cmp::min(length, max_blocks_per_request(&caller()));
-        let blocks_len = LEDGER.read().unwrap().blockchain.num_unarchived_blocks() as usize;
-        let start = std::cmp::min(start, blocks_len);
+        let length = std::cmp::min(length, max_blocks_per_request(&caller())) as u64;
+        let blocks_len = LEDGER.read().unwrap().blockchain.num_unarchived_blocks();
+        let start = std::cmp::min(start as u64, blocks_len);
         let end = std::cmp::min(start + length, blocks_len);
         let blocks = LEDGER
             .read()
             .unwrap()
             .blockchain
             .blocks
-            .get_blocks(std::ops::Range {
-                start: start as u64,
-                end: end as u64,
-            });
+            .get_blocks(start..end);
         IterBlocksRes(blocks)
     });
 }

From 834770fdb667da5380efb1f573f33ed20daeb3f9 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 24 Feb 2025 13:05:13 +0000
Subject: [PATCH 31/38] fix test

---
 rs/ledger_suite/tests/sm-tests/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs
index 55943269330..2345f384024 100644
--- a/rs/ledger_suite/tests/sm-tests/src/lib.rs
+++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs
@@ -3177,7 +3177,7 @@ pub fn test_migration_resumes_from_frozen<T>(
     unfreeze(&env, canister_id);
     // even though 1000s passed, the ledger did not migrate when it was frozen
     assert!(!is_ledger_ready());
-    wait_ledger_ready(&env, canister_id, 20);
+    wait_ledger_ready(&env, canister_id, 30);
     check_approvals();
     check_balances();
 }

From 37e224a3f10190e1d2319afdb9ca32c6ca157aa5 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 24 Feb 2025 14:35:21 +0000
Subject: [PATCH 32/38] remove unused fn

---
 .../common/ledger_canister_core/src/blockchain.rs        | 9 ---------
 rs/ledger_suite/icrc1/ledger/src/lib.rs                  | 8 --------
 2 files changed, 17 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index cd13ee08c79..3530461d075 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -36,7 +36,6 @@ pub trait BlockData {
     fn remove_oldest_blocks(&mut self, num_blocks: u64);
     fn len(&self) -> u64;
     fn is_empty(&self) -> bool;
-    fn last(&self) -> Option<EncodedBlock>;
     fn migrate_one_block(&mut self, num_archived_blocks: u64) -> bool;
 }
 
@@ -75,10 +74,6 @@ impl BlockData for HeapBlockData {
         self.blocks.is_empty()
     }
 
-    fn last(&self) -> Option<EncodedBlock> {
-        self.blocks.last().cloned()
-    }
-
     fn migrate_one_block(&mut self, _num_archived_blocks: u64) -> bool {
         panic!("HeapBlockData cannot perform migration!");
     }
@@ -166,10 +161,6 @@ where
         }
     }
 
-    pub fn last(&self) -> Option<EncodedBlock> {
-        self.blocks.last()
-    }
-
     pub fn num_archived_blocks(&self) -> u64 {
         self.num_archived_blocks
     }
diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 590482a2514..77ed2ea1d33 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -1378,14 +1378,6 @@ impl BlockData for StableBlockData {
         BLOCKS_MEMORY.with_borrow(|blocks| blocks.is_empty())
     }
 
-    fn last(&self) -> Option<EncodedBlock> {
-        BLOCKS_MEMORY.with_borrow(|blocks| {
-            blocks
-                .last_key_value()
-                .map(|kv| EncodedBlock::from_vec(kv.1))
-        })
-    }
-
     fn migrate_one_block(&mut self, num_archived_blocks: u64) -> bool {
         let num_migrated = self.len();
         if num_migrated < self.blocks.len() as u64 {

From cad2ce37132f1c341875cc9ab03f7c3dbf841bd1 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Mon, 24 Feb 2025 14:43:55 +0000
Subject: [PATCH 33/38] small refactor

---
 rs/ledger_suite/icrc1/ledger/src/lib.rs | 26 +++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs
index 77ed2ea1d33..77726cae1ed 100644
--- a/rs/ledger_suite/icrc1/ledger/src/lib.rs
+++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs
@@ -1339,24 +1339,26 @@ impl BlockData for StableBlockData {
     }
 
     fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock> {
-        BLOCKS_MEMORY.with_borrow(|blocks| {
-            let mut result = vec![];
-            let first_index = blocks.first_key_value().unwrap_or((0, vec![])).0;
-            let start = range.start + first_index;
-            let end = range.end + first_index;
-            for block in blocks.range(start..end) {
-                result.push(EncodedBlock::from_vec(block.1));
+        BLOCKS_MEMORY.with_borrow(|blocks| match blocks.first_key_value() {
+            Some((first_index, _)) => {
+                let mut result = vec![];
+                let start = range.start + first_index;
+                let end = range.end + first_index;
+                for block in blocks.range(start..end) {
+                    result.push(EncodedBlock::from_vec(block.1));
+                }
+                result
             }
-            result
+            None => vec![],
         })
     }
 
     fn get_block(&self, index: u64) -> Option<EncodedBlock> {
-        BLOCKS_MEMORY.with_borrow(|blocks| {
-            let first_index = blocks.first_key_value().unwrap_or((0, vec![])).0;
-            blocks
+        BLOCKS_MEMORY.with_borrow(|blocks| match blocks.first_key_value() {
+            Some((first_index, _)) => blocks
                 .get(&(index + first_index))
-                .map(EncodedBlock::from_vec)
+                .map(EncodedBlock::from_vec),
+            None => None,
         })
     }
 

From f08e3d3240c7868800b5c5fb3c6787d7501f839a Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Tue, 25 Feb 2025 13:00:25 +0000
Subject: [PATCH 34/38] update canbench

---
 .../ledger/canbench_results/canbench_u64.yml  | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
index 8e647a0291b..fc82ba2fdca 100644
--- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
+++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
@@ -1,46 +1,46 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 65554136279
-      heap_increase: 264
+      instructions: 65467206107
+      heap_increase: 263
       stable_memory_increase: 256
     scopes:
       icrc1_transfer:
-        instructions: 15857556107
-        heap_increase: 30
+        instructions: 15887391603
+        heap_increase: 34
         stable_memory_increase: 0
       icrc2_approve:
-        instructions: 22630062632
-        heap_increase: 29
+        instructions: 22551561638
+        heap_increase: 25
         stable_memory_increase: 128
       icrc2_transfer_from:
-        instructions: 26322782282
+        instructions: 26286355650
         heap_increase: 3
         stable_memory_increase: 0
       icrc3_get_blocks:
-        instructions: 8191444
+        instructions: 8189132
         heap_increase: 0
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 354450730
-        heap_increase: 73
+        instructions: 352257721
+        heap_increase: 72
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 149186269
+        instructions: 149186270
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 503639063
-        heap_increase: 202
+        instructions: 501446055
+        heap_increase: 201
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8684960
+      instructions: 8683207
       heap_increase: 258
       stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8605982
+        instructions: 8604229
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
@@ -48,7 +48,7 @@ benches:
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 8684271
+        instructions: 8682518
         heap_increase: 258
         stable_memory_increase: 128
-version: 0.1.8
+version: 0.1.9

From e591ad9104b68d1748bfe302f550108d57200026 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 26 Feb 2025 12:51:23 +0000
Subject: [PATCH 35/38] rewrite comments

---
 .../common/ledger_canister_core/src/blockchain.rs    | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index 3530461d075..c5240ee81e2 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -13,7 +13,7 @@ use std::ops::Range;
 
 // There is a discrepancy in the way the trait uses indices for
 // adding and getting blocks - `add_block` uses global indices
-// while `get_blocks` uses 0-based indices (local).
+// while `get_blocks` uses indices relative to the first unarchived block.
 // This is due to the fact that `HeapBlockData` doesn't store
 // block indices. Once `HeapBlockData` is removed, the getters
 // can be switched to global indices and `Blockchain` code can
@@ -24,13 +24,11 @@ pub trait BlockData {
     // I.e. if there are 10 archived blocks and we add 11th block
     // to the ledger, it should be added with index 10.
     fn add_block(&mut self, index: u64, block: EncodedBlock);
-    // The `range` should be 0 based - independently of the number
-    // of archived blocks. I.e. `get_blocks(0..1)` should always return
-    // the first block stored in the ledger.
+    // The `range` should be relative to the first unarchived block.
+    // I.e. `get_blocks(0..1)` should always return the first block stored in the ledger.
     fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock>;
-    // The `index` should be 0 based - independently of the number
-    // of archived blocks. I.e. `get_block(0)` should always return
-    // the first block stored in the ledger.
+    // The `index` should be relative to the first unarchived block.
+    // I.e. `get_block(0)` should always return the first block stored in the ledger.
     fn get_block(&self, index: u64) -> Option<EncodedBlock>;
     /// Removes `num_blocks` with the smallest index.
     fn remove_oldest_blocks(&mut self, num_blocks: u64);

From 8d1b413e68806c1a6e654471504c2a0b3c076c5d Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Wed, 26 Feb 2025 13:30:09 +0000
Subject: [PATCH 36/38] clamp range inside get_blocks

---
 .../common/ledger_canister_core/src/blockchain.rs         | 8 +++-----
 rs/ledger_suite/icp/ledger/src/main.rs                    | 6 ++----
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
index c5240ee81e2..62b08633138 100644
--- a/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
+++ b/rs/ledger_suite/common/ledger_canister_core/src/blockchain.rs
@@ -49,11 +49,9 @@ impl BlockData for HeapBlockData {
     }
 
     fn get_blocks(&self, range: Range<u64>) -> Vec<EncodedBlock> {
-        let range = Range {
-            start: range.start as usize,
-            end: range.end as usize,
-        };
-        self.blocks[range].to_vec()
+        let start = std::cmp::min(range.start, self.len()) as usize;
+        let end = std::cmp::min(range.end, self.len()) as usize;
+        self.blocks[start..end].to_vec()
     }
 
     fn get_block(&self, index: u64) -> Option<EncodedBlock> {
diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index 469725eabcb..53454b0969e 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -1323,15 +1323,13 @@ fn icrc1_total_supply_candid() {
 fn iter_blocks_() {
     over(protobuf, |IterBlocksArgs { start, length }| {
         let length = std::cmp::min(length, max_blocks_per_request(&caller())) as u64;
-        let blocks_len = LEDGER.read().unwrap().blockchain.num_unarchived_blocks();
-        let start = std::cmp::min(start as u64, blocks_len);
-        let end = std::cmp::min(start + length, blocks_len);
+        let start = start as u64;
         let blocks = LEDGER
             .read()
             .unwrap()
             .blockchain
             .blocks
-            .get_blocks(start..end);
+            .get_blocks(start..start + length);
         IterBlocksRes(blocks)
     });
 }

From 9420d73622ee41a24e8fad1eb1e8eec457dd8bb7 Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 27 Feb 2025 10:22:32 +0000
Subject: [PATCH 37/38] build fix

---
 rs/ledger_suite/icp/ledger/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rs/ledger_suite/icp/ledger/src/lib.rs b/rs/ledger_suite/icp/ledger/src/lib.rs
index c1142e8d94e..61fb97211c3 100644
--- a/rs/ledger_suite/icp/ledger/src/lib.rs
+++ b/rs/ledger_suite/icp/ledger/src/lib.rs
@@ -192,7 +192,7 @@ pub struct Ledger {
     approvals: LedgerAllowances,
     #[serde(default)]
     stable_approvals: AllowanceTable<StableAllowancesData>,
-    pub blockchain: Blockchain<CdkRuntime, IcpLedgerArchiveWasm>,
+    pub blockchain: Blockchain<CdkRuntime, IcpLedgerArchiveWasm, HeapBlockData>,
     // DEPRECATED
     pub maximum_number_of_accounts: usize,
     // DEPRECATED

From 60ce7c7593ccf5407f0c9cb751ce7647568083ac Mon Sep 17 00:00:00 2001
From: Maciej Modelski <maciej.modelski@dfinity.org>
Date: Thu, 27 Feb 2025 10:25:29 +0000
Subject: [PATCH 38/38] build fix

---
 rs/ledger_suite/icp/ledger/src/main.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs
index bf37d2e7a6b..e0aabac0cad 100644
--- a/rs/ledger_suite/icp/ledger/src/main.rs
+++ b/rs/ledger_suite/icp/ledger/src/main.rs
@@ -1325,7 +1325,8 @@ fn icrc1_total_supply_candid() {
 #[export_name = "canister_query iter_blocks_pb"]
 fn iter_blocks_() {
     over(protobuf, |IterBlocksArgs { start, length }| {
-        let length = std::cmp::min(length, max_blocks_per_request(&caller())) as u64;
+        let length =
+            std::cmp::min(length, max_blocks_per_request(&PrincipalId::from(caller()))) as u64;
         let start = start as u64;
         let blocks = LEDGER
             .read()