diff --git a/Cargo.lock b/Cargo.lock index a0390d68d..2dab9fe9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,6 +316,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -601,9 +610,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "caret" @@ -754,6 +763,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -853,6 +868,12 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.12" @@ -1221,6 +1242,12 @@ dependencies = [ "zcash_protocol", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1410,6 +1437,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "enum-ordinalize" version = "3.1.15" @@ -1858,6 +1897,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1889,6 +1937,20 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -3046,6 +3108,19 @@ dependencies = [ "thiserror", ] +[[package]] +name = "postcard" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3080,6 +3155,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.20" @@ -3161,9 +3246,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ "bytes", "prost-derive", @@ -3171,9 +3256,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" +checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", "heck 0.5.0", @@ -3192,9 +3277,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", "itertools 0.13.0", @@ -3205,9 +3290,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" dependencies = [ "prost", ] @@ -4030,6 +4115,9 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -4275,6 +4363,7 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "js-sys", "num-conv", "powerfmt", "serde", @@ -5691,6 +5780,17 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "wasm_sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff360cade7fec41ff0e9d2cda57fe58258c5f16def0e21302394659e6bbb0ea" +dependencies = [ + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "weak-table" version = "0.3.2" @@ -5992,6 +6092,12 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "zcash" version = "0.1.0" @@ -6080,6 +6186,48 @@ dependencies = [ "zip321", ] +[[package]] +name = "zcash_client_memory" +version = "0.1.0" +dependencies = [ + "async-trait", + "bip32", + "bs58", + "byteorder", + "bytes", + "ciborium", + "group", + "incrementalmerkletree", + "jubjub", + "nonempty", + "orchard", + "postcard", + "pretty_assertions", + "proptest", + "prost", + "prost-build", + "rayon", + "sapling-crypto", + "secrecy", + "serde_json", + "shardtree", + "static_assertions", + "subtle", + "thiserror", + "time", + "tokio", + "tracing", + "wasm_sync", + "which", + "zcash_address", + "zcash_client_backend", + "zcash_encoding", + "zcash_keys", + "zcash_primitives", + "zcash_protocol", + "zip32", +] + [[package]] name = "zcash_client_sqlite" version = "0.13.0" @@ -6285,9 +6433,9 @@ dependencies = [ [[package]] name = "zcash_spec" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a3bf58b673cb3dacd8ae09ba345998923a197ab0da70d6239d8e8838949e9b" +checksum = "9cede95491c2191d3e278cab76e097a44b17fde8d6ca0d4e3a22cf4807b2d857" dependencies = [ "blake2b_simd", ] @@ -6334,13 +6482,14 @@ dependencies = [ [[package]] name = "zip32" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4226d0aee9c9407c27064dfeec9d7b281c917de3374e1e5a2e2cfad9e09de19e" +checksum = "92022ac1e47c7b78f9cee29efac8a1a546e189506f3bb5ad46d525be7c519bf6" dependencies = [ "blake2b_simd", "memuse", "subtle", + "zcash_spec", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ff0e81c5b..aa0d4c573 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "zcash", "zcash_client_backend", "zcash_client_sqlite", + "zcash_client_memory", "zcash_extensions", "zcash_history", "zcash_keys", @@ -126,7 +127,7 @@ subtle = "2.2.3" rusqlite = { version = "0.32", features = ["bundled"] } schemerz = "0.2" schemerz-rusqlite = "0.320" -time = "0.3.22" +time = { version = "0.3.22", default-features = false } uuid = "1.1" # Static constants and assertions @@ -155,7 +156,7 @@ trait-variant = "0.1" # ZIP 32 aes = "0.8" fpe = "0.6" -zip32 = "0.1.1" +zip32 = "0.1.2" [profile.release] lto = true diff --git a/components/zcash_protocol/src/lib.rs b/components/zcash_protocol/src/lib.rs index f73564751..224b78d3f 100644 --- a/components/zcash_protocol/src/lib.rs +++ b/components/zcash_protocol/src/lib.rs @@ -25,7 +25,7 @@ pub mod memo; pub mod value; /// A Zcash shielded transfer protocol. -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ShieldedProtocol { /// The Sapling protocol Sapling, diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 144faef43..bc8ace46c 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -37,7 +37,7 @@ zip321.workspace = true # Dependencies exposed in a public API: # (Breaking upgrades to these require a breaking upgrade to this crate.) # - Data Access API -time = "0.3.22" +time.workspace = true nonempty.workspace = true # - CSPRNG @@ -151,6 +151,8 @@ zcash_proofs = { workspace = true, features = ["bundled-prover"] } zcash_protocol = { workspace = true, features = ["local-consensus"] } [features] +default = ["time/default"] + ## Enables the `tonic` gRPC client bindings for connecting to a `lightwalletd` server. lightwalletd-tonic = ["dep:tonic", "hyper-util?/tokio"] @@ -228,6 +230,11 @@ unstable-serialization = ["dep:byteorder"] ## Exposes the [`data_api::scanning::spanning_tree`] module. unstable-spanning-tree = [] +## feature to allow building for wasm using wasm-bindgen +wasm-bindgen = [ + "time/wasm-bindgen", +] + [lib] bench = false diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 6eaca5cb7..1b25339e9 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -703,7 +703,7 @@ pub enum TransactionDataRequest { } /// Metadata about the status of a transaction obtained by inspecting the chain state. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum TransactionStatus { /// The requested transaction ID was not recognized by the node. TxidNotRecognized, @@ -1530,6 +1530,10 @@ pub trait WalletTest: InputSource + WalletRead { &self, protocol: ShieldedProtocol, ) -> Result>, ::Error>; + + /// Optionally perform final checks at the conclusion of each test + /// Allows wallet backend developers to perform any necessary consistency checks or cleanup + fn finally(&self) {} } /// The output of a transaction sent by the wallet. @@ -1973,7 +1977,7 @@ impl SentTransactionOutput { /// A data structure used to set the birthday height for an account, and ensure that the initial /// note commitment tree state is recorded at that height. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct AccountBirthday { prior_chain_state: ChainState, recover_until: Option, @@ -2068,6 +2072,11 @@ impl AccountBirthday { self.recover_until } + /// Returns the [`ChainState`] corresponding to the last block prior to the wallet's birthday + pub fn prior_chain_state(&self) -> &ChainState { + &self.prior_chain_state + } + #[cfg(any(test, feature = "test-dependencies"))] /// Constructs a new [`AccountBirthday`] at the given network upgrade's activation, /// with no "recover until" height. diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index f05394fe4..243801f53 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -501,7 +501,7 @@ impl ScanSummary { } /// The final note commitment tree state for each shielded pool, as of a particular block height. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct ChainState { block_height: BlockHeight, block_hash: BlockHash, diff --git a/zcash_client_backend/src/data_api/testing.rs b/zcash_client_backend/src/data_api/testing.rs index 4e37ae968..cfa9abbbd 100644 --- a/zcash_client_backend/src/data_api/testing.rs +++ b/zcash_client_backend/src/data_api/testing.rs @@ -439,6 +439,12 @@ impl TestState } } +impl Drop for TestState { + fn drop(&mut self) { + self.wallet_data.finally(); + } +} + impl TestState { diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index f3bca04ba..0cb2d1eaf 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -28,7 +28,7 @@ use crate::fees::orchard as orchard_fees; use zcash_primitives::legacy::keys::{NonHardenedChildIndex, TransparentKeyScope}; /// A unique identifier for a shielded transaction output -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct NoteId { txid: TxId, protocol: ShieldedProtocol, @@ -67,7 +67,7 @@ impl NoteId { /// * for external unified addresses, the pool to which the payment is sent; /// * for ephemeral transparent addresses, the internal account ID and metadata about the outpoint; /// * for wallet-internal outputs, the internal account ID and metadata about the note. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum Recipient { External(ZcashAddress, PoolType), EphemeralTransparent { @@ -171,6 +171,7 @@ impl Recipient, O> { /// The shielded subset of a [`Transaction`]'s data that is relevant to a particular wallet. /// /// [`Transaction`]: zcash_primitives::transaction::Transaction +#[derive(Clone)] pub struct WalletTx { txid: TxId, block_index: usize, @@ -300,6 +301,7 @@ impl transparent_fees::InputView for WalletTransparentOutput { } /// A reference to a spent note belonging to the wallet within a transaction. +#[derive(Clone)] pub struct WalletSpend { index: usize, nf: Nf, @@ -339,6 +341,7 @@ pub type WalletSaplingSpend = WalletSpend = WalletSpend; /// An output that was successfully decrypted in the process of wallet scanning. +#[derive(Clone)] pub struct WalletOutput { index: usize, ephemeral_key: EphemeralKeyBytes, diff --git a/zcash_client_memory/Cargo.toml b/zcash_client_memory/Cargo.toml new file mode 100644 index 000000000..14cba2a23 --- /dev/null +++ b/zcash_client_memory/Cargo.toml @@ -0,0 +1,95 @@ +[package] +name = "zcash_client_memory" +version = "0.1.0" +repository.workspace = true +# readme = "README.md" +license.workspace = true +edition.workspace = true +rust-version.workspace = true +categories.workspace = true +build = "build.rs" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +zcash_address.workspace = true +zcash_client_backend = { workspace = true, features = ["unstable-serialization", "unstable-spanning-tree", "sync"] } +zcash_encoding.workspace = true +zcash_keys = { workspace = true, features = ["sapling"] } +zcash_primitives.workspace = true +zcash_protocol.workspace = true +zip32.workspace = true + +tokio = { workspace = true, features = ["sync"] } + +# Dependencies exposed in a public API: +# (Breaking upgrades to these require a breaking upgrade to this crate.) +# - Errors +bip32 = { workspace = true, optional = true } +bs58.workspace = true + +# - Logging and metrics +tracing.workspace = true + +# - Serialization +byteorder.workspace = true +nonempty.workspace = true +prost.workspace = true +group.workspace = true +jubjub.workspace = true + +# - Secret management +secrecy.workspace = true +subtle.workspace = true + +# - Static assertions +static_assertions.workspace = true + +# - Shielded protocols +orchard = { workspace = true, optional = true } +sapling.workspace = true + +# - Note commitment trees +incrementalmerkletree.workspace = true +shardtree = { workspace = true, features = ["legacy-api"] } +thiserror = "1.0.61" + +rayon.workspace = true +async-trait = { version = "0.1" } + +# - Test dependencies +proptest = { workspace = true, optional = true } +wasm_sync = "0.1.2" +time.workspace = true +bytes = "1.9.0" + +[dev-dependencies] +ciborium = "0.2.2" +serde_json.workspace = true +postcard = { version = "1.0.10", features = ["alloc"] } +pretty_assertions = "1.4.1" + + +[features] +default = ["multicore"] +local-consensus = ["zcash_protocol/local-consensus"] +## Enables multithreading support for creating proofs and building subtrees. +multicore = ["zcash_primitives/multicore"] + +## Enables support for storing data related to the sending and receiving of +## Orchard funds. +orchard = ["dep:orchard", "zcash_client_backend/orchard", "zcash_keys/orchard"] + +## Exposes APIs that are useful for testing, such as `proptest` strategies. +test-dependencies = ["dep:proptest", "incrementalmerkletree/test-dependencies", "shardtree/test-dependencies", "zcash_primitives/test-dependencies", "zcash_client_backend/test-dependencies", "incrementalmerkletree/test-dependencies"] + +## Enables receiving transparent funds and sending to transparent recipients +transparent-inputs = ["dep:bip32", "zcash_keys/transparent-inputs", "zcash_client_backend/transparent-inputs"] + +#! ### Experimental features + +## Exposes unstable APIs. Their behaviour may change at any time. +unstable = ["zcash_client_backend/unstable"] + +[build-dependencies] +prost-build = "0.13.3" +which = "6" diff --git a/zcash_client_memory/build.rs b/zcash_client_memory/build.rs new file mode 100644 index 000000000..df1110f31 --- /dev/null +++ b/zcash_client_memory/build.rs @@ -0,0 +1,41 @@ +use std::env; +use std::fs; +use std::io; +use std::path::PathBuf; + +fn main() -> io::Result<()> { + // - We check for the existence of protoc in the same way as prost_build, so that + // people building from source do not need to have protoc installed. If they make + // changes to the proto files, the discrepancy will be caught by CI. + if env::var_os("PROTOC") + .map(PathBuf::from) + .or_else(|| which::which("protoc").ok()) + .is_some() + { + build()?; + } + + Ok(()) +} + +fn build() -> io::Result<()> { + let out: PathBuf = env::var_os("OUT_DIR") + .expect("Cannot find OUT_DIR environment variable") + .into(); + + prost_build::compile_protos( + &[ + "src/proto/memory_wallet.proto", + "src/proto/notes.proto", + "src/proto/primitives.proto", + "src/proto/shardtree.proto", + "src/proto/transparent.proto", + ], + &["src/"], + )?; + + // Copy the generated types into the source tree so changes can be committed. + fs::copy(out.join("memwallet.rs"), "src/proto/generated.rs")?; + + Ok(()) +} diff --git a/zcash_client_memory/src/block_source.rs b/zcash_client_memory/src/block_source.rs new file mode 100644 index 000000000..0e8ac0805 --- /dev/null +++ b/zcash_client_memory/src/block_source.rs @@ -0,0 +1,112 @@ +use async_trait::async_trait; +use std::collections::BTreeMap; +use std::convert::Infallible; +use wasm_sync::RwLock; +use zcash_client_backend::data_api::chain::{BlockCache, BlockSource}; +use zcash_client_backend::data_api::scanning::ScanRange; +use zcash_client_backend::proto::compact_formats::CompactBlock; +use zcash_protocol::consensus::BlockHeight; + +/// A block cache that just holds blocks in a map in memory +#[derive(Default)] +pub struct MemBlockCache(pub(crate) RwLock>); + +impl MemBlockCache { + /// Constructs a new empty [`MemBlockCache`]. + pub fn new() -> Self { + Default::default() + } + + /// Returns the [`CompactBlock`] at the given height, if it exists in the cache. + pub fn find_block(&self, block_height: BlockHeight) -> Option { + self.0.read().unwrap().get(&block_height).cloned() + } +} + +impl BlockSource for MemBlockCache { + type Error = Infallible; + + fn with_blocks( + &self, + from_height: Option, + limit: Option, + mut with_block: F, + ) -> Result<(), zcash_client_backend::data_api::chain::error::Error> + where + F: FnMut( + CompactBlock, + ) -> Result< + (), + zcash_client_backend::data_api::chain::error::Error, + >, + { + let inner = self.0.read().unwrap(); + let block_iter = inner + .iter() + .filter(|(_, cb)| { + if let Some(from_height) = from_height { + cb.height() >= from_height + } else { + true + } + }) + .take(limit.unwrap_or(usize::MAX)); + + for (_, cb) in block_iter { + with_block(cb.clone())?; + } + Ok(()) + } +} + +#[async_trait] +impl BlockCache for MemBlockCache { + fn get_tip_height( + &self, + range: Option<&ScanRange>, + ) -> Result, Self::Error> { + let inner = self.0.read().unwrap(); + if let Some(range) = range { + let range = range.block_range(); + for h in (u32::from(range.start)..u32::from(range.end)).rev() { + if let Some(cb) = inner.get(&h.into()) { + return Ok(Some(cb.height())); + } + } + } else { + return Ok(inner.last_key_value().map(|(h, _)| *h)); + } + + Ok(None) + } + + async fn read(&self, range: &ScanRange) -> Result, Self::Error> { + let inner = self.0.read().unwrap(); + let mut ret = Vec::with_capacity(range.len()); + let range = range.block_range(); + for height in u32::from(range.start)..u32::from(range.end) { + if let Some(cb) = inner.get(&height.into()) { + ret.push(cb.clone()); + } + } + + Ok(ret) + } + + async fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error> { + let mut inner = self.0.write().unwrap(); + compact_blocks.into_iter().for_each(|compact_block| { + inner.insert(compact_block.height(), compact_block); + }); + Ok(()) + } + + async fn delete(&self, range: ScanRange) -> Result<(), Self::Error> { + let mut inner = self.0.write().unwrap(); + let range = range.block_range(); + for height in u32::from(range.start)..u32::from(range.end) { + inner.remove(&height.into()); + } + Ok(()) + } +} diff --git a/zcash_client_memory/src/error.rs b/zcash_client_memory/src/error.rs new file mode 100644 index 000000000..7bda6a2ec --- /dev/null +++ b/zcash_client_memory/src/error.rs @@ -0,0 +1,163 @@ +use std::{array::TryFromSliceError, convert::Infallible}; + +use shardtree::error::ShardTreeError; +use zcash_address::ConversionError; +use zcash_keys::{ + encoding::TransparentCodecError, + keys::{AddressGenerationError, DerivationError}, +}; +use zcash_primitives::{legacy::TransparentAddress, transaction::TxId}; +use zcash_protocol::{consensus::BlockHeight, memo}; + +use crate::AccountId; + +pub type Result = std::result::Result; + +/// Helper macro for reading optional fields from a protobuf messages +/// it will return a Result type with a custom error based on the +/// field name +#[macro_export] +macro_rules! read_optional { + ($proto:expr, $field:ident) => { + $proto + .$field + .ok_or(Error::ProtoMissingField(stringify!($field))) + }; +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Account not found: {0:?}")] + AccountUnknown(AccountId), + #[error("Account out of range.")] + AccountOutOfRange, + #[error("Address Conversion error: {0}")] + ConversionError(ConversionError<&'static str>), + #[error("Address not recognized: {0:?}")] + AddressNotRecognized(TransparentAddress), + #[error("Error generating address: {0}")] + AddressGeneration(AddressGenerationError), + #[error("Balance error: {0}")] + Balance(#[from] zcash_protocol::value::BalanceError), + #[error("An error occurred while processing an account due to a failure in deriving the account's keys: {0}")] + BadAccountData(String), + #[error("Error converting byte vec to array: {0:?}")] + ByteVecToArrayConversion(Vec), + #[error("Chain height unknown")] + ChainHeightUnknown, + #[error("Conflicting Tx Locator map entry")] + ConflictingTxLocator, + #[error("Corrupted Data: {0}")] + CorruptedData(String), + #[error("Error deriving key: {0}")] + KeyDerivation(DerivationError), + #[error("Failed to convert between integer types")] + IntegerConversion(#[from] std::num::TryFromIntError), + #[error("Infallible")] + Infallible(#[from] Infallible), + #[error("Invalid scan range start {0}, end {1}: {2}")] + InvalidScanRange(BlockHeight, BlockHeight, String), + #[error("Seed must be between 32 and 252 bytes in length.")] + InvalidSeedLength, + #[error("Io Error: {0}")] + Io(std::io::Error), + #[error("Memo decryption failed: {0}")] + MemoDecryption(memo::Error), + #[error("Expected field missing: {0}")] + Missing(String), + #[error("Note not found")] + NoteNotFound, + #[error("Blocks are non sequental")] + NonSequentialBlocks, + #[error("Orchard specific code was called without the 'orchard' feature enabled")] + OrchardNotEnabled, + #[error("Other error: {0}")] + Other(String), + #[error("Proto Decoding Error: {0}")] + ProtoDecodingError(#[from] prost::DecodeError), + #[error("Proto Encoding Error: {0}")] + ProtoEncodingError(#[from] prost::EncodeError), + #[error("Missing proto field: {0}")] + ProtoMissingField(&'static str), + #[error( + "Requested rewind to invalid block height. Safe height: {0:?}, requested height {1:?}" + )] + RequestedRewindInvalid(Option, BlockHeight), + #[cfg(feature = "transparent-inputs")] + #[error("Requested gap limit {1} reached for account {0:?}")] + ReachedGapLimit(AccountId, u32), + #[error("ShardTree error: {0}")] + ShardTree(ShardTreeError), + #[error("String Conversion error: {0}")] + StringConversion(#[from] std::string::FromUtf8Error), + #[error("Transaction not in table: {0}")] + TransactionNotFound(TxId), + #[error("Error converting transparent address: {0}")] + TransparentCodec(#[from] TransparentCodecError), + #[cfg(feature = "transparent-inputs")] + #[error("Transparent derivation: {0}")] + TransparentDerivation(bip32::Error), + #[error("Unsupported proto version: {1} (expected {0})")] + UnsupportedProtoVersion(u32, u32), + #[error("Error converting nullifier from slice: {0}")] + NullifierFromSlice(#[from] TryFromSliceError), + #[error("Error decoding ufvk string: {0}")] + UfvkDecodeError(String), + #[error("Viewing key not found for account: {0:?}")] + ViewingKeyNotFound(AccountId), + #[error("Error parsing zcash address: {0}")] + ParseZcashAddress(#[from] zcash_address::ParseError), + #[error("Unknown zip32 derivation error")] + UnknownZip32Derivation, + + #[error("Error converting int to zip32: {0}")] + Zip32FromInt(#[from] zip32::TryFromIntError), +} + +#[cfg(feature = "transparent-inputs")] +impl From for Error { + fn from(value: bip32::Error) -> Self { + Error::TransparentDerivation(value) + } +} +impl From> for Error { + fn from(value: ConversionError<&'static str>) -> Self { + Error::ConversionError(value) + } +} + +impl From for Error { + fn from(value: DerivationError) -> Self { + Error::KeyDerivation(value) + } +} + +impl From for Error { + fn from(value: AddressGenerationError) -> Self { + Error::AddressGeneration(value) + } +} + +impl From for Error { + fn from(value: memo::Error) -> Self { + Error::MemoDecryption(value) + } +} + +impl From for Error { + fn from(value: std::io::Error) -> Self { + Error::Io(value) + } +} + +impl From> for Error { + fn from(value: ShardTreeError) -> Self { + Error::ShardTree(value) + } +} + +impl From> for Error { + fn from(value: Vec) -> Self { + Error::ByteVecToArrayConversion(value) + } +} diff --git a/zcash_client_memory/src/input_source.rs b/zcash_client_memory/src/input_source.rs new file mode 100644 index 000000000..5700a2c70 --- /dev/null +++ b/zcash_client_memory/src/input_source.rs @@ -0,0 +1,345 @@ +use zcash_client_backend::{ + data_api::{AccountMeta, InputSource, NoteFilter, PoolMeta, TransactionStatus, WalletRead}, + wallet::NoteId, +}; +use zcash_primitives::transaction::components::OutPoint; +#[cfg(feature = "orchard")] +use zcash_protocol::ShieldedProtocol::Orchard; +use zcash_protocol::{ + consensus, consensus::BlockHeight, value::Zatoshis, ShieldedProtocol, ShieldedProtocol::Sapling, +}; +#[cfg(feature = "transparent-inputs")] +use { + zcash_client_backend::wallet::WalletTransparentOutput, + zcash_primitives::legacy::TransparentAddress, +}; + +use crate::{error::Error, to_spendable_notes, AccountId, MemoryWalletDb}; + +impl InputSource for MemoryWalletDb

{ + type Error = crate::error::Error; + type AccountId = AccountId; + type NoteRef = NoteId; + + /// Find the note with the given index (output index for Sapling, action index for Orchard) + /// that belongs to the given transaction + fn get_spendable_note( + &self, + txid: &zcash_primitives::transaction::TxId, + protocol: zcash_protocol::ShieldedProtocol, + index: u32, + ) -> Result< + Option< + zcash_client_backend::wallet::ReceivedNote< + Self::NoteRef, + zcash_client_backend::wallet::Note, + >, + >, + Self::Error, + > { + let note = self.received_notes.iter().find(|rn| { + &rn.txid == txid && rn.note.protocol() == protocol && rn.output_index == index + }); + + Ok(if let Some(note) = note { + if self.note_is_spent(note, 0)? { + None + } else { + Some(zcash_client_backend::wallet::ReceivedNote::from_parts( + note.note_id, + *txid, + index.try_into().unwrap(), // this overflow can never happen or else the chain is broken + note.note.clone(), + note.recipient_key_scope + .ok_or(Error::Missing("recipient key scope".into()))?, + note.commitment_tree_position + .ok_or(Error::Missing("commitment tree position".into()))?, + )) + } + } else { + None + }) + } + + fn select_spendable_notes( + &self, + account: Self::AccountId, + target_value: zcash_protocol::value::Zatoshis, + sources: &[zcash_protocol::ShieldedProtocol], + anchor_height: zcash_protocol::consensus::BlockHeight, + exclude: &[Self::NoteRef], + ) -> Result, Self::Error> { + let sapling_eligible_notes = if sources.contains(&Sapling) { + self.select_spendable_notes_from_pool( + account, + target_value, + &Sapling, + anchor_height, + exclude, + )? + } else { + Vec::new() + }; + + #[cfg(feature = "orchard")] + let orchard_eligible_notes = if sources.contains(&Orchard) { + self.select_spendable_notes_from_pool( + account, + target_value, + &Orchard, + anchor_height, + exclude, + )? + } else { + Vec::new() + }; + + to_spendable_notes( + &sapling_eligible_notes, + #[cfg(feature = "orchard")] + &orchard_eligible_notes, + ) + } + + /// Returns the list of spendable transparent outputs received by this wallet at `address` + /// such that, at height `target_height`: + /// * the transaction that produced the output had or will have at least `min_confirmations` + /// confirmations; and + /// * the output is unspent as of the current chain tip. + /// + /// An output that is potentially spent by an unmined transaction in the mempool is excluded + /// iff the spending transaction will not be expired at `target_height`. + #[cfg(feature = "transparent-inputs")] + fn get_spendable_transparent_outputs( + &self, + address: &TransparentAddress, + target_height: BlockHeight, + min_confirmations: u32, + ) -> Result, Self::Error> { + let txos = self + .transparent_received_outputs + .iter() + .filter(|(_, txo)| txo.address == *address) + .map(|(outpoint, txo)| (outpoint, txo, self.tx_table.get(&txo.transaction_id))) + .filter(|(outpoint, _, _)| { + self.utxo_is_spendable(outpoint, target_height, min_confirmations) + .unwrap_or(false) + }) + .filter_map(|(outpoint, txo, tx)| { + txo.to_wallet_transparent_output(outpoint, tx.and_then(|tx| tx.mined_height())) + }) + .collect(); + Ok(txos) + } + + /// Fetches the transparent output corresponding to the provided `outpoint`. + /// + /// Returns `Ok(None)` if the UTXO is not known to belong to the wallet or is not + /// spendable as of the chain tip height. + #[cfg(feature = "transparent-inputs")] + fn get_unspent_transparent_output( + &self, + outpoint: &OutPoint, + ) -> Result, Self::Error> { + Ok(self + .transparent_received_outputs + .get(outpoint) + .map(|txo| (txo, self.tx_table.get(&txo.transaction_id))) + .and_then(|(txo, tx)| { + txo.to_wallet_transparent_output(outpoint, tx.and_then(|tx| tx.mined_height())) + })) + } + + /// Returns metadata for the spendable notes in the wallet. + fn get_account_metadata( + &self, + account_id: Self::AccountId, + selector: &NoteFilter, + exclude: &[Self::NoteRef], + ) -> Result { + let chain_tip_height = self.chain_height()?.ok_or(Error::ChainHeightUnknown)?; + + let sapling_pool_meta = self.spendable_notes_meta( + ShieldedProtocol::Sapling, + chain_tip_height, + account_id, + selector, + exclude, + )?; + + #[cfg(feature = "orchard")] + let orchard_pool_meta = self.spendable_notes_meta( + ShieldedProtocol::Orchard, + chain_tip_height, + account_id, + selector, + exclude, + )?; + #[cfg(not(feature = "orchard"))] + let orchard_pool_meta = None; + + Ok(AccountMeta::new(sapling_pool_meta, orchard_pool_meta)) + } +} + +impl MemoryWalletDb

{ + // Select the spendable notes to cover the given target value considering only a single pool + // Returns the notes sorted oldest to newest + fn select_spendable_notes_from_pool( + &self, + account: AccountId, + target_value: Zatoshis, + pool: &zcash_protocol::ShieldedProtocol, + anchor_height: consensus::BlockHeight, + exclude: &[NoteId], + ) -> Result, Error> { + let birthday_height = match self.get_wallet_birthday()? { + Some(birthday) => birthday, + None => { + // the wallet birthday can only be unknown if there are no accounts in the wallet; in + // such a case, the wallet has no notes to spend. + return Ok(Vec::new()); + } + }; + // First grab all eligible (unspent, spendable, fully scanned) notes into a vec. + let mut eligible_notes = self + .received_notes + .iter() + .filter(|note| note.account_id == account) + .filter(|note| note.note.protocol() == *pool) + .filter(|note| { + self.note_is_spendable(note, birthday_height, anchor_height, exclude) + .unwrap() + }) + .collect::>(); + + // sort by oldest first (use location in commitment tree since this gives a total order) + eligible_notes.sort_by(|a, b| a.commitment_tree_position.cmp(&b.commitment_tree_position)); + + // now take notes until we have enough to cover the target value + let mut value_acc = Zatoshis::ZERO; + let selection: Vec<_> = eligible_notes + .into_iter() + .take_while(|note| { + let take = value_acc <= target_value; + value_acc = (value_acc + note.note.value()).expect("value overflow"); + take + }) + .collect(); + + Ok(selection) + } + + pub(crate) fn utxo_is_spendable( + &self, + outpoint: &OutPoint, + target_height: BlockHeight, + min_confirmations: u32, + ) -> Result { + let confirmed_height = target_height - min_confirmations; + let utxo = self + .transparent_received_outputs + .get(outpoint) + .ok_or(Error::NoteNotFound)?; + if let Some(tx) = self.tx_table.get(&utxo.transaction_id) { + Ok( + tx.is_mined_or_unexpired_at(confirmed_height) // tx that created it is mined + && !self.utxo_is_spent(outpoint, min_confirmations)?, // not spent + ) + } else { + Ok(false) + } + } + + fn utxo_is_spent(&self, outpoint: &OutPoint, min_confirmations: u32) -> Result { + let spend = self.transparent_received_output_spends.get(outpoint); + + let spent = match spend { + Some(txid) => { + let spending_tx = self + .tx_table + .get(txid) + .ok_or_else(|| Error::TransactionNotFound(*txid))?; + match spending_tx.status() { + TransactionStatus::Mined(_height) => true, + TransactionStatus::TxidNotRecognized => unreachable!(), + TransactionStatus::NotInMainChain => { + // check the expiry + spending_tx.expiry_height().is_none() // no expiry, tx could be mined any time so we consider it spent + // expiry is in the future so it could still be mined + || spending_tx.expiry_height() > self.summary_height(min_confirmations)? + } + } + } + None => false, + }; + Ok(spent) + } + + fn spendable_notes_meta( + &self, + protocol: ShieldedProtocol, + chain_tip_height: BlockHeight, + account: AccountId, + filter: &NoteFilter, + exclude: &[NoteId], + ) -> Result, Error> { + let birthday_height = match self.get_wallet_birthday()? { + Some(birthday) => birthday, + None => { + return Ok(None); + } + }; + let (count, total) = self + .received_notes + .iter() + .filter(|note| note.account_id == account) + .filter(|note| note.note.protocol() == protocol) + .filter(|note| { + self.note_is_spendable(note, birthday_height, chain_tip_height, exclude) + .unwrap() + }) + .filter(|note| { + self.matches_note_filter(note, filter) + .unwrap() + .is_some_and(|b| b) + }) + .fold((0, Zatoshis::ZERO), |(count, total), note| { + (count + 1, (total + note.note.value()).unwrap()) + }); + + Ok(Some(PoolMeta::new(count, total))) + } + + #[allow(clippy::only_used_in_recursion)] + fn matches_note_filter( + &self, + note: &crate::ReceivedNote, + filter: &NoteFilter, + ) -> Result, Error> { + match filter { + NoteFilter::ExceedsMinValue(min_value) => Ok(Some(note.note.value() > *min_value)), + NoteFilter::ExceedsPriorSendPercentile(_n) => todo!(), + NoteFilter::ExceedsBalancePercentage(_p) => todo!(), + // evaluate both conditions. + // If one cannot be evaluated (e.g. it returns None) it is ignored + NoteFilter::Combine(a, b) => { + let matches_a = self.matches_note_filter(note, a)?.unwrap_or(true); + let matches_b = self.matches_note_filter(note, b)?.unwrap_or(true); + Ok(Some(matches_a && matches_b)) + } + // Evaluate the first condition and return the result. + // If the first condition cannot be evaluated then use the fallback instead + NoteFilter::Attempt { + condition, + fallback, + } => { + if let Some(b) = self.matches_note_filter(note, condition)? { + Ok(Some(b)) + } else { + self.matches_note_filter(note, fallback) + } + } + } + } +} diff --git a/zcash_client_memory/src/lib.rs b/zcash_client_memory/src/lib.rs new file mode 100644 index 000000000..8e9724e0a --- /dev/null +++ b/zcash_client_memory/src/lib.rs @@ -0,0 +1,23 @@ +mod block_source; +mod error; +mod input_source; +pub mod proto; +mod types; +mod wallet_commitment_trees; +mod wallet_read; +mod wallet_write; + +#[cfg(test)] +pub mod testing; +pub use block_source::*; +pub use error::Error; +pub use types::MemoryWalletDb; +pub(crate) use types::*; + +/// The maximum number of blocks the wallet is allowed to rewind. This is +/// consistent with the bound in zcashd, and allows block data deeper than +/// this delta from the chain tip to be pruned. +pub(crate) const PRUNING_DEPTH: u32 = 100; + +/// The number of blocks to verify ahead when the chain tip is updated. +pub(crate) const VERIFY_LOOKAHEAD: u32 = 10; diff --git a/zcash_client_memory/src/proto.rs b/zcash_client_memory/src/proto.rs new file mode 100644 index 000000000..ff291d21a --- /dev/null +++ b/zcash_client_memory/src/proto.rs @@ -0,0 +1,2 @@ +pub mod generated; +pub use generated as memwallet; diff --git a/zcash_client_memory/src/proto/generated.rs b/zcash_client_memory/src/proto/generated.rs new file mode 100644 index 000000000..08b65e4d8 --- /dev/null +++ b/zcash_client_memory/src/proto/generated.rs @@ -0,0 +1,748 @@ +// This file is @generated by prost-build. +/// Unique identifier for a zcash transaction +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxId { + #[prost(bytes = "vec", tag = "1")] + pub hash: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Address { + #[prost(bytes = "vec", tag = "1")] + pub diversifier_index: ::prost::alloc::vec::Vec, + #[prost(string, tag = "2")] + pub address: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NoteId { + #[prost(message, optional, tag = "1")] + pub tx_id: ::core::option::Option, + #[prost(enumeration = "PoolType", tag = "2")] + pub pool: i32, + #[prost(uint32, tag = "3")] + pub output_index: u32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Memo { + #[prost(message, optional, tag = "1")] + pub note_id: ::core::option::Option, + #[prost(bytes = "vec", tag = "2")] + pub memo: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Nullifier { + #[prost(enumeration = "ShieldedProtocol", tag = "1")] + pub protocol: i32, + #[prost(bytes = "vec", tag = "2")] + pub nullifier: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OutPoint { + #[prost(bytes = "vec", tag = "1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(uint32, tag = "2")] + pub n: u32, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum PoolType { + Transparent = 0, + ShieldedSapling = 1, + ShieldedOrchard = 2, +} +impl PoolType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Transparent => "Transparent", + Self::ShieldedSapling => "ShieldedSapling", + Self::ShieldedOrchard => "ShieldedOrchard", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Transparent" => Some(Self::Transparent), + "ShieldedSapling" => Some(Self::ShieldedSapling), + "ShieldedOrchard" => Some(Self::ShieldedOrchard), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ShieldedProtocol { + Sapling = 0, + Orchard = 1, +} +impl ShieldedProtocol { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Sapling => "sapling", + Self::Orchard => "orchard", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "sapling" => Some(Self::Sapling), + "orchard" => Some(Self::Orchard), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum TransactionStatus { + TxidNotRecognized = 0, + NotInMainChain = 1, + Mined = 2, +} +impl TransactionStatus { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::TxidNotRecognized => "TxidNotRecognized", + Self::NotInMainChain => "NotInMainChain", + Self::Mined => "Mined", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TxidNotRecognized" => Some(Self::TxidNotRecognized), + "NotInMainChain" => Some(Self::NotInMainChain), + "Mined" => Some(Self::Mined), + _ => None, + } + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Note { + #[prost(enumeration = "ShieldedProtocol", tag = "1")] + pub protocol: i32, + #[prost(bytes = "vec", tag = "2")] + pub recipient: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub value: u64, + #[prost(bytes = "vec", optional, tag = "4")] + pub rho: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(message, optional, tag = "5")] + pub rseed: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RSeed { + #[prost(enumeration = "RSeedType", optional, tag = "1")] + pub rseed_type: ::core::option::Option, + #[prost(bytes = "vec", tag = "2")] + pub payload: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReceivedNote { + #[prost(message, optional, tag = "1")] + pub note_id: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub tx_id: ::core::option::Option, + #[prost(uint32, tag = "3")] + pub output_index: u32, + #[prost(uint32, tag = "4")] + pub account_id: u32, + #[prost(message, optional, tag = "5")] + pub note: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub nullifier: ::core::option::Option, + #[prost(bool, tag = "7")] + pub is_change: bool, + #[prost(bytes = "vec", tag = "8")] + pub memo: ::prost::alloc::vec::Vec, + #[prost(uint64, optional, tag = "9")] + pub commitment_tree_position: ::core::option::Option, + #[prost(enumeration = "Scope", optional, tag = "10")] + pub recipient_key_scope: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SentNote { + #[prost(uint32, tag = "1")] + pub from_account_id: u32, + #[prost(message, optional, tag = "2")] + pub to: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub value: u64, + #[prost(bytes = "vec", tag = "4")] + pub memo: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Recipient { + #[prost(enumeration = "RecipientType", tag = "1")] + pub recipient_type: i32, + /// either the zcash address if external or transparent address if EphemeralTransparent + #[prost(string, optional, tag = "2")] + pub address: ::core::option::Option<::prost::alloc::string::String>, + /// the shielded protocol if External + #[prost(enumeration = "PoolType", optional, tag = "3")] + pub pool_type: ::core::option::Option, + /// the account id if EphemeralTransparent or InternalAccount + #[prost(uint32, optional, tag = "4")] + pub account_id: ::core::option::Option, + /// the outpoint metadata if InternalAccount + #[prost(message, optional, tag = "5")] + pub outpoint_metadata: ::core::option::Option, + /// the note if InternalAccount + #[prost(message, optional, tag = "6")] + pub note: ::core::option::Option, +} +/// associates a note and a transaction where it was spent +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReceivedNoteSpendRecord { + #[prost(message, optional, tag = "1")] + pub note_id: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub tx_id: ::core::option::Option, +} +/// records where a nullifier was spent by block height and tx index in that block +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NullifierRecord { + #[prost(message, optional, tag = "1")] + pub nullifier: ::core::option::Option, + #[prost(uint32, tag = "2")] + pub block_height: u32, + #[prost(uint32, tag = "3")] + pub tx_index: u32, +} +/// Record storing the sent information for a given note +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SentNoteRecord { + #[prost(message, optional, tag = "1")] + pub sent_note_id: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub sent_note: ::core::option::Option, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RSeedType { + BeforeZip212 = 0, + AfterZip212 = 1, +} +impl RSeedType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::BeforeZip212 => "BeforeZip212", + Self::AfterZip212 => "AfterZip212", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "BeforeZip212" => Some(Self::BeforeZip212), + "AfterZip212" => Some(Self::AfterZip212), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Scope { + Internal = 0, + External = 1, +} +impl Scope { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Internal => "Internal", + Self::External => "External", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Internal" => Some(Self::Internal), + "External" => Some(Self::External), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RecipientType { + ExternalRecipient = 0, + EphemeralTransparent = 1, + InternalAccount = 2, +} +impl RecipientType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::ExternalRecipient => "ExternalRecipient", + Self::EphemeralTransparent => "EphemeralTransparent", + Self::InternalAccount => "InternalAccount", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "ExternalRecipient" => Some(Self::ExternalRecipient), + "EphemeralTransparent" => Some(Self::EphemeralTransparent), + "InternalAccount" => Some(Self::InternalAccount), + _ => None, + } + } +} +/// A shard tree defined by a cap subtree and shard subtrees +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ShardTree { + #[prost(bytes = "vec", tag = "1")] + pub cap: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "2")] + pub shards: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub checkpoints: ::prost::alloc::vec::Vec, +} +/// A shard in a shard tree +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TreeShard { + #[prost(uint64, tag = "1")] + pub shard_index: u64, + #[prost(bytes = "vec", tag = "3")] + pub shard_data: ::prost::alloc::vec::Vec, +} +/// A checkpoint in a shard tree +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct TreeCheckpoint { + #[prost(uint32, tag = "1")] + pub checkpoint_id: u32, + #[prost(uint64, tag = "2")] + pub position: u64, +} +/// Stores the block height corresponding to the last note commitment in a shard +/// as defined by its level and index in the tree +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct TreeEndHeightsRecord { + #[prost(uint32, tag = "1")] + pub level: u32, + #[prost(uint64, tag = "2")] + pub index: u64, + #[prost(uint32, tag = "3")] + pub block_height: u32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReceivedTransparentOutput { + #[prost(bytes = "vec", tag = "1")] + pub transaction_id: ::prost::alloc::vec::Vec, + #[prost(uint32, tag = "2")] + pub account_id: u32, + #[prost(string, tag = "3")] + pub address: ::prost::alloc::string::String, + #[prost(message, optional, tag = "4")] + pub txout: ::core::option::Option, + #[prost(uint32, optional, tag = "5")] + pub max_observed_unspent_height: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxOut { + #[prost(uint64, tag = "1")] + pub value: u64, + #[prost(bytes = "vec", tag = "2")] + pub script: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransparentReceivedOutputRecord { + #[prost(message, optional, tag = "1")] + pub outpoint: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub output: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransparentReceivedOutputSpendRecord { + #[prost(message, optional, tag = "1")] + pub outpoint: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub tx_id: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransparentSpendCacheRecord { + #[prost(message, optional, tag = "1")] + pub tx_id: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub outpoint: ::core::option::Option, +} +/// A serialized zcash wallet state +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MemoryWallet { + /// the version of the wallet serialization format + #[prost(uint32, tag = "1")] + pub version: u32, + /// the accounts in this wallet + #[prost(message, optional, tag = "2")] + pub accounts: ::core::option::Option, + /// map from block height to block data + #[prost(message, repeated, tag = "3")] + pub blocks: ::prost::alloc::vec::Vec, + /// map from transaction id to transaction data + #[prost(message, repeated, tag = "4")] + pub tx_table: ::prost::alloc::vec::Vec, + /// the notes received by this wallet + #[prost(message, repeated, tag = "5")] + pub received_note_table: ::prost::alloc::vec::Vec, + /// the notes spent by this wallet + #[prost(message, repeated, tag = "6")] + pub received_note_spends: ::prost::alloc::vec::Vec, + /// the nullifiers for notes spent by this wallet + #[prost(message, repeated, tag = "7")] + pub nullifiers: ::prost::alloc::vec::Vec, + /// the notes sent by this wallet + #[prost(message, repeated, tag = "8")] + pub sent_notes: ::prost::alloc::vec::Vec, + /// map between txIds and their inclusion in blocks + #[prost(message, repeated, tag = "9")] + pub tx_locator: ::prost::alloc::vec::Vec, + /// the scan queue (which blocks the wallet should scan next and with what priority) + #[prost(message, repeated, tag = "10")] + pub scan_queue: ::prost::alloc::vec::Vec, + /// Sapling shielded pool shard tree + #[prost(message, optional, tag = "11")] + pub sapling_tree: ::core::option::Option, + /// the block heights corresponding to the last note commitment for each shard in the sapling tree + #[prost(message, repeated, tag = "12")] + pub sapling_tree_shard_end_heights: ::prost::alloc::vec::Vec, + /// Orchard shielded pool shard tree + #[prost(message, optional, tag = "13")] + pub orchard_tree: ::core::option::Option, + /// the block heights corresponding to the last note commitment for each shard in the orchard tree + #[prost(message, repeated, tag = "14")] + pub orchard_tree_shard_end_heights: ::prost::alloc::vec::Vec, + /// UTXOs known to this wallet + #[prost(message, repeated, tag = "15")] + pub transparent_received_outputs: ::prost::alloc::vec::Vec, + /// UTXOs spent by this wallet + #[prost(message, repeated, tag = "16")] + pub transparent_received_output_spends: + ::prost::alloc::vec::Vec, + /// Map from spends to their location in the blockchain + #[prost(message, repeated, tag = "17")] + pub transparent_spend_map: ::prost::alloc::vec::Vec, + /// Queue of transaction data requests the wallet should make to the lightwalletd provided to obtain more complete information + #[prost(message, repeated, tag = "18")] + pub transaction_data_requests: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Accounts { + /// map from account index to account data + #[prost(message, repeated, tag = "1")] + pub accounts: ::prost::alloc::vec::Vec, + /// the nonce for the next account + #[prost(uint32, tag = "2")] + pub account_nonce: u32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Account { + /// the index of this account + #[prost(uint32, tag = "1")] + pub account_id: u32, + /// derived or imported + #[prost(enumeration = "AccountKind", tag = "2")] + pub kind: i32, + #[prost(bytes = "vec", optional, tag = "3")] + pub seed_fingerprint: ::core::option::Option<::prost::alloc::vec::Vec>, + /// HD index to derive account from seed + #[prost(uint32, optional, tag = "5")] + pub account_index: ::core::option::Option, + /// spending or view-only + #[prost(enumeration = "AccountPurpose", optional, tag = "6")] + pub purpose: ::core::option::Option, + /// the viewing key for this account + #[prost(string, tag = "7")] + pub viewing_key: ::prost::alloc::string::String, + /// the block height at which this account was created + #[prost(message, optional, tag = "8")] + pub birthday: ::core::option::Option, + /// account addresses + #[prost(message, repeated, tag = "9")] + pub addresses: ::prost::alloc::vec::Vec

, + /// map from index to encoded unified address + #[prost(message, repeated, tag = "10")] + pub ephemeral_addresses: ::prost::alloc::vec::Vec, + /// human readable name for the account + #[prost(string, tag = "11")] + pub account_name: ::prost::alloc::string::String, + /// key source metadata + #[prost(string, optional, tag = "12")] + pub key_source: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AccountBirthday { + /// the chain state at the block height before the account was created + #[prost(message, optional, tag = "1")] + pub prior_chain_state: ::core::option::Option, + /// the block height until which the account should stop being in recovery mode + #[prost(uint32, optional, tag = "2")] + pub recover_until: ::core::option::Option, +} +/// A record storing transaction data in the transaction table +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionTableRecord { + #[prost(message, optional, tag = "1")] + pub tx_id: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub tx_entry: ::core::option::Option, +} +/// Maps a block height and transaction index to a transaction ID. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxLocatorRecord { + #[prost(uint32, tag = "1")] + pub block_height: u32, + #[prost(uint32, tag = "2")] + pub tx_index: u32, + #[prost(message, optional, tag = "3")] + pub tx_id: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EphemeralAddress { + #[prost(string, tag = "1")] + pub address: ::prost::alloc::string::String, + #[prost(bytes = "vec", optional, tag = "2")] + pub used_in_tx: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(bytes = "vec", optional, tag = "3")] + pub seen_in_tx: ::core::option::Option<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EphemeralAddressRecord { + #[prost(uint32, tag = "1")] + pub index: u32, + #[prost(message, optional, tag = "2")] + pub ephemeral_address: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChainState { + /// the height of this block + #[prost(uint32, tag = "1")] + pub block_height: u32, + #[prost(bytes = "vec", tag = "2")] + pub block_hash: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "3")] + pub final_sapling_tree: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "4")] + pub final_orchard_tree: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WalletBlock { + /// the height of this block + #[prost(uint32, tag = "1")] + pub height: u32, + /// the ID (hash) of this block, same as in block explorers + #[prost(bytes = "vec", tag = "2")] + pub hash: ::prost::alloc::vec::Vec, + /// Unix epoch time when the block was mined + #[prost(uint32, tag = "3")] + pub block_time: u32, + /// the txids of transactions in this block + #[prost(bytes = "vec", repeated, tag = "4")] + pub transactions: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// map from note id to memo + #[prost(message, repeated, tag = "5")] + pub memos: ::prost::alloc::vec::Vec, + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, optional, tag = "6")] + pub sapling_commitment_tree_size: ::core::option::Option, + /// the number of Sapling outputs in this block + #[prost(uint32, optional, tag = "7")] + pub sapling_output_count: ::core::option::Option, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, optional, tag = "8")] + pub orchard_commitment_tree_size: ::core::option::Option, + /// the number of Orchard actions in this block + #[prost(uint32, optional, tag = "9")] + pub orchard_action_count: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionEntry { + #[prost(enumeration = "TransactionStatus", tag = "1")] + pub tx_status: i32, + #[prost(uint32, optional, tag = "2")] + pub block: ::core::option::Option, + #[prost(uint32, optional, tag = "3")] + pub tx_index: ::core::option::Option, + #[prost(uint32, optional, tag = "4")] + pub expiry_height: ::core::option::Option, + #[prost(bytes = "vec", optional, tag = "5")] + pub raw_tx: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(uint64, optional, tag = "6")] + pub fee: ::core::option::Option, + #[prost(uint32, optional, tag = "7")] + pub target_height: ::core::option::Option, + #[prost(uint32, optional, tag = "8")] + pub mined_height: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionDataRequest { + #[prost(enumeration = "TransactionDataRequestType", tag = "1")] + pub request_type: i32, + /// for the GetStatus and Enhancement variants + #[prost(message, optional, tag = "2")] + pub tx_id: ::core::option::Option, + /// for the SpendsFromAddress variant + #[prost(bytes = "vec", optional, tag = "3")] + pub address: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(uint32, optional, tag = "4")] + pub block_range_start: ::core::option::Option, + #[prost(uint32, optional, tag = "5")] + pub block_range_end: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct ScanQueueRecord { + #[prost(uint32, tag = "1")] + pub start_height: u32, + #[prost(uint32, tag = "2")] + pub end_height: u32, + #[prost(enumeration = "ScanPriority", tag = "3")] + pub priority: i32, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum AccountKind { + Derived = 0, + Imported = 1, +} +impl AccountKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Derived => "Derived", + Self::Imported => "Imported", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Derived" => Some(Self::Derived), + "Imported" => Some(Self::Imported), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum AccountPurpose { + Spending = 0, + ViewOnly = 1, +} +impl AccountPurpose { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Spending => "Spending", + Self::ViewOnly => "ViewOnly", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Spending" => Some(Self::Spending), + "ViewOnly" => Some(Self::ViewOnly), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum TransactionDataRequestType { + GetStatus = 0, + Enhancement = 1, + SpendsFromAddress = 2, +} +impl TransactionDataRequestType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::GetStatus => "GetStatus", + Self::Enhancement => "Enhancement", + Self::SpendsFromAddress => "SpendsFromAddress", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "GetStatus" => Some(Self::GetStatus), + "Enhancement" => Some(Self::Enhancement), + "SpendsFromAddress" => Some(Self::SpendsFromAddress), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ScanPriority { + /// / Block ranges that are ignored have lowest priority. + Ignored = 0, + /// / Block ranges that have already been scanned will not be re-scanned. + Scanned = 1, + /// / Block ranges to be scanned to advance the fully-scanned height. + Historic = 2, + /// / Block ranges adjacent to heights at which the user opened the wallet. + OpenAdjacent = 3, + /// / Blocks that must be scanned to complete note commitment tree shards adjacent to found notes. + FoundNote = 4, + /// / Blocks that must be scanned to complete the latest note commitment tree shard. + ChainTip = 5, + /// / A previously scanned range that must be verified to check it is still in the + /// / main chain, has highest priority. + Verify = 6, +} +impl ScanPriority { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Ignored => "Ignored", + Self::Scanned => "Scanned", + Self::Historic => "Historic", + Self::OpenAdjacent => "OpenAdjacent", + Self::FoundNote => "FoundNote", + Self::ChainTip => "ChainTip", + Self::Verify => "Verify", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Ignored" => Some(Self::Ignored), + "Scanned" => Some(Self::Scanned), + "Historic" => Some(Self::Historic), + "OpenAdjacent" => Some(Self::OpenAdjacent), + "FoundNote" => Some(Self::FoundNote), + "ChainTip" => Some(Self::ChainTip), + "Verify" => Some(Self::Verify), + _ => None, + } + } +} diff --git a/zcash_client_memory/src/proto/memory_wallet.proto b/zcash_client_memory/src/proto/memory_wallet.proto new file mode 100644 index 000000000..f0a8ca2cb --- /dev/null +++ b/zcash_client_memory/src/proto/memory_wallet.proto @@ -0,0 +1,186 @@ +syntax = "proto3"; + +package memwallet; + +import "proto/notes.proto"; +import "proto/primitives.proto"; +import "proto/shardtree.proto"; +import "proto/transparent.proto"; + +// A serialized zcash wallet state +message MemoryWallet { + // the version of the wallet serialization format + uint32 version = 1; + // the accounts in this wallet + Accounts accounts = 2; + + // map from block height to block data + repeated WalletBlock blocks = 3; + // map from transaction id to transaction data + repeated TransactionTableRecord tx_table = 4; + // the notes received by this wallet + repeated ReceivedNote received_note_table = 5; + // the notes spent by this wallet + repeated ReceivedNoteSpendRecord received_note_spends = 6; + // the nullifiers for notes spent by this wallet + repeated NullifierRecord nullifiers = 7; + // the notes sent by this wallet + repeated SentNoteRecord sent_notes = 8; + + // map between txIds and their inclusion in blocks + repeated TxLocatorRecord tx_locator = 9; + // the scan queue (which blocks the wallet should scan next and with what priority) + repeated ScanQueueRecord scan_queue = 10; + + // Sapling shielded pool shard tree + ShardTree sapling_tree = 11; + // the block heights corresponding to the last note commitment for each shard in the sapling tree + repeated TreeEndHeightsRecord sapling_tree_shard_end_heights = 12; + + // Orchard shielded pool shard tree + ShardTree orchard_tree = 13; + // the block heights corresponding to the last note commitment for each shard in the orchard tree + repeated TreeEndHeightsRecord orchard_tree_shard_end_heights = 14; + + // UTXOs known to this wallet + repeated TransparentReceivedOutputRecord transparent_received_outputs = 15; + // UTXOs spent by this wallet + repeated TransparentReceivedOutputSpendRecord transparent_received_output_spends = 16; + // Map from spends to their location in the blockchain + repeated TransparentSpendCacheRecord transparent_spend_map = 17; + // Queue of transaction data requests the wallet should make to the lightwalletd provided to obtain more complete information + repeated TransactionDataRequest transaction_data_requests = 18; +} + +message Accounts { + repeated Account accounts = 1; // map from account index to account data + uint32 account_nonce = 2; // the nonce for the next account +} + +message Account { + uint32 account_id = 1; // the index of this account + AccountKind kind = 2; // derived or imported + + optional bytes seed_fingerprint = 3; + optional uint32 account_index = 5; // HD index to derive account from seed + + optional AccountPurpose purpose = 6; // spending or view-only + + string viewing_key = 7; // the viewing key for this account + AccountBirthday birthday = 8; // the block height at which this account was created + repeated Address addresses = 9; // account addresses + + repeated EphemeralAddressRecord ephemeral_addresses = 10; // map from index to encoded unified address + string account_name = 11; // human readable name for the account + optional string key_source = 12; // key source metadata +} + +enum AccountKind { + Derived = 0; + Imported = 1; +} + +enum AccountPurpose { + Spending = 0; + ViewOnly = 1; +} + +message AccountBirthday { + ChainState prior_chain_state = 1; // the chain state at the block height before the account was created + optional uint32 recover_until = 2; // the block height until which the account should stop being in recovery mode +} + +// A record storing transaction data in the transaction table +message TransactionTableRecord { + TxId tx_id = 1; + TransactionEntry tx_entry = 2; +} + +// Maps a block height and transaction index to a transaction ID. +message TxLocatorRecord { + uint32 block_height = 1; + uint32 tx_index = 2; + TxId tx_id = 3; +} + +message EphemeralAddress { + string address = 1; + optional bytes used_in_tx = 2; + optional bytes seen_in_tx = 3; +} + +message EphemeralAddressRecord { + uint32 index = 1; + EphemeralAddress ephemeral_address = 2; +} + +message ChainState { + uint32 block_height = 1; // the height of this block + bytes block_hash = 2; + bytes final_sapling_tree = 3; + bytes final_orchard_tree = 4; +} + +message WalletBlock { + uint32 height = 1; // the height of this block + bytes hash = 2; // the ID (hash) of this block, same as in block explorers + uint32 block_time = 3; // Unix epoch time when the block was mined + repeated bytes transactions = 4; // the txids of transactions in this block + repeated Memo memos = 5; // map from note id to memo + optional uint32 sapling_commitment_tree_size = 6; // the size of the Sapling note commitment tree as of the end of this block + optional uint32 sapling_output_count = 7; // the number of Sapling outputs in this block + optional uint32 orchard_commitment_tree_size = 8; // the size of the Orchard note commitment tree as of the end of this block + optional uint32 orchard_action_count = 9; // the number of Orchard actions in this block +} + +message TransactionEntry { + TransactionStatus tx_status = 1; + optional uint32 block = 2; + optional uint32 tx_index = 3; + optional uint32 expiry_height = 4; + optional bytes raw_tx = 5; + optional uint64 fee = 6; + optional uint32 target_height = 7; + optional uint32 mined_height = 8; +} + +message TransactionDataRequest { + TransactionDataRequestType request_type = 1; + // for the GetStatus and Enhancement variants + optional TxId tx_id = 2; + + // for the SpendsFromAddress variant + optional bytes address = 3; + optional uint32 block_range_start = 4; + optional uint32 block_range_end = 5; +} + +enum TransactionDataRequestType { + GetStatus = 0; + Enhancement = 1; + SpendsFromAddress = 2; +} + +message ScanQueueRecord { + uint32 start_height = 1; + uint32 end_height = 2; + ScanPriority priority = 3; +} + +enum ScanPriority { + /// Block ranges that are ignored have lowest priority. + Ignored = 0; + /// Block ranges that have already been scanned will not be re-scanned. + Scanned = 1; + /// Block ranges to be scanned to advance the fully-scanned height. + Historic = 2; + /// Block ranges adjacent to heights at which the user opened the wallet. + OpenAdjacent = 3; + /// Blocks that must be scanned to complete note commitment tree shards adjacent to found notes. + FoundNote = 4; + /// Blocks that must be scanned to complete the latest note commitment tree shard. + ChainTip = 5; + /// A previously scanned range that must be verified to check it is still in the + /// main chain, has highest priority. + Verify = 6; +} diff --git a/zcash_client_memory/src/proto/notes.proto b/zcash_client_memory/src/proto/notes.proto new file mode 100644 index 000000000..3d8c48553 --- /dev/null +++ b/zcash_client_memory/src/proto/notes.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package memwallet; + +import "proto/primitives.proto"; + +message Note { + ShieldedProtocol protocol = 1; + bytes recipient = 2; + uint64 value = 3; + optional bytes rho = 4; + RSeed rseed = 5; +} + +message RSeed { + optional RSeedType rseed_type = 1; + bytes payload = 2; +} + +enum RSeedType { + BeforeZip212 = 0; + AfterZip212 = 1; +} + +message ReceivedNote { + NoteId note_id = 1; + TxId tx_id = 2; + uint32 output_index = 3; + uint32 account_id = 4; + Note note = 5; + optional Nullifier nullifier = 6; + bool is_change = 7; + bytes memo = 8; + optional uint64 commitment_tree_position = 9; + optional Scope recipient_key_scope = 10; +} + +enum Scope { + Internal = 0; + External = 1; +} + +message SentNote { + uint32 from_account_id = 1; + Recipient to = 2; + uint64 value = 3; + bytes memo = 4; +} + +message Recipient { + RecipientType recipient_type = 1; + + optional string address = 2; // either the zcash address if external or transparent address if EphemeralTransparent + optional PoolType pool_type = 3; // the shielded protocol if External + optional uint32 account_id = 4; // the account id if EphemeralTransparent or InternalAccount + optional OutPoint outpoint_metadata = 5; // the outpoint metadata if InternalAccount + optional Note note = 6; // the note if InternalAccount +} + +enum RecipientType { + ExternalRecipient = 0; + EphemeralTransparent = 1; + InternalAccount = 2; +} + +// associates a note and a transaction where it was spent +message ReceivedNoteSpendRecord { + NoteId note_id = 1; + TxId tx_id = 2; +} + +// records where a nullifier was spent by block height and tx index in that block +message NullifierRecord { + Nullifier nullifier = 1; + uint32 block_height = 2; + uint32 tx_index = 3; +} + +// Record storing the sent information for a given note +message SentNoteRecord { + NoteId sent_note_id = 1; + SentNote sent_note = 2; +} diff --git a/zcash_client_memory/src/proto/primitives.proto b/zcash_client_memory/src/proto/primitives.proto new file mode 100644 index 000000000..80922cbff --- /dev/null +++ b/zcash_client_memory/src/proto/primitives.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +package memwallet; + +// Unique identifier for a zcash transaction +message TxId { + bytes hash = 1; +} + +message Address { + bytes diversifier_index = 1; + string address = 2; +} + +message NoteId { + TxId tx_id = 1; + PoolType pool = 2; + uint32 output_index = 3; +} + +enum PoolType { + Transparent = 0; + ShieldedSapling = 1; + ShieldedOrchard = 2; +} + +enum ShieldedProtocol { + sapling = 0; + orchard = 1; +} + +message Memo { + NoteId note_id = 1; + bytes memo = 2; +} + +message Nullifier { + ShieldedProtocol protocol = 1; + bytes nullifier = 2; +} + +enum TransactionStatus { + TxidNotRecognized = 0; + NotInMainChain = 1; + Mined = 2; +} + +message OutPoint { + bytes hash = 1; + uint32 n = 2; +} diff --git a/zcash_client_memory/src/proto/shardtree.proto b/zcash_client_memory/src/proto/shardtree.proto new file mode 100644 index 000000000..ce3724cf2 --- /dev/null +++ b/zcash_client_memory/src/proto/shardtree.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package memwallet; + +// A shard tree defined by a cap subtree and shard subtrees +message ShardTree { + bytes cap = 1; + repeated TreeShard shards = 2; + repeated TreeCheckpoint checkpoints = 3; +} + +// A shard in a shard tree +message TreeShard { + uint64 shard_index = 1; + bytes shard_data = 3; +} + +// A checkpoint in a shard tree +message TreeCheckpoint { + uint32 checkpoint_id = 1; + uint64 position = 2; +} + +// Stores the block height corresponding to the last note commitment in a shard +// as defined by its level and index in the tree +message TreeEndHeightsRecord { + uint32 level = 1; + uint64 index = 2; + uint32 block_height = 3; +} diff --git a/zcash_client_memory/src/proto/transparent.proto b/zcash_client_memory/src/proto/transparent.proto new file mode 100644 index 000000000..35a39a0ec --- /dev/null +++ b/zcash_client_memory/src/proto/transparent.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package memwallet; + +import "proto/primitives.proto"; + +message ReceivedTransparentOutput { + bytes transaction_id = 1; + uint32 account_id = 2; + string address = 3; + TxOut txout = 4; + optional uint32 max_observed_unspent_height = 5; +} + +message TxOut { + uint64 value = 1; + bytes script = 2; +} + +message TransparentReceivedOutputRecord { + OutPoint outpoint = 1; + ReceivedTransparentOutput output = 2; +} + +message TransparentReceivedOutputSpendRecord { + OutPoint outpoint = 1; + TxId tx_id = 2; +} + +message TransparentSpendCacheRecord { + TxId tx_id = 1; + OutPoint outpoint = 2; +} diff --git a/zcash_client_memory/src/testing/mod.rs b/zcash_client_memory/src/testing/mod.rs new file mode 100644 index 000000000..7c44b0911 --- /dev/null +++ b/zcash_client_memory/src/testing/mod.rs @@ -0,0 +1,406 @@ +use std::convert::{identity, Infallible}; +use std::fmt::Debug; + +use zcash_client_backend::data_api::InputSource; +use zcash_client_backend::data_api::OutputOfSentTx; +use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; +use zcash_client_backend::wallet::Note; +use zcash_client_backend::wallet::Recipient; +use zcash_client_backend::wallet::WalletTransparentOutput; +use zcash_client_backend::{ + data_api::{ + testing::{DataStoreFactory, Reset, TestCache, TestState}, + WalletRead, WalletTest, + }, + proto::compact_formats::CompactBlock, +}; +use zcash_keys::address::Address; +use zcash_primitives::transaction::components::amount::NonNegativeAmount; +use zcash_protocol::value::ZatBalance; +use zcash_protocol::ShieldedProtocol; + +use shardtree::store::ShardStore; +use zcash_client_backend::wallet::NoteId; +use zcash_client_backend::wallet::ReceivedNote; + +use zcash_primitives::transaction::TxId; +use zcash_protocol::consensus::BlockHeight; +use zcash_protocol::local_consensus::LocalNetwork; + +use crate::{Account, AccountId, Error, MemBlockCache, MemoryWalletDb, SentNoteId}; + +pub mod pool; + +#[cfg(test)] +#[cfg(feature = "transparent-inputs")] +mod transparent; + +/// A test data store factory for in-memory databases +/// Very simple implementation just creates a new MemoryWalletDb +pub(crate) struct TestMemDbFactory; + +impl TestMemDbFactory { + pub(crate) fn new() -> Self { + Self + } +} + +impl DataStoreFactory for TestMemDbFactory { + type Error = (); + type AccountId = AccountId; + type Account = Account; + type DsError = Error; + type DataStore = MemoryWalletDb; + + fn new_data_store(&self, network: LocalNetwork) -> Result { + Ok(MemoryWalletDb::new(network, 100)) + } +} + +impl TestCache for MemBlockCache { + type BsError = Infallible; + type BlockSource = MemBlockCache; + type InsertResult = (); + + fn block_source(&self) -> &Self::BlockSource { + self + } + + fn insert(&mut self, cb: &CompactBlock) -> Self::InsertResult { + self.0.write().unwrap().insert(cb.height(), cb.clone()); + } + + fn truncate_to_height(&mut self, height: BlockHeight) { + self.0.write().unwrap().retain(|k, _| *k <= height); + } +} + +impl

Reset for MemoryWalletDb

+where + P: zcash_primitives::consensus::Parameters + Clone + Debug + PartialEq, +{ + type Handle = (); + + fn reset(st: &mut TestState) -> Self::Handle { + let new_wallet = MemoryWalletDb::new(st.wallet().params.clone(), 100); + let _ = std::mem::replace(st.wallet_mut(), new_wallet); + } +} + +impl

WalletTest for MemoryWalletDb

+where + P: zcash_primitives::consensus::Parameters + Clone + Debug + PartialEq, +{ + #[allow(clippy::type_complexity)] + fn get_sent_outputs(&self, txid: &TxId) -> Result, Error> { + self + .sent_notes + .iter() + .filter(|(note_id, _)| note_id.txid() == txid) + .map(|(_, note)| match note.to.clone() { + Recipient::External(zcash_address, _) => Ok(( + note.value.into_u64(), + Some( + Address::try_from_zcash_address(&self.params, zcash_address) + .map_err(Error::from)?, + ), + None, + )), + Recipient::EphemeralTransparent { + ephemeral_address, + receiving_account, + .. + } => { + #[cfg(feature = "transparent-inputs")] + { + let account = self.get_account(receiving_account)?.unwrap(); + let (_addr, meta) = account + .ephemeral_addresses()? + .into_iter() + .find(|(addr, _)| addr == &ephemeral_address) + .unwrap(); + Ok(( + // TODO: Use the ephemeral address index to look up the address + // and find the correct index + note.value.into_u64(), + Some(Address::from(ephemeral_address)), + Some(( + Address::from(ephemeral_address), + meta.address_index().index(), + )), + )) + } + #[cfg(not(feature = "transparent-inputs"))] + { + unimplemented!("EphemeralTransparent recipients are not supported without the `transparent-inputs` feature.") + } + } + Recipient::InternalAccount { .. } => Ok((note.value.into_u64(), None, None)), + }) + .map(|res: Result<_, Error>| { + let (amount, external_recipient, ephemeral_address) = res?; + Ok::<_, ::Error>(OutputOfSentTx::from_parts( + NonNegativeAmount::from_u64(amount)?, + external_recipient, + ephemeral_address, + )) + }) + .collect::>() + } + + /// Fetches the transparent output corresponding to the provided `outpoint`. + /// Allows selecting unspendable outputs for testing purposes. + /// + /// Returns `Ok(None)` if the UTXO is not known to belong to the wallet or is not + /// spendable as of the chain tip height. + #[cfg(feature = "transparent-inputs")] + fn get_transparent_output( + &self, + outpoint: &zcash_primitives::transaction::components::OutPoint, + _allow_unspendable: bool, + ) -> Result, ::Error> { + Ok(self + .transparent_received_outputs + .get(outpoint) + .map(|txo| (txo, self.tx_table.get(&txo.transaction_id))) + .and_then(|(txo, tx)| { + txo.to_wallet_transparent_output(outpoint, tx.and_then(|tx| tx.mined_height())) + })) + } + + fn get_notes( + &self, + protocol: zcash_protocol::ShieldedProtocol, + ) -> Result>, Error> { + Ok(self + .received_notes + .iter() + .filter(|rn| rn.note.protocol() == protocol) + .cloned() + .map(Into::into) + .collect()) + } + + /// Returns the note IDs for shielded notes sent by the wallet in a particular + /// transaction. + fn get_sent_note_ids( + &self, + txid: &TxId, + protocol: ShieldedProtocol, + ) -> Result, Error> { + Ok(self + .get_sent_notes() + .iter() + .filter_map(|(id, _)| { + if let SentNoteId::Shielded(id) = id { + if id.txid() == txid && id.protocol() == protocol { + Some(*id) + } else { + None + } + } else { + None + } + }) + .collect()) + } + + /// Returns a vector of transaction summaries. + /// + /// Currently test-only, as production use could return a very large number of results; either + /// pagination or a streaming design will be necessary to stabilize this feature for production + /// use.⁄ + fn get_tx_history( + &self, + ) -> Result>, Error> + { + let mut history = self + .tx_table + .iter() + .map(|(txid, tx)| { + // find all the notes associated with this transaction + // A transaction may send and/or receive one or more notes + + // notes spent (consumed) by the transaction + let spent_notes = self + .received_note_spends + .iter() + .filter(|(_, spend_txid)| *spend_txid == txid) + .collect::>(); + + let spent_utxos = self + .transparent_received_output_spends + .iter() + .filter(|(_, spend_txid)| *spend_txid == txid) + .collect::>(); + + // notes produced (sent) by the transaction (excluding change) + let sent_notes = self + .sent_notes + .iter() + .filter(|(note_id, _)| note_id.txid() == txid) + .filter(|(note_id, _)| { + // use a join on the received notes table to detect which are change + self.received_notes.iter().any(|received_note| { + SentNoteId::from(received_note.note_id) == **note_id + && !received_note.is_change + }) + }) + .collect::>(); + + let received_txo = self + .transparent_received_outputs + .iter() + .filter(|(outpoint, _received_output)| outpoint.txid() == txid) + .collect::>(); + + let sent_txo_value: u64 = received_txo + .iter() + .map(|(_, o)| u64::from(o.txout.value)) + .sum(); + + // notes received by the transaction + let received_notes = self + .received_notes + .iter() + .filter(|received_note| received_note.txid() == *txid) + .collect::>(); + + // A transaction can send and receive notes to/from multiple accounts + // For a transaction to be visible to this wallet it must have either scanned it from the chain + // or been created by this wallet so there are number of ways we can detect the account ID + let receiving_account_id = received_notes.first().map(|note| note.account_id()); + let sending_account_id = sent_notes.first().map(|(_, note)| note.from_account_id); + let receiving_transparent_account_id = received_txo + .first() + .map(|(_, received)| received.account_id); + let sent_txo_account_id = spent_utxos.first().and_then(|(outpoint, _)| { + // any spent txo was first a received txo + self.transparent_received_outputs + .get(outpoint) + .map(|txo| txo.account_id) + }); + + // take the first non-none account_id + let account_id = vec![ + receiving_account_id, + sending_account_id, + receiving_transparent_account_id, + sent_txo_account_id, + ] + .into_iter() + .find_map(identity) + .ok_or(Error::Other( + format!("Account id could not be found for tx: {}", txid).to_string(), + ))?; + + let balance_gained: u64 = received_notes + .iter() + .map(|note| note.note.value().into_u64()) + .sum::() + + sent_txo_value; + + let balance_lost: u64 = self // includes change + .sent_notes + .iter() + .filter(|(note_id, _)| note_id.txid() == txid) + .map(|(_, sent_note)| sent_note.value.into_u64()) + .sum::() + + tx.fee().map(u64::from).unwrap_or(0); + + let is_shielding = { + //All of the wallet-spent and wallet-received notes are consistent with a shielding transaction. + // e.g. only transparent outputs are spend and only shielded notes are received + spent_notes.is_empty() && !spent_utxos.is_empty() + // The transaction contains at least one wallet-received note. + && !received_notes.is_empty() + // We do not know about any external outputs of the transaction. + && sent_notes.is_empty() + }; + + let has_change = received_notes.iter().any(|note| note.is_change); + + Ok( + zcash_client_backend::data_api::testing::TransactionSummary::from_parts( + account_id, // account_id + *txid, // txid + tx.expiry_height(), // expiry_height + tx.mined_height(), // mined_height + ZatBalance::const_from_i64((balance_gained as i64) - (balance_lost as i64)), // account_value_delta + tx.fee(), // fee_paid + spent_notes.len() + spent_utxos.len(), // spent_note_count + has_change, // has_change + sent_notes.len(), // sent_note_count (excluding change) + received_notes.iter().filter(|note| !note.is_change).count(), // received_note_count (excluding change) + 0, // Unimplemented: memo_count + false, // Unimplemented: expired_unmined + is_shielding, // is_shielding + ), + ) + }) + .collect::, Error>>()?; + history.sort_by(|a, b| { + b.mined_height() + .cmp(&a.mined_height()) + .then(b.txid().cmp(&a.txid())) + }); + Ok(history) + } + + fn get_checkpoint_history( + &self, + protocol: &ShieldedProtocol, + ) -> Result)>, Error> { + let mut checkpoints = Vec::new(); + + match protocol { + ShieldedProtocol::Sapling => { + self.sapling_tree + .store() + .for_each_checkpoint(usize::MAX, |id, cp| { + checkpoints.push((*id, cp.position())); + Ok(()) + })?; + } + #[cfg(feature = "orchard")] + ShieldedProtocol::Orchard => { + self.orchard_tree + .store() + .for_each_checkpoint(usize::MAX, |id, cp| { + checkpoints.push((*id, cp.position())); + Ok(()) + })?; + } + #[cfg(not(feature = "orchard"))] + _ => {} + } + + checkpoints.sort_by(|(a, _), (b, _)| a.cmp(b)); + + Ok(checkpoints) + } + + fn finally(&self) { + // ensure the wallet state at the conclusion of each test can be round-tripped through serialization + let proto = crate::proto::memwallet::MemoryWallet::from(self); + let recovered_wallet = + MemoryWalletDb::new_from_proto(proto.clone(), self.params.clone(), 100).unwrap(); + + assert_eq!(self, &recovered_wallet); + + // ensure the trees can be roundtripped + use crate::wallet_commitment_trees::serialization::{tree_from_protobuf, tree_to_protobuf}; + + let tree_proto = tree_to_protobuf(&self.sapling_tree).unwrap().unwrap(); + let recovered_tree: shardtree::ShardTree< + shardtree::store::memory::MemoryShardStore, + { SAPLING_SHARD_HEIGHT * 2 }, + SAPLING_SHARD_HEIGHT, + > = tree_from_protobuf(tree_proto, 100, 16.into()).unwrap(); + + assert_eq!( + self.sapling_tree.store().get_shard_roots(), + recovered_tree.store().get_shard_roots() + ); + } +} diff --git a/zcash_client_memory/src/testing/pool.rs b/zcash_client_memory/src/testing/pool.rs new file mode 100644 index 000000000..3a80e2cc2 --- /dev/null +++ b/zcash_client_memory/src/testing/pool.rs @@ -0,0 +1,217 @@ +use zcash_client_backend::data_api::testing::pool::ShieldedPoolTester; + +use crate::testing::{MemBlockCache, TestMemDbFactory}; + +#[cfg(test)] +mod sapling; + +#[cfg(test)] +#[cfg(feature = "orchard")] +mod orchard; + +pub(crate) fn send_single_step_proposed_transfer() { + zcash_client_backend::data_api::testing::pool::send_single_step_proposed_transfer::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn send_multi_step_proposed_transfer() { + zcash_client_backend::data_api::testing::pool::send_multi_step_proposed_transfer::( + TestMemDbFactory::new(), + MemBlockCache::new(), + |e, account_id, expected_bad_index| { + matches!( + e, + crate::Error::ReachedGapLimit(acct, bad_index) + if acct == &account_id && bad_index == &expected_bad_index) + }, + ) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { + zcash_client_backend::data_api::testing::pool::proposal_fails_if_not_all_ephemeral_outputs_consumed::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[allow(deprecated)] +pub(crate) fn create_to_address_fails_on_incorrect_usk() { + zcash_client_backend::data_api::testing::pool::create_to_address_fails_on_incorrect_usk::( + TestMemDbFactory::new(), + ) +} + +#[allow(deprecated)] +pub(crate) fn proposal_fails_with_no_blocks() { + zcash_client_backend::data_api::testing::pool::proposal_fails_with_no_blocks::( + TestMemDbFactory::new(), + ) +} + +pub(crate) fn spend_fails_on_unverified_notes() { + zcash_client_backend::data_api::testing::pool::spend_fails_on_unverified_notes::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn spend_fails_on_locked_notes() { + zcash_client_backend::data_api::testing::pool::spend_fails_on_locked_notes::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn ovk_policy_prevents_recovery_from_chain() { + zcash_client_backend::data_api::testing::pool::ovk_policy_prevents_recovery_from_chain::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn spend_succeeds_to_t_addr_zero_change() { + zcash_client_backend::data_api::testing::pool::spend_succeeds_to_t_addr_zero_change::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn change_note_spends_succeed() { + zcash_client_backend::data_api::testing::pool::change_note_spends_succeed::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +// TODO: Implement reset for memdb +pub(crate) fn external_address_change_spends_detected_in_restore_from_seed< + T: ShieldedPoolTester, +>() { + zcash_client_backend::data_api::testing::pool::external_address_change_spends_detected_in_restore_from_seed::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[allow(dead_code)] +pub(crate) fn zip317_spend() { + zcash_client_backend::data_api::testing::pool::zip317_spend::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn shield_transparent() { + zcash_client_backend::data_api::testing::pool::shield_transparent::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn birthday_in_anchor_shard() { + zcash_client_backend::data_api::testing::pool::birthday_in_anchor_shard::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn checkpoint_gaps() { + zcash_client_backend::data_api::testing::pool::checkpoint_gaps::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(feature = "orchard")] +pub(crate) fn pool_crossing_required() { + zcash_client_backend::data_api::testing::pool::pool_crossing_required::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(feature = "orchard")] +pub(crate) fn fully_funded_fully_private() { + zcash_client_backend::data_api::testing::pool::fully_funded_fully_private::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(all(feature = "orchard", feature = "transparent-inputs"))] +pub(crate) fn fully_funded_send_to_t() { + zcash_client_backend::data_api::testing::pool::fully_funded_send_to_t::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(feature = "orchard")] +pub(crate) fn multi_pool_checkpoint() { + zcash_client_backend::data_api::testing::pool::multi_pool_checkpoint::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +#[cfg(feature = "orchard")] +pub(crate) fn multi_pool_checkpoints_with_pruning() { + zcash_client_backend::data_api::testing::pool::multi_pool_checkpoints_with_pruning::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn valid_chain_states() { + zcash_client_backend::data_api::testing::pool::valid_chain_states::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn invalid_chain_cache_disconnected() { + zcash_client_backend::data_api::testing::pool::invalid_chain_cache_disconnected::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn data_db_truncation() { + zcash_client_backend::data_api::testing::pool::data_db_truncation::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn scan_cached_blocks_allows_blocks_out_of_order() { + zcash_client_backend::data_api::testing::pool::scan_cached_blocks_allows_blocks_out_of_order::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn scan_cached_blocks_finds_received_notes() { + zcash_client_backend::data_api::testing::pool::scan_cached_blocks_finds_received_notes::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn scan_cached_blocks_finds_change_notes() { + zcash_client_backend::data_api::testing::pool::scan_cached_blocks_finds_change_notes::( + TestMemDbFactory::new(), + MemBlockCache::new(), + ) +} + +pub(crate) fn scan_cached_blocks_detects_spends_out_of_order() { + zcash_client_backend::data_api::testing::pool::scan_cached_blocks_detects_spends_out_of_order::< + T, + _, + >(TestMemDbFactory::new(), MemBlockCache::new()) +} diff --git a/zcash_client_memory/src/testing/pool/orchard.rs b/zcash_client_memory/src/testing/pool/orchard.rs new file mode 100644 index 000000000..5ef3470c2 --- /dev/null +++ b/zcash_client_memory/src/testing/pool/orchard.rs @@ -0,0 +1,152 @@ +use crate::testing; + +use zcash_client_backend::data_api::testing::orchard::OrchardPoolTester; +use zcash_client_backend::data_api::testing::sapling::SaplingPoolTester; + +#[test] +fn send_single_step_proposed_transfer() { + testing::pool::send_single_step_proposed_transfer::() +} + +#[test] +#[cfg(feature = "transparent-inputs")] +fn send_multi_step_proposed_transfer() { + testing::pool::send_multi_step_proposed_transfer::() +} + +#[test] +#[cfg(feature = "transparent-inputs")] +fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { + testing::pool::proposal_fails_if_not_all_ephemeral_outputs_consumed::() +} + +#[test] +#[allow(deprecated)] +fn create_to_address_fails_on_incorrect_usk() { + testing::pool::create_to_address_fails_on_incorrect_usk::() +} + +#[test] +#[allow(deprecated)] +fn proposal_fails_with_no_blocks() { + testing::pool::proposal_fails_with_no_blocks::() +} + +#[test] +fn spend_fails_on_unverified_notes() { + testing::pool::spend_fails_on_unverified_notes::() +} + +#[test] +fn spend_fails_on_locked_notes() { + testing::pool::spend_fails_on_locked_notes::() +} + +#[test] +fn ovk_policy_prevents_recovery_from_chain() { + testing::pool::ovk_policy_prevents_recovery_from_chain::() +} + +#[test] +fn spend_succeeds_to_t_addr_zero_change() { + testing::pool::spend_succeeds_to_t_addr_zero_change::() +} + +#[test] +fn change_note_spends_succeed() { + testing::pool::change_note_spends_succeed::() +} + +#[test] +fn external_address_change_spends_detected_in_restore_from_seed() { + testing::pool::external_address_change_spends_detected_in_restore_from_seed::( + ) +} + +#[test] +#[ignore] // FIXME: #1316 This requires support for dust outputs. +#[cfg(not(feature = "expensive-tests"))] +fn zip317_spend() { + testing::pool::zip317_spend::() +} + +#[test] +#[cfg(feature = "transparent-inputs")] +fn shield_transparent() { + testing::pool::shield_transparent::() +} + +#[test] +fn birthday_in_anchor_shard() { + testing::pool::birthday_in_anchor_shard::() +} + +#[test] +fn checkpoint_gaps() { + testing::pool::checkpoint_gaps::() +} + +#[test] +#[cfg(feature = "orchard")] +fn pool_crossing_required() { + testing::pool::pool_crossing_required::() +} + +#[test] +#[cfg(feature = "orchard")] +fn fully_funded_fully_private() { + testing::pool::fully_funded_fully_private::() +} + +#[test] +#[cfg(all(feature = "orchard", feature = "transparent-inputs"))] +fn fully_funded_send_to_t() { + testing::pool::fully_funded_send_to_t::() +} + +#[test] +#[cfg(feature = "orchard")] +fn multi_pool_checkpoint() { + testing::pool::multi_pool_checkpoint::() +} + +#[test] +#[cfg(feature = "orchard")] +fn multi_pool_checkpoints_with_pruning() { + testing::pool::multi_pool_checkpoints_with_pruning::() +} + +#[test] +fn valid_chain_states() { + testing::pool::valid_chain_states::() +} + +#[test] +fn invalid_chain_cache_disconnected() { + testing::pool::invalid_chain_cache_disconnected::() +} + +#[test] +fn data_db_truncation() { + testing::pool::data_db_truncation::() +} + +#[test] +fn scan_cached_blocks_allows_blocks_out_of_order() { + testing::pool::scan_cached_blocks_allows_blocks_out_of_order::() +} + +#[test] +fn scan_cached_blocks_finds_received_notes() { + testing::pool::scan_cached_blocks_finds_received_notes::() +} + +#[test] +fn scan_cached_blocks_finds_change_notes() { + testing::pool::scan_cached_blocks_finds_change_notes::() +} + +#[test] +fn scan_cached_blocks_detects_spends_out_of_order() { + testing::pool::scan_cached_blocks_detects_spends_out_of_order::() +} diff --git a/zcash_client_memory/src/testing/pool/sapling.rs b/zcash_client_memory/src/testing/pool/sapling.rs new file mode 100644 index 000000000..7b40461db --- /dev/null +++ b/zcash_client_memory/src/testing/pool/sapling.rs @@ -0,0 +1,153 @@ +use crate::testing; + +#[cfg(feature = "orchard")] +use zcash_client_backend::data_api::testing::orchard::OrchardPoolTester; +use zcash_client_backend::data_api::testing::sapling::SaplingPoolTester; + +#[test] +fn send_single_step_proposed_transfer() { + testing::pool::send_single_step_proposed_transfer::() +} + +#[test] +#[cfg(feature = "transparent-inputs")] +fn send_multi_step_proposed_transfer() { + testing::pool::send_multi_step_proposed_transfer::() +} + +#[test] +#[cfg(feature = "transparent-inputs")] +fn proposal_fails_if_not_all_ephemeral_outputs_consumed() { + testing::pool::proposal_fails_if_not_all_ephemeral_outputs_consumed::() +} + +#[test] +#[allow(deprecated)] +fn create_to_address_fails_on_incorrect_usk() { + testing::pool::create_to_address_fails_on_incorrect_usk::() +} + +#[test] +#[allow(deprecated)] +fn proposal_fails_with_no_blocks() { + testing::pool::proposal_fails_with_no_blocks::() +} + +#[test] +fn spend_fails_on_unverified_notes() { + testing::pool::spend_fails_on_unverified_notes::() +} + +#[test] +fn spend_fails_on_locked_notes() { + testing::pool::spend_fails_on_locked_notes::() +} + +#[test] +fn ovk_policy_prevents_recovery_from_chain() { + testing::pool::ovk_policy_prevents_recovery_from_chain::() +} + +#[test] +fn spend_succeeds_to_t_addr_zero_change() { + testing::pool::spend_succeeds_to_t_addr_zero_change::() +} + +#[test] +fn change_note_spends_succeed() { + testing::pool::change_note_spends_succeed::() +} + +#[test] +fn external_address_change_spends_detected_in_restore_from_seed() { + testing::pool::external_address_change_spends_detected_in_restore_from_seed::( + ) +} + +#[test] +#[ignore] // FIXME: #1316 This requires support for dust outputs. +#[cfg(not(feature = "expensive-tests"))] +fn zip317_spend() { + testing::pool::zip317_spend::() +} + +#[test] +#[cfg(feature = "transparent-inputs")] +fn shield_transparent() { + testing::pool::shield_transparent::() +} + +#[test] +fn birthday_in_anchor_shard() { + testing::pool::birthday_in_anchor_shard::() +} + +#[test] +fn checkpoint_gaps() { + testing::pool::checkpoint_gaps::() +} + +#[test] +#[cfg(feature = "orchard")] +fn pool_crossing_required() { + testing::pool::pool_crossing_required::() +} + +#[test] +#[cfg(feature = "orchard")] +fn fully_funded_fully_private() { + testing::pool::fully_funded_fully_private::() +} + +#[test] +#[cfg(all(feature = "orchard", feature = "transparent-inputs"))] +fn fully_funded_send_to_t() { + testing::pool::fully_funded_send_to_t::() +} + +#[test] +#[cfg(feature = "orchard")] +fn multi_pool_checkpoint() { + testing::pool::multi_pool_checkpoint::() +} + +#[test] +#[cfg(feature = "orchard")] +fn multi_pool_checkpoints_with_pruning() { + testing::pool::multi_pool_checkpoints_with_pruning::() +} + +#[test] +fn valid_chain_states() { + testing::pool::valid_chain_states::() +} + +#[test] +fn invalid_chain_cache_disconnected() { + testing::pool::invalid_chain_cache_disconnected::() +} + +#[test] +fn data_db_truncation() { + testing::pool::data_db_truncation::() +} + +#[test] +fn scan_cached_blocks_allows_blocks_out_of_order() { + testing::pool::scan_cached_blocks_allows_blocks_out_of_order::() +} + +#[test] +fn scan_cached_blocks_finds_received_notes() { + testing::pool::scan_cached_blocks_finds_received_notes::() +} + +#[test] +fn scan_cached_blocks_detects_change_notes() { + testing::pool::scan_cached_blocks_finds_change_notes::() +} + +#[test] +fn scan_cached_blocks_detects_spends_out_of_order() { + testing::pool::scan_cached_blocks_detects_spends_out_of_order::() +} diff --git a/zcash_client_memory/src/testing/transparent.rs b/zcash_client_memory/src/testing/transparent.rs new file mode 100644 index 000000000..4e69c37ea --- /dev/null +++ b/zcash_client_memory/src/testing/transparent.rs @@ -0,0 +1,16 @@ +use crate::testing::{MemBlockCache, TestMemDbFactory}; + +#[test] +fn put_received_transparent_utxo() { + zcash_client_backend::data_api::testing::transparent::put_received_transparent_utxo( + TestMemDbFactory::new(), + ); +} + +#[test] +fn transparent_balance_across_shielding() { + zcash_client_backend::data_api::testing::transparent::transparent_balance_across_shielding( + TestMemDbFactory::new(), + MemBlockCache::new(), + ); +} diff --git a/zcash_client_memory/src/types/account.rs b/zcash_client_memory/src/types/account.rs new file mode 100644 index 000000000..e747a4c54 --- /dev/null +++ b/zcash_client_memory/src/types/account.rs @@ -0,0 +1,780 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::{Deref, DerefMut}, +}; + +use subtle::ConditionallySelectable; +use zcash_address::ZcashAddress; +#[cfg(feature = "transparent-inputs")] +use zcash_client_backend::wallet::TransparentAddressMetadata; +use zcash_client_backend::{ + address::UnifiedAddress, + data_api::{Account as _, AccountBirthday, AccountPurpose, AccountSource, GAP_LIMIT}, + keys::{UnifiedAddressRequest, UnifiedFullViewingKey}, + wallet::NoteId, +}; +use zcash_keys::{ + address::Receiver, + keys::{AddressGenerationError, UnifiedIncomingViewingKey}, +}; +#[cfg(feature = "transparent-inputs")] +use zcash_primitives::legacy::keys::{ + AccountPubKey, EphemeralIvk, IncomingViewingKey, NonHardenedChildIndex, TransparentKeyScope, +}; +use zcash_primitives::{legacy::TransparentAddress, transaction::TxId}; +use zcash_protocol::consensus::NetworkType; +use zip32::DiversifierIndex; + +use crate::error::Error; + +/// Internal representation of ID type for accounts. Will be unique for each account. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] +pub struct AccountId(u32); + +impl From for AccountId { + fn from(id: u32) -> Self { + AccountId(id) + } +} + +impl Deref for AccountId { + type Target = u32; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl ConditionallySelectable for AccountId { + fn conditional_select(a: &Self, b: &Self, choice: subtle::Choice) -> Self { + AccountId(ConditionallySelectable::conditional_select( + &a.0, &b.0, choice, + )) + } +} + +/// This is the top-level struct that handles accounts. We could theoretically have this just be a Vec +/// but we want to have control over the internal AccountId values. The account ids are unique. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Accounts { + pub(crate) nonce: u32, + pub(crate) accounts: BTreeMap, +} + +impl Accounts { + pub(crate) fn new() -> Self { + Self { + nonce: 0, + accounts: BTreeMap::new(), + } + } + + /// Creates a new account. The account id will be determined by the internal nonce. + /// Do not call this directly, use the `Wallet` methods instead. + /// Otherwise the scan queue will not be correctly updated + pub(crate) fn new_account( + &mut self, + account_name: &str, + kind: AccountSource, + viewing_key: UnifiedFullViewingKey, + birthday: AccountBirthday, + ) -> Result<(AccountId, Account), Error> { + self.nonce += 1; + let account_id = AccountId(self.nonce); + + let acc = Account::new( + account_name.to_string(), + account_id, + kind, + viewing_key, + birthday, + )?; + + self.accounts.insert(account_id, acc.clone()); + + Ok((account_id, acc)) + } + + pub(crate) fn get(&self, account_id: AccountId) -> Option<&Account> { + self.accounts.get(&account_id) + } + + pub(crate) fn get_mut(&mut self, account_id: AccountId) -> Option<&mut Account> { + self.accounts.get_mut(&account_id) + } + /// Gets the account ids of all accounts + pub(crate) fn account_ids(&self) -> impl Iterator { + self.accounts.keys() + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn find_account_for_transparent_address( + &self, + address: &TransparentAddress, + ) -> Result, Error> { + // Look for transparent receivers generated as part of a Unified Address + if let Some(id) = self + .accounts + .iter() + .find(|(_, account)| { + account + .addresses() + .iter() + .any(|(_, unified_address)| unified_address.transparent() == Some(address)) + }) + .map(|(id, _)| *id) + { + Ok(Some(id)) + } else { + // then look at ephemeral addresses + if let Some(id) = self.find_account_for_ephemeral_address(address)? { + Ok(Some(id)) + } else { + for (account_id, account) in self.accounts.iter() { + if account.get_legacy_transparent_address()?.is_some() { + return Ok(Some(*account_id)); + } + } + Ok(None) + } + } + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn find_account_for_ephemeral_address( + &self, + address: &TransparentAddress, + ) -> Result, Error> { + for (account_id, account) in self.accounts.iter() { + let contains = account + .ephemeral_addresses()? + .iter() + .any(|(eph_addr, _)| eph_addr == address); + if contains { + return Ok(Some(*account_id)); + } + } + Ok(None) + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn mark_ephemeral_address_as_seen( + &mut self, + address: &TransparentAddress, + tx_id: TxId, + ) -> Result<(), Error> { + for (_, account) in self.accounts.iter_mut() { + account.mark_ephemeral_address_as_seen(address, tx_id)? + } + Ok(()) + } +} + +impl Deref for Accounts { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.accounts + } +} + +impl DerefMut for Accounts { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.accounts + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct EphemeralAddress { + pub(crate) address: TransparentAddress, + // Used implies seen + pub(crate) used: Option, + pub(crate) seen: Option, +} + +impl EphemeralAddress { + fn mark_used(&mut self, tx: TxId) { + // We update both `used_in_tx` and `seen_in_tx` here, because a used address has + // necessarily been seen in a transaction. We will not treat this as extending the + // range of addresses that are safe to reserve unless and until the transaction is + // observed as mined. + self.used.replace(tx); + self.seen.replace(tx); + } + fn mark_seen(&mut self, tx: TxId) -> Option { + self.seen.replace(tx) + } +} + +/// An internal representation account stored in the database. +#[derive(Debug, Clone)] +pub struct Account { + account_name: String, + account_id: AccountId, + kind: AccountSource, + viewing_key: UnifiedFullViewingKey, + birthday: AccountBirthday, + /// Stores diversified Unified Addresses that have been generated from accounts in the wallet. + addresses: BTreeMap, + pub(crate) ephemeral_addresses: BTreeMap, // NonHardenedChildIndex (< 1 << 31) + _notes: BTreeSet, +} + +impl PartialEq for Account { + fn eq(&self, other: &Self) -> bool { + self.account_name == other.account_name + && self.account_id == other.account_id + && self.kind == other.kind + && self + .viewing_key + .encode(&zcash_primitives::consensus::MainNetwork) + == other + .viewing_key + .encode(&zcash_primitives::consensus::MainNetwork) + && self.birthday == other.birthday + && self.addresses == other.addresses + && self.ephemeral_addresses == other.ephemeral_addresses + && self._notes == other._notes + } +} + +impl Account { + pub(crate) fn new( + account_name: String, + account_id: AccountId, + kind: AccountSource, + viewing_key: UnifiedFullViewingKey, + birthday: AccountBirthday, + ) -> Result { + let mut acc = Self { + account_name, + account_id, + kind, + viewing_key, + birthday, + ephemeral_addresses: BTreeMap::new(), + addresses: BTreeMap::new(), + _notes: BTreeSet::new(), + }; + + // populate the addresses map with the default address + let ua_request = acc + .viewing_key + .to_unified_incoming_viewing_key() + .to_address_request() + .and_then(|ua_request| ua_request.intersect(&UnifiedAddressRequest::all().unwrap())) + .ok_or_else(|| { + Error::AddressGeneration(AddressGenerationError::ShieldedReceiverRequired) + })?; + let (ua, diversifier_index) = acc.default_address(ua_request)?; + acc.addresses.insert(diversifier_index, ua); + #[cfg(feature = "transparent-inputs")] + acc.reserve_until(0)?; + Ok(acc) + } + + pub(crate) fn addresses(&self) -> &BTreeMap { + &self.addresses + } + + pub(crate) fn select_receiving_address( + &self, + network: NetworkType, + receiver: &Receiver, + ) -> Result, Error> { + Ok(self + .addresses + .values() + .map(|ua| ua.to_address(network)) + .find(|addr| receiver.corresponds(addr))) + } + + /// Returns the default Unified Address for the account, + /// along with the diversifier index that generated it. + /// + /// The diversifier index may be non-zero if the Unified Address includes a Sapling + /// receiver, and there was no valid Sapling receiver at diversifier index zero. + pub(crate) fn default_address( + &self, + request: UnifiedAddressRequest, + ) -> Result<(UnifiedAddress, DiversifierIndex), AddressGenerationError> { + self.uivk().default_address(request) + } + + pub(crate) fn birthday(&self) -> &AccountBirthday { + &self.birthday + } + + pub(crate) fn current_address(&self) -> Result<(UnifiedAddress, DiversifierIndex), Error> { + Ok(self + .addresses + .iter() + .last() + .map(|(diversifier_index, ua)| (ua.clone(), *diversifier_index)) + .unwrap()) // can unwrap as the map is never empty + } + + pub(crate) fn kind(&self) -> &AccountSource { + &self.kind + } + + pub(crate) fn next_available_address( + &mut self, + request: UnifiedAddressRequest, + ) -> Result, Error> { + match self.ufvk() { + Some(ufvk) => { + let search_from = self + .current_address() + .map(|(_, mut diversifier_index)| { + diversifier_index.increment().map_err(|_| { + Error::AddressGeneration( + AddressGenerationError::DiversifierSpaceExhausted, + ) + })?; + Ok::<_, Error>(diversifier_index) + }) + .unwrap_or(Ok(DiversifierIndex::default()))?; + let (ua, diversifier_index) = ufvk.find_address(search_from, request)?; + self.addresses.insert(diversifier_index, ua.clone()); + Ok(Some(ua)) + } + None => Ok(None), + } + } + + pub(crate) fn account_id(&self) -> AccountId { + self.account_id + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn get_legacy_transparent_address( + &self, + ) -> Result, Error> { + Ok(self + .uivk() + .transparent() + .as_ref() + .map(|tivk| tivk.default_address())) + } +} +#[cfg(feature = "transparent-inputs")] +impl Account { + pub(crate) fn ephemeral_addresses( + &self, + ) -> Result, Error> { + Ok(self + .ephemeral_addresses + .iter() + .map(|(idx, addr)| { + ( + addr.address, + TransparentAddressMetadata::new( + TransparentKeyScope::EPHEMERAL, + NonHardenedChildIndex::from_index(*idx).unwrap(), + ), + ) + }) + .collect()) + } + pub(crate) fn ephemeral_ivk(&self) -> Result, Error> { + self.viewing_key + .transparent() + .map(AccountPubKey::derive_ephemeral_ivk) + .transpose() + .map_err(Into::into) + } + + pub(crate) fn first_unstored_index(&self) -> Result { + if let Some((idx, _)) = self.ephemeral_addresses.last_key_value() { + if *idx >= (1 << 31) + GAP_LIMIT { + unreachable!("violates constraint index_range_and_address_nullity") + } else { + Ok(idx.checked_add(1).unwrap()) + } + } else { + Ok(0) + } + } + + pub(crate) fn first_unreserved_index(&self) -> Result { + self.first_unstored_index()? + .checked_sub(GAP_LIMIT) + .ok_or(Error::CorruptedData( + "ephemeral_addresses corrupted".to_owned(), + )) + } + + pub(crate) fn reserve_until( + &mut self, + next_to_reserve: u32, + ) -> Result, Error> { + if let Some(ephemeral_ivk) = self.ephemeral_ivk()? { + let first_unstored = self.first_unstored_index()?; + let range_to_store = first_unstored..(next_to_reserve.checked_add(GAP_LIMIT).unwrap()); + if range_to_store.is_empty() { + return Ok(Vec::new()); + } + return range_to_store + .map(|raw_index| { + NonHardenedChildIndex::from_index(raw_index) + .map(|address_index| { + ephemeral_ivk + .derive_ephemeral_address(address_index) + .map(|addr| { + self.ephemeral_addresses.insert( + raw_index, + EphemeralAddress { + address: addr, + seen: None, + used: None, + }, + ); + ( + addr, + TransparentAddressMetadata::new( + TransparentKeyScope::EPHEMERAL, + address_index, + ), + ) + }) + }) + .unwrap() + .map_err(Into::into) + }) + .collect::, _>>(); + } + Ok(Vec::new()) + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn mark_ephemeral_address_as_used( + &mut self, + address: &TransparentAddress, + tx_id: TxId, + ) -> Result<(), Error> { + // TODO: ephemeral_address_reuse_check + for (idx, addr) in self.ephemeral_addresses.iter_mut() { + if addr.address == *address { + addr.mark_used(tx_id); + + // Maintain the invariant that the last `GAP_LIMIT` addresses are used and unseen. + let next_to_reserve = idx.checked_add(1).expect("ensured by constraint"); + self.reserve_until(next_to_reserve)?; + return Ok(()); + } + } + Ok(()) + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn mark_ephemeral_address_as_seen( + &mut self, + // txns: &TransactionTable, + address: &TransparentAddress, + tx_id: TxId, + ) -> Result<(), Error> { + for (idx, addr) in self.ephemeral_addresses.iter_mut() { + if addr.address == *address { + // TODO: this + // Figure out which transaction was mined earlier: `tx_ref`, or any existing + // tx referenced by `seen_in_tx` for the given address. Prefer the existing + // reference in case of a tie or if both transactions are unmined. + // This slightly reduces the chance of unnecessarily reaching the gap limit + // too early in some corner cases (because the earlier transaction is less + // likely to be unmined). + // + // The query should always return a value if `tx_ref` is valid. + + addr.mark_seen(tx_id); + // Maintain the invariant that the last `GAP_LIMIT` addresses are used and unseen. + let next_to_reserve = idx.checked_add(1).expect("ensured by constraint"); + self.reserve_until(next_to_reserve)?; + return Ok(()); + } + } + Ok(()) + } +} + +impl zcash_client_backend::data_api::Account for Account { + type AccountId = AccountId; + + fn id(&self) -> AccountId { + self.account_id + } + + fn source(&self) -> &AccountSource { + &self.kind + } + + fn ufvk(&self) -> Option<&UnifiedFullViewingKey> { + Some(&self.viewing_key) + } + + fn uivk(&self) -> UnifiedIncomingViewingKey { + self.viewing_key.to_unified_incoming_viewing_key() + } + + fn name(&self) -> Option<&str> { + todo!() + } +} + +mod serialization { + use zcash_client_backend::data_api::chain::ChainState; + use zcash_keys::encoding::AddressCodec; + use zcash_primitives::block::BlockHash; + use zcash_primitives::consensus::Network::MainNetwork as EncodingParams; + use zcash_primitives::merkle_tree::{read_frontier_v1, write_frontier_v1}; + use zip32::fingerprint::SeedFingerprint; + + use super::*; + use crate::proto::memwallet as proto; + use crate::read_optional; + + impl From for proto::Accounts { + fn from(accounts: Accounts) -> Self { + Self { + account_nonce: accounts.nonce, + accounts: accounts + .accounts + .into_values() + .map(|acc| acc.into()) + .collect(), + } + } + } + + impl From for Accounts { + fn from(accounts: proto::Accounts) -> Self { + Self { + nonce: accounts.account_nonce, + accounts: accounts + .accounts + .into_iter() + .map(|acc| (AccountId(acc.account_id), acc.try_into().unwrap())) + .collect(), + } + } + } + + impl From for proto::Account { + fn from(acc: Account) -> Self { + Self { + account_name: acc.account_name.clone(), + account_id: *acc.account_id, + kind: match acc.kind { + AccountSource::Derived { .. } => 0, + AccountSource::Imported { .. } => 1, + }, + seed_fingerprint: match acc.kind { + AccountSource::Derived { + seed_fingerprint, .. + } => Some(seed_fingerprint.to_bytes().to_vec()), + AccountSource::Imported { .. } => None, + }, + account_index: match acc.kind { + AccountSource::Derived { account_index, .. } => Some(account_index.into()), + AccountSource::Imported { .. } => None, + }, + purpose: match acc.kind { + AccountSource::Derived { .. } => None, + AccountSource::Imported { purpose, .. } => match purpose { + AccountPurpose::Spending => Some(0), + AccountPurpose::ViewOnly => Some(1), + }, + }, + key_source: match acc.kind { + AccountSource::Derived { ref key_source, .. } => key_source, + AccountSource::Imported { ref key_source, .. } => key_source, + } + .clone(), + viewing_key: acc.viewing_key.encode(&EncodingParams), + birthday: Some(acc.birthday().clone().try_into().unwrap()), + addresses: acc + .addresses() + .iter() + .map(|(di, a)| proto::Address { + diversifier_index: di.as_bytes().to_vec(), + address: a.encode(&EncodingParams), // convention is to encode using mainnet encoding regardless of network + }) + .collect(), + #[cfg(feature = "transparent-inputs")] + ephemeral_addresses: acc + .ephemeral_addresses + .into_iter() + .map(|(index, address)| proto::EphemeralAddressRecord { + index, + ephemeral_address: Some(proto::EphemeralAddress { + address: address.address.encode(&EncodingParams), + used_in_tx: address.used.map(|u| u.as_ref().to_vec()), + seen_in_tx: address.seen.map(|s| s.as_ref().to_vec()), + }), + }) + .collect(), + #[cfg(not(feature = "transparent-inputs"))] + ephemeral_addresses: Default::default(), + } + } + } + + impl TryFrom for Account { + type Error = crate::Error; + + fn try_from(acc: proto::Account) -> Result { + Ok(Self { + account_name: acc.account_name.clone(), + account_id: acc.account_id.into(), + kind: match acc.kind { + 0 => AccountSource::Derived { + seed_fingerprint: SeedFingerprint::from_bytes( + acc.seed_fingerprint().try_into()?, + ), + account_index: read_optional!(acc, account_index)?.try_into()?, + key_source: acc.key_source, + }, + 1 => AccountSource::Imported { + purpose: match read_optional!(acc, purpose)? { + 0 => AccountPurpose::Spending, + 1 => AccountPurpose::ViewOnly, + _ => unreachable!(), + }, + key_source: acc.key_source, + }, + _ => unreachable!(), + }, + viewing_key: UnifiedFullViewingKey::decode(&EncodingParams, &acc.viewing_key) + .map_err(Error::UfvkDecodeError)?, + birthday: read_optional!(acc, birthday)?.try_into()?, + addresses: acc + .addresses + .into_iter() + .map(|a| { + Ok(( + DiversifierIndex::from(TryInto::<[u8; 11]>::try_into( + a.diversifier_index, + )?), + UnifiedAddress::decode(&EncodingParams, &a.address) + .map_err(Error::UfvkDecodeError)?, + )) + }) + .collect::>()?, + #[cfg(feature = "transparent-inputs")] + ephemeral_addresses: acc + .ephemeral_addresses + .into_iter() + .map(|address_record| { + let address = read_optional!(address_record, ephemeral_address)?; + Ok(( + address_record.index, + EphemeralAddress { + address: TransparentAddress::decode( + &EncodingParams, + &address.address, + )?, + used: address + .used_in_tx + .map::, _>(|s| { + Ok(TxId::from_bytes(s.try_into()?)) + }) + .transpose()?, + seen: address + .seen_in_tx + .map::, _>(|s| { + Ok(TxId::from_bytes(s.try_into()?)) + }) + .transpose()?, + }, + )) + }) + .collect::>()?, + #[cfg(not(feature = "transparent-inputs"))] + ephemeral_addresses: Default::default(), + _notes: Default::default(), + }) + } + } + + impl TryFrom for proto::AccountBirthday { + type Error = crate::Error; + fn try_from(birthday: AccountBirthday) -> Result { + let cstate = birthday.prior_chain_state(); + + let mut sapling_tree_bytes = vec![]; + write_frontier_v1(&mut sapling_tree_bytes, cstate.final_sapling_tree())?; + + #[cfg(feature = "orchard")] + let orchard_tree_bytes = { + let mut orchard_tree_bytes = vec![]; + write_frontier_v1(&mut orchard_tree_bytes, cstate.final_orchard_tree())?; + orchard_tree_bytes + }; + #[cfg(not(feature = "orchard"))] + let orchard_tree_bytes = vec![]; + + Ok(Self { + prior_chain_state: Some(proto::ChainState { + block_height: cstate.block_height().into(), + block_hash: cstate.block_hash().0.to_vec(), + final_sapling_tree: sapling_tree_bytes, + final_orchard_tree: orchard_tree_bytes, + }), + recover_until: birthday.recover_until().map(|r| r.into()), + }) + } + } + + impl TryFrom for AccountBirthday { + type Error = crate::Error; + fn try_from(birthday: proto::AccountBirthday) -> Result { + let cs = read_optional!(birthday, prior_chain_state)?; + + let cstate = ChainState::new( + cs.block_height.into(), + BlockHash::from_slice(&cs.block_hash), + read_frontier_v1(&cs.final_sapling_tree[..])?, + #[cfg(feature = "orchard")] + read_frontier_v1(&cs.final_orchard_tree[..])?, + ); + + let recover_until = birthday.recover_until.map(|r| r.into()); + + Ok(Self::from_parts(cstate, recover_until)) + } + } + + #[cfg(test)] + mod tests { + use super::*; + use crate::proto::memwallet as proto; + use pretty_assertions::assert_eq; + use zcash_primitives::block::BlockHash; + + const TEST_VK: &str = "uview1tg6rpjgju2s2j37gkgjq79qrh5lvzr6e0ed3n4sf4hu5qd35vmsh7avl80xa6mx7ryqce9hztwaqwrdthetpy4pc0kce25x453hwcmax02p80pg5savlg865sft9reat07c5vlactr6l2pxtlqtqunt2j9gmvr8spcuzf07af80h5qmut38h0gvcfa9k4rwujacwwca9vu8jev7wq6c725huv8qjmhss3hdj2vh8cfxhpqcm2qzc34msyrfxk5u6dqttt4vv2mr0aajreww5yufpk0gn4xkfm888467k7v6fmw7syqq6cceu078yw8xja502jxr0jgum43lhvpzmf7eu5dmnn6cr6f7p43yw8znzgxg598mllewnx076hljlvynhzwn5es94yrv65tdg3utuz2u3sras0wfcq4adxwdvlk387d22g3q98t5z74quw2fa4wed32escx8dwh4mw35t4jwf35xyfxnu83mk5s4kw2glkgsshmxk"; + + #[test] + fn test_account_serialization_roundtrip() { + let acc = Account::new( + "test_account_name".to_string(), + AccountId(0), + AccountSource::Imported { + purpose: AccountPurpose::Spending, + key_source: Some("test_key_source".to_string()), + }, + UnifiedFullViewingKey::decode(&EncodingParams, TEST_VK).unwrap(), + AccountBirthday::from_sapling_activation( + &EncodingParams, + BlockHash::from_slice(&[0; 32]), + ), + ) + .unwrap(); + + let proto_acc: proto::Account = acc.clone().into(); + let acc2: Account = proto_acc.clone().try_into().unwrap(); + let proto_acc2: proto::Account = acc2.clone().into(); + + assert_eq!(proto_acc, proto_acc2); + } + } +} diff --git a/zcash_client_memory/src/types/block.rs b/zcash_client_memory/src/types/block.rs new file mode 100644 index 000000000..9df023434 --- /dev/null +++ b/zcash_client_memory/src/types/block.rs @@ -0,0 +1,155 @@ +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, +}; + +use zcash_client_backend::wallet::NoteId; +use zcash_primitives::{block::BlockHash, consensus::BlockHeight, transaction::TxId}; +use zcash_protocol::memo::MemoBytes; + +/// Internal wallet representation of a Block. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct MemoryWalletBlock { + pub(crate) height: BlockHeight, + pub(crate) hash: BlockHash, + pub(crate) block_time: u32, + // Just the transactions that involve an account in this wallet + pub(crate) _transactions: HashSet, + pub(crate) _memos: HashMap, + pub(crate) sapling_commitment_tree_size: Option, + pub(crate) sapling_output_count: Option, + #[cfg(feature = "orchard")] + pub(crate) orchard_commitment_tree_size: Option, + #[cfg(feature = "orchard")] + pub(crate) orchard_action_count: Option, +} + +impl Eq for MemoryWalletBlock {} + +impl PartialOrd for MemoryWalletBlock { + fn partial_cmp(&self, other: &Self) -> Option { + Some((self.height, self.block_time).cmp(&(other.height, other.block_time))) + } +} + +impl Ord for MemoryWalletBlock { + fn cmp(&self, other: &Self) -> Ordering { + (self.height, self.block_time).cmp(&(other.height, other.block_time)) + } +} + +mod serialization { + use super::*; + use crate::error::{Error, Result}; + use crate::proto::memwallet as proto; + use crate::read_optional; + + impl From for proto::WalletBlock { + fn from(block: MemoryWalletBlock) -> Self { + Self { + height: block.height.into(), + hash: block.hash.0.to_vec(), + block_time: block.block_time, + transactions: block + ._transactions + .into_iter() + .map(|txid| txid.as_ref().to_vec()) + .collect(), + memos: block + ._memos + .into_iter() + .map(|(note_id, memo)| proto::Memo { + note_id: Some(note_id.into()), + memo: memo.as_array().to_vec(), + }) + .collect(), + sapling_commitment_tree_size: block.sapling_commitment_tree_size, + sapling_output_count: block.sapling_output_count, + #[cfg(feature = "orchard")] + orchard_commitment_tree_size: block.orchard_commitment_tree_size, + #[cfg(not(feature = "orchard"))] + orchard_commitment_tree_size: None, + #[cfg(feature = "orchard")] + orchard_action_count: block.orchard_action_count, + #[cfg(not(feature = "orchard"))] + orchard_action_count: None, + } + } + } + + impl TryFrom for MemoryWalletBlock { + type Error = crate::Error; + fn try_from(block: proto::WalletBlock) -> Result { + Ok(Self { + height: block.height.into(), + hash: BlockHash(block.hash.try_into()?), + block_time: block.block_time, + _transactions: block + .transactions + .into_iter() + .map(|txid| Ok(TxId::from_bytes(txid.try_into()?))) + .collect::>()?, + _memos: block + .memos + .into_iter() + .map(|memo| { + let note_id = read_optional!(memo, note_id)?; + Ok(( + NoteId::new( + read_optional!(note_id.clone(), tx_id)?.try_into()?, + match note_id.pool() { + proto::PoolType::ShieldedSapling => { + zcash_protocol::ShieldedProtocol::Sapling + } + #[cfg(feature = "orchard")] + proto::PoolType::ShieldedOrchard => { + zcash_protocol::ShieldedProtocol::Orchard + } + _ => unreachable!(), + }, + note_id.output_index as u16, + ), + MemoBytes::from_bytes(&memo.memo)?, + )) + }) + .collect::>()?, + sapling_commitment_tree_size: block.sapling_commitment_tree_size, + sapling_output_count: block.sapling_output_count, + #[cfg(feature = "orchard")] + orchard_commitment_tree_size: block.orchard_commitment_tree_size, + #[cfg(feature = "orchard")] + orchard_action_count: block.orchard_action_count, + }) + } + } + + #[cfg(test)] + mod tests { + use super::*; + use crate::proto::memwallet as proto; + use pretty_assertions::assert_eq; + use zcash_primitives::block::BlockHash; + + #[test] + fn test_block_roundtrip() { + let block = MemoryWalletBlock { + height: 1.into(), + hash: BlockHash([0; 32]), + block_time: 2, + _transactions: HashSet::new(), + _memos: HashMap::new(), + sapling_commitment_tree_size: Some(3), + sapling_output_count: Some(4), + #[cfg(feature = "orchard")] + orchard_commitment_tree_size: Some(5), + #[cfg(feature = "orchard")] + orchard_action_count: Some(6), + }; + + let proto: proto::WalletBlock = block.clone().into(); + let recovered: MemoryWalletBlock = proto.clone().try_into().unwrap(); + + assert_eq!(block, recovered); + } + } +} diff --git a/zcash_client_memory/src/types/data_requests.rs b/zcash_client_memory/src/types/data_requests.rs new file mode 100644 index 000000000..f6318a9da --- /dev/null +++ b/zcash_client_memory/src/types/data_requests.rs @@ -0,0 +1,95 @@ +use std::{collections::VecDeque, ops::Deref}; + +use zcash_client_backend::data_api::TransactionDataRequest; +use zcash_primitives::transaction::TxId; + +#[derive(Debug, Default, PartialEq)] +pub struct TransactionDataRequestQueue(pub(crate) VecDeque); + +impl TransactionDataRequestQueue { + pub fn new() -> Self { + Self(VecDeque::new()) + } + + pub fn queue_status_retrieval(&mut self, txid: &TxId) { + self.0.push_back(TransactionDataRequest::GetStatus(*txid)); + } +} + +impl Deref for TransactionDataRequestQueue { + type Target = VecDeque; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +mod serialization { + use super::*; + use crate::{error::Error, proto::memwallet as proto, read_optional}; + use zcash_keys::encoding::AddressCodec; + use zcash_primitives::{ + consensus::Network::MainNetwork as EncodingParams, legacy::TransparentAddress, + }; + + impl From for proto::TransactionDataRequest { + fn from(request: TransactionDataRequest) -> Self { + match request { + TransactionDataRequest::GetStatus(txid) => Self { + request_type: proto::TransactionDataRequestType::GetStatus as i32, + tx_id: Some(txid.into()), + address: None, + block_range_start: None, + block_range_end: None, + }, + TransactionDataRequest::Enhancement(txid) => Self { + request_type: proto::TransactionDataRequestType::Enhancement as i32, + tx_id: Some(txid.into()), + address: None, + block_range_start: None, + block_range_end: None, + }, + #[cfg(feature = "transparent-inputs")] + TransactionDataRequest::SpendsFromAddress { + address, + block_range_start, + block_range_end, + } => Self { + request_type: proto::TransactionDataRequestType::SpendsFromAddress as i32, + tx_id: None, + address: Some(address.encode(&EncodingParams).as_bytes().to_vec()), + block_range_start: Some(block_range_start.into()), + block_range_end: block_range_end.map(Into::into), + }, + } + } + } + + impl TryFrom for TransactionDataRequest { + type Error = crate::Error; + + fn try_from(request: proto::TransactionDataRequest) -> Result { + Ok(match request.request_type() { + proto::TransactionDataRequestType::GetStatus => { + TransactionDataRequest::GetStatus(read_optional!(request, tx_id)?.try_into()?) + } + proto::TransactionDataRequestType::Enhancement => { + TransactionDataRequest::Enhancement(read_optional!(request, tx_id)?.try_into()?) + } + #[cfg(feature = "transparent-inputs")] + proto::TransactionDataRequestType::SpendsFromAddress => { + TransactionDataRequest::SpendsFromAddress { + address: TransparentAddress::decode( + &EncodingParams, + &String::from_utf8(read_optional!(request, address)?)?, + )?, + block_range_start: read_optional!(request, block_range_start)?.into(), + block_range_end: Some(read_optional!(request, block_range_end)?.into()), + } + } + #[cfg(not(feature = "transparent-inputs"))] + _ => panic!("invalid request type"), + }) + } + } +} diff --git a/zcash_client_memory/src/types/memory_wallet/mod.rs b/zcash_client_memory/src/types/memory_wallet/mod.rs new file mode 100644 index 000000000..f8601ddb0 --- /dev/null +++ b/zcash_client_memory/src/types/memory_wallet/mod.rs @@ -0,0 +1,1092 @@ +#![allow(dead_code)] + +mod serialization; + +use std::{ + cmp::min, + collections::{btree_map::Entry, BTreeMap, BTreeSet}, + num::NonZeroU32, + ops::{Range, RangeInclusive}, + usize, +}; + +use incrementalmerkletree::{Address, Level, Marking, Position, Retention}; +use scanning::ScanQueue; +use shardtree::{ + store::{memory::MemoryShardStore, ShardStore}, + ShardTree, +}; +use transparent::{ + TransparentReceivedOutputSpends, TransparentReceivedOutputs, TransparentSpendCache, +}; +use zcash_client_backend::{ + data_api::{ + scanning::{ScanPriority, ScanRange}, + Account as _, AccountBirthday, AccountSource, InputSource, Ratio, TransactionStatus, + WalletRead, GAP_LIMIT, SAPLING_SHARD_HEIGHT, + }, + wallet::{NoteId, WalletSaplingOutput, WalletTransparentOutput}, +}; +use zcash_keys::keys::UnifiedFullViewingKey; +use zcash_primitives::{ + consensus::{self, BlockHeight, NetworkUpgrade}, + legacy::TransparentAddress, + transaction::{components::OutPoint, Transaction, TxId}, +}; +use zcash_protocol::ShieldedProtocol; +use zip32::fingerprint::SeedFingerprint; + +#[cfg(feature = "orchard")] +use zcash_client_backend::{data_api::ORCHARD_SHARD_HEIGHT, wallet::WalletOrchardOutput}; + +use crate::error::Error; +use crate::types::*; + +/// The main in-memory wallet database. Implements all the traits needed to be used as a backend. +#[derive(Debug)] +pub struct MemoryWalletDb { + /// Zcash network parameters for wallet + pub(crate) params: P, + /// The accounts in the wallet + pub(crate) accounts: Accounts, + /// The wallet that have been scanned and cached that contain relevant wallet data + pub(crate) blocks: BTreeMap, + /// Scanned transactions relevant to accounts in this wallet + pub(crate) tx_table: TransactionTable, + /// Notes that an account has received + pub(crate) received_notes: ReceivedNoteTable, + /// Notes that have been spent + pub(crate) received_note_spends: ReceievedNoteSpends, + /// Nullifiers for notes that have been spent + pub(crate) nullifiers: NullifierMap, + /// Stores the outputs of transactions created by the wallet. + pub(crate) sent_notes: SentNoteTable, + /// Maps transaction ids to their block height and index + pub(crate) tx_locator: TxLocatorMap, + /// Sapling commitment tree + pub(crate) sapling_tree: ShardTree< + MemoryShardStore, + { SAPLING_SHARD_HEIGHT * 2 }, + SAPLING_SHARD_HEIGHT, + >, + /// Stores the block height corresponding to the last note commitment in a shard + pub(crate) sapling_tree_shard_end_heights: BTreeMap, + /// Orchard commitment tree + #[cfg(feature = "orchard")] + pub(crate) orchard_tree: ShardTree< + MemoryShardStore, + { ORCHARD_SHARD_HEIGHT * 2 }, + ORCHARD_SHARD_HEIGHT, + >, + #[cfg(feature = "orchard")] + /// Stores the block height corresponding to the last note commitment in a shard + pub(crate) orchard_tree_shard_end_heights: BTreeMap, + + /// Transparent outputs received by the wallet + pub(crate) transparent_received_outputs: TransparentReceivedOutputs, + /// Transparent outputs received by the wallet that have been spent + pub(crate) transparent_received_output_spends: TransparentReceivedOutputSpends, + /// Map between transparent outpoints and their spend transactions + pub(crate) transparent_spend_map: TransparentSpendCache, + + /// Pending requests to the external data provider to enhance transaction data + pub(crate) transaction_data_request_queue: TransactionDataRequestQueue, + /// Queue of block ranges that should be scanned along with their priority + pub(crate) scan_queue: ScanQueue, +} + +impl PartialEq for MemoryWalletDb

{ + /// Tests for equality between two `MemoryWalletDb` instances. + /// but does NOT compare the sapling_tree and orchard_tree fields. + fn eq(&self, other: &Self) -> bool { + #[cfg(feature = "orchard")] + let orchard_comparisons = + { self.orchard_tree_shard_end_heights == other.orchard_tree_shard_end_heights }; + #[cfg(not(feature = "orchard"))] + let orchard_comparisons = true; + + #[cfg(feature = "transparent-inputs")] + let transparent_comparisons = { + self.transparent_received_outputs == other.transparent_received_outputs + && self.transparent_received_output_spends + == other.transparent_received_output_spends + && self.transparent_spend_map == other.transparent_spend_map + }; + #[cfg(not(feature = "transparent-inputs"))] + let transparent_comparisons = true; + + self.params == other.params + && self.accounts == other.accounts + && self.blocks == other.blocks + && self.tx_table == other.tx_table + && self.received_notes == other.received_notes + && self.received_note_spends == other.received_note_spends + && self.nullifiers == other.nullifiers + && self.sent_notes == other.sent_notes + && self.tx_locator == other.tx_locator + && self.scan_queue == other.scan_queue + && self.sapling_tree_shard_end_heights == other.sapling_tree_shard_end_heights + && orchard_comparisons + && transparent_comparisons + && self.transaction_data_request_queue == other.transaction_data_request_queue + } +} + +impl MemoryWalletDb

{ + pub fn new(params: P, max_checkpoints: usize) -> Self { + Self { + accounts: Accounts::new(), + params, + blocks: BTreeMap::new(), + sapling_tree: ShardTree::new(MemoryShardStore::empty(), max_checkpoints), + sapling_tree_shard_end_heights: BTreeMap::new(), + #[cfg(feature = "orchard")] + orchard_tree: ShardTree::new(MemoryShardStore::empty(), max_checkpoints), + #[cfg(feature = "orchard")] + orchard_tree_shard_end_heights: BTreeMap::new(), + tx_table: TransactionTable::new(), + received_notes: ReceivedNoteTable::new(), + sent_notes: SentNoteTable::new(), + nullifiers: NullifierMap::new(), + tx_locator: TxLocatorMap::new(), + received_note_spends: ReceievedNoteSpends::new(), + scan_queue: ScanQueue::new(), + transparent_received_outputs: TransparentReceivedOutputs::new(), + transparent_received_output_spends: TransparentReceivedOutputSpends::new(), + transparent_spend_map: TransparentSpendCache::new(), + transaction_data_request_queue: TransactionDataRequestQueue::new(), + } + } + + pub fn params(&self) -> &P { + &self.params + } + + pub(crate) fn add_account( + &mut self, + account_name: &str, + kind: AccountSource, + viewing_key: UnifiedFullViewingKey, + birthday: AccountBirthday, + ) -> Result<(AccountId, Account), Error> { + let (id, account) = self.accounts.new_account( + account_name, + kind, + viewing_key.to_owned(), + birthday.clone(), + )?; + + // If a birthday frontier is available, insert it into the note commitment tree. If the + // birthday frontier is the empty frontier, we don't need to do anything. + if let Some(frontier) = birthday.sapling_frontier().value() { + tracing::debug!("Inserting Sapling frontier into ShardTree: {:?}", frontier); + self.sapling_tree.insert_frontier_nodes( + frontier.clone(), + Retention::Checkpoint { + // This subtraction is safe, because all leaves in the tree appear in blocks, and + // the invariant that birthday.height() always corresponds to the block for which + // `frontier` is the tree state at the start of the block. Together, this means + // there exists a prior block for which frontier is the tree state at the end of + // the block. + id: birthday.height() - 1, + marking: Marking::Reference, + }, + )?; + } + + #[cfg(feature = "orchard")] + if let Some(frontier) = birthday.orchard_frontier().value() { + tracing::debug!("Inserting Orchard frontier into ShardTree: {:?}", frontier); + self.orchard_tree.insert_frontier_nodes( + frontier.clone(), + Retention::Checkpoint { + // This subtraction is safe, because all leaves in the tree appear in blocks, and + // the invariant that birthday.height() always corresponds to the block for which + // `frontier` is the tree state at the start of the block. Together, this means + // there exists a prior block for which frontier is the tree state at the end of + // the block. + id: birthday.height() - 1, + marking: Marking::Reference, + }, + )?; + } + + // The ignored range always starts at Sapling activation + let sapling_activation_height = self + .params + .activation_height(NetworkUpgrade::Sapling) + .expect("Sapling activation height must be available."); + + // Add the ignored range up to the birthday height. + if sapling_activation_height < birthday.height() { + let ignored_range = sapling_activation_height..birthday.height(); + self.scan_queue.replace_queue_entries( + &ignored_range, + Some(ScanRange::from_parts( + ignored_range.clone(), + ScanPriority::Ignored, + )) + .into_iter(), + false, + )?; + }; + + // Rewrite the scan ranges from the birthday height up to the chain tip so that we'll ensure we + // re-scan to find any notes that might belong to the newly added account. + if let Some(t) = self.chain_height()? { + let rescan_range = birthday.height()..(t + 1); + self.scan_queue.replace_queue_entries( + &rescan_range, + Some(ScanRange::from_parts( + rescan_range.clone(), + ScanPriority::Historic, + )) + .into_iter(), + true, // force rescan + )?; + } + + Ok((id, account)) + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn first_unsafe_index(&self, account_id: AccountId) -> Result { + let first_unmined_index = if let Some(account) = self.accounts.get(account_id) { + let mut idx = 0; + for (tidx, eph_addr) in account.ephemeral_addresses.iter().rev() { + if eph_addr + .seen + .and_then(|txid| self.tx_table.get(&txid)) + .and_then(|tx| tx.mined_height()) + .is_some() + { + idx = tidx.checked_add(1).unwrap(); + break; + } + } + idx + } else { + 0 + }; + + Ok(min( + 1 << 31, + first_unmined_index.checked_add(GAP_LIMIT).unwrap(), + )) + } + + pub(crate) fn get_funding_accounts( + &self, + tx: &Transaction, + ) -> Result, Error> { + let mut funding_accounts = BTreeSet::new(); + #[cfg(feature = "transparent-inputs")] + funding_accounts.extend( + self.transparent_received_outputs.detect_spending_accounts( + tx.transparent_bundle() + .iter() + .flat_map(|bundle| bundle.vin.iter().map(|txin| &txin.prevout)), + )?, + ); + + funding_accounts.extend(self.received_notes.detect_sapling_spending_accounts( + tx.sapling_bundle().iter().flat_map(|bundle| { + bundle + .shielded_spends() + .iter() + .map(|spend| spend.nullifier()) + }), + )?); + + #[cfg(feature = "orchard")] + funding_accounts.extend( + self.received_notes.detect_orchard_spending_accounts( + tx.orchard_bundle() + .iter() + .flat_map(|bundle| bundle.actions().iter().map(|action| action.nullifier())), + )?, + ); + + Ok(funding_accounts) + } + + pub(crate) fn get_received_notes(&self) -> &ReceivedNoteTable { + &self.received_notes + } + + // TODO: Update this if we switch from using a vec to store received notes to + // someething with more efficient lookups + pub(crate) fn get_received_note(&self, note_id: NoteId) -> Option<&ReceivedNote> { + self.received_notes.iter().find(|v| v.note_id() == note_id) + } + + pub(crate) fn mark_sapling_note_spent( + &mut self, + nf: sapling::Nullifier, + txid: TxId, + ) -> Result<(), Error> { + let note_id = self + .received_notes + .iter() + .filter(|v| v.nullifier() == Some(&Nullifier::Sapling(nf))) + .map(|v| v.note_id()) + .next() + .ok_or_else(|| Error::NoteNotFound)?; + self.received_note_spends.insert_spend(note_id, txid); + Ok(()) + } + + /// Returns true if the note is in the spent notes table and the transaction that spent it is + /// in the transaction table and has either been mined or can be mined in the future + /// (i.e. it hasn't or will not expire) + pub(crate) fn note_is_spent( + &self, + note: &ReceivedNote, + min_confirmations: u32, + ) -> Result { + let spend = self.received_note_spends.get(¬e.note_id()); + + let spent = match spend { + Some(txid) => { + let spending_tx = self + .tx_table + .get(txid) + .ok_or_else(|| Error::TransactionNotFound(*txid))?; + match spending_tx.status() { + TransactionStatus::Mined(_height) => true, + TransactionStatus::TxidNotRecognized => unreachable!(), + TransactionStatus::NotInMainChain => { + // check the expiry + spending_tx.expiry_height().is_none() // no expiry, tx could be mined any time so we consider it spent + // expiry is in the future so it could still be mined + || spending_tx.expiry_height() > self.summary_height(min_confirmations)? + } + } + } + None => false, + }; + Ok(spent) + } + + /// To be spendable a note must be: + /// - unspent (obviously) + /// - not dust (value > 5000 ZATs) + /// - be associated with an account with a ufvk + /// - have a recipient key scope + /// - We know the nullifier + /// - We know the commitment tree position + /// - be in a block less than or equal to the anchor height + /// - not be in the given exclude list + /// + /// Additionally the tree shard containing the node must not be in an unscanned range + /// excluding ranges that start above the anchor height or end below the wallet birthday. + /// This is determined by looking at the scan queue + pub(crate) fn note_is_spendable( + &self, + note: &ReceivedNote, + birthday_height: BlockHeight, + anchor_height: BlockHeight, + exclude: &[ as InputSource>::NoteRef], + ) -> Result { + let note_account = self + .get_account(note.account_id())? + .ok_or_else(|| Error::AccountUnknown(note.account_id))?; + let note_txn = self + .tx_table + .get(¬e.txid()) + .ok_or_else(|| Error::TransactionNotFound(note.txid()))?; + + let unscanned_ranges = self.unscanned_ranges(); + + let note_in_unscanned_range = + unscanned_ranges + .iter() + .any(|(start_height, end_height, start, end_exclusive)| { + let in_range = note.commitment_tree_position.map_or(false, |pos| { + if let (Some(start), Some(end_exclusive)) = (start, end_exclusive) { + pos >= *start && pos < *end_exclusive + } else { + true + } + }); + in_range && *end_height > birthday_height && *start_height <= anchor_height + }); + + Ok(!self.note_is_spent(note, 0)? + && !note_in_unscanned_range + && note.note.value().into_u64() > 5000 + && note_account.ufvk().is_some() + && note.recipient_key_scope.is_some() + && note.nullifier().is_some() + && note.commitment_tree_position.is_some() + && note_txn.mined_height().is_some() + && note_txn.mined_height().unwrap() <= anchor_height + && !exclude.contains(¬e.note_id())) + } + + /// To be pending a note must be: + /// - ? + pub(crate) fn note_is_pending( + &self, + note: &ReceivedNote, + min_confirmations: u32, + ) -> Result { + if let (Some(summary_height), Some(received_height)) = ( + self.summary_height(min_confirmations)?, + self.tx_table + .get(¬e.txid()) + .ok_or_else(|| Error::TransactionNotFound(note.txid()))? + .mined_height(), + ) { + Ok(received_height > summary_height) + } else { + Ok(true) // no summary height or note not mined means it's pending + } + } + + pub(crate) fn summary_height( + &self, + min_confirmations: u32, + ) -> Result, Error> { + let chain_tip_height = match self.chain_height()? { + Some(height) => height, + None => return Ok(None), + }; + let summary_height = + (chain_tip_height + 1).saturating_sub(std::cmp::max(min_confirmations, 1)); + Ok(Some(summary_height)) + } + + #[cfg(feature = "orchard")] + pub(crate) fn mark_orchard_note_spent( + &mut self, + nf: orchard::note::Nullifier, + txid: TxId, + ) -> Result<(), Error> { + let note_id = self + .received_notes + .iter() + .filter(|v| v.nullifier() == Some(&Nullifier::Orchard(nf))) + .map(|v| v.note_id()) + .next() + .ok_or_else(|| Error::NoteNotFound)?; + self.received_note_spends.insert_spend(note_id, txid); + Ok(()) + } + + pub(crate) fn max_zip32_account_index( + &self, + seed_fingerprint: &SeedFingerprint, + ) -> Result, Error> { + Ok(self + .accounts + .iter() + .filter_map(|(_, a)| match a.source() { + AccountSource::Derived { + seed_fingerprint: sf, + account_index, + .. + } => { + if sf == seed_fingerprint { + Some(account_index) + } else { + None + } + } + _ => None, + }) + .max() + .copied()) + } + pub(crate) fn insert_received_sapling_note( + &mut self, + note_id: NoteId, + output: &WalletSaplingOutput, + spent_in: Option, + ) { + self.received_notes + .insert_received_note(ReceivedNote::from_wallet_sapling_output(note_id, output)); + if let Some(spent_in) = spent_in { + self.received_note_spends.insert_spend(note_id, spent_in); + } + } + #[cfg(feature = "orchard")] + pub(crate) fn insert_received_orchard_note( + &mut self, + note_id: NoteId, + output: &WalletOrchardOutput, + spent_in: Option, + ) { + self.received_notes + .insert_received_note(ReceivedNote::from_wallet_orchard_output(note_id, output)); + if let Some(spent_in) = spent_in { + self.received_note_spends.insert_spend(note_id, spent_in); + } + } + pub(crate) fn insert_sapling_nullifier_map( + &mut self, + block_height: BlockHeight, + new_entries: &[(TxId, u16, Vec)], + ) -> Result<(), Error> { + for (txid, tx_index, nullifiers) in new_entries { + for nf in nullifiers.iter() { + self.nullifiers + .insert(block_height, *tx_index as u32, Nullifier::Sapling(*nf)); + } + match self.tx_locator.entry((block_height, *tx_index as u32)) { + Entry::Occupied(x) => { + if txid == x.get() { + // This is a duplicate entry + continue; + } else { + return Err(Error::ConflictingTxLocator); + } + } + Entry::Vacant(entry) => { + entry.insert(*txid); + } + } + } + Ok(()) + } + + #[cfg(feature = "orchard")] + pub(crate) fn insert_orchard_nullifier_map( + &mut self, + block_height: BlockHeight, + new_entries: &[(TxId, u16, Vec)], + ) -> Result<(), Error> { + for (txid, tx_index, nullifiers) in new_entries { + for nf in nullifiers.iter() { + self.nullifiers + .insert(block_height, *tx_index as u32, Nullifier::Orchard(*nf)); + } + match self.tx_locator.entry((block_height, *tx_index as u32)) { + Entry::Occupied(x) => { + if txid == x.get() { + // This is a duplicate entry + continue; + } else { + return Err(Error::ConflictingTxLocator); + } + } + Entry::Vacant(entry) => { + entry.insert(*txid); + } + } + } + Ok(()) + } + + pub(crate) fn block_height_extrema(&self) -> Option> { + let (min, max) = self.blocks.keys().fold((None, None), |(min, max), height| { + ( + Some(min.map_or(height, |min| std::cmp::min(min, height))), + Some(max.map_or(height, |max| std::cmp::max(max, height))), + ) + }); + if let (Some(min), Some(max)) = (min, max) { + Some(*min..=*max) + } else { + None + } + } + + pub(crate) fn sapling_tip_shard_end_height(&self) -> Option { + self.sapling_tree_shard_end_heights.values().max().copied() + } + + #[cfg(feature = "orchard")] + pub(crate) fn orchard_tip_shard_end_height(&self) -> Option { + self.orchard_tree_shard_end_heights.values().max().copied() + } + + pub(crate) fn get_sapling_max_checkpointed_height( + &self, + chain_tip_height: BlockHeight, + min_confirmations: NonZeroU32, + ) -> Result, Error> { + let max_checkpoint_height = + u32::from(chain_tip_height).saturating_sub(u32::from(min_confirmations) - 1); + // scan backward and find the first checkpoint that matches a blockheight prior to max_checkpoint_height + for height in (0..=max_checkpoint_height).rev() { + let height = BlockHeight::from_u32(height); + if self.sapling_tree.store().get_checkpoint(&height)?.is_some() { + return Ok(Some(height)); + } + } + Ok(None) + } + + #[cfg(feature = "orchard")] + pub(crate) fn get_orchard_max_checkpointed_height( + &self, + chain_tip_height: BlockHeight, + min_confirmations: NonZeroU32, + ) -> Result, Error> { + let max_checkpoint_height = + u32::from(chain_tip_height).saturating_sub(u32::from(min_confirmations) - 1); + // scan backward and find the first checkpoint that matches a blockheight prior to max_checkpoint_height + for height in (0..=max_checkpoint_height).rev() { + let height = BlockHeight::from_u32(height); + if self.orchard_tree.store().get_checkpoint(&height)?.is_some() { + return Ok(Some(height)); + } + } + Ok(None) + } + + /// Get the unscanned ranges from the scan queue and their corresponding sapling tree indices + /// This can be used to determine if a note is in an unscanned range and therefore not spendable + pub(crate) fn unscanned_ranges( + &self, + ) -> Vec<(BlockHeight, BlockHeight, Option, Option)> { + self.scan_queue + .iter() + .filter(|(_, _, priority)| priority > &ScanPriority::Scanned) + .map(|(start, end, _)| { + ( + *start, + *end, + self.first_subtree_for_height(start) + .map(|a| a.position_range_start()), + self.last_subtree_for_height(end) + .map(|a| a.position_range_end()), + ) + }) + .collect() + } + + /// Return the address of the last subtree in the sapling tree where note for a give block height was found + pub(crate) fn last_subtree_for_height(&self, height: &BlockHeight) -> Option

{ + self.sapling_tree_shard_end_heights + .iter() + .filter(|(_, h)| *h == height) + .map(|(a, _)| *a) + .max() + } + + /// Return the address of the first subtree in the sapling tree where note for a give block height was found + pub(crate) fn first_subtree_for_height(&self, height: &BlockHeight) -> Option
{ + // The first subtree is the last subtree for the previous height + self.last_subtree_for_height(&height.saturating_sub(1)) + } + + /// Makes the required changes to the scan queue to reflect the completion of a scan + pub(crate) fn scan_complete( + &mut self, + range: Range, + wallet_note_positions: &[(ShieldedProtocol, Position)], + ) -> Result<(), Error> { + let wallet_birthday = self.get_wallet_birthday()?; + + // Determine the range of block heights for which we will be updating the scan queue. + let extended_range = { + // If notes have been detected in the scan, we need to extend any adjacent un-scanned + // ranges starting from the wallet birthday to include the blocks needed to complete + // the note commitment tree subtrees containing the positions of the discovered notes. + // We will query by subtree index to find these bounds. + let mut required_sapling_subtrees = BTreeSet::new(); + #[cfg(feature = "orchard")] + let mut required_orchard_subtrees = BTreeSet::new(); + for (protocol, position) in wallet_note_positions { + match protocol { + ShieldedProtocol::Sapling => { + required_sapling_subtrees.insert( + Address::above_position(SAPLING_SHARD_HEIGHT.into(), *position).index(), + ); + } + ShieldedProtocol::Orchard => { + #[cfg(feature = "orchard")] + required_orchard_subtrees.insert( + Address::above_position(ORCHARD_SHARD_HEIGHT.into(), *position).index(), + ); + + #[cfg(not(feature = "orchard"))] + return Err(Error::OrchardNotEnabled); + } + } + } + + let extended_range = self.extend_range( + &ShieldedProtocol::Sapling, + &range, + required_sapling_subtrees, + self.params.activation_height(NetworkUpgrade::Sapling), + wallet_birthday, + )?; + + #[cfg(feature = "orchard")] + let extended_range = self + .extend_range( + &ShieldedProtocol::Orchard, + extended_range.as_ref().unwrap_or(&range), + required_orchard_subtrees, + self.params.activation_height(NetworkUpgrade::Nu5), + wallet_birthday, + )? + .or(extended_range); + + #[allow(clippy::let_and_return)] + extended_range + }; + + let query_range = extended_range.clone().unwrap_or_else(|| range.clone()); + + let scanned = ScanRange::from_parts(range.clone(), ScanPriority::Scanned); + + // If any of the extended range actually extends beyond the scanned range, we need to + // scan that extension in order to make the found note(s) spendable. We need to avoid + // creating empty ranges here, as that acts as an optimization barrier preventing + // `SpanningTree` from merging non-empty scanned ranges on either side. + let extended_before = extended_range + .as_ref() + .map(|extended| { + ScanRange::from_parts(extended.start..range.start, ScanPriority::FoundNote) + }) + .filter(|range| !range.is_empty()); + let extended_after = extended_range + .map(|extended| ScanRange::from_parts(range.end..extended.end, ScanPriority::FoundNote)) + .filter(|range| !range.is_empty()); + + let replacement = Some(scanned) + .into_iter() + .chain(extended_before) + .chain(extended_after); + + self.scan_queue + .replace_queue_entries(&query_range, replacement, false) + } + + // Given a range of block heights, extend the range to include the subtrees containing the + // given subtree indices, bounded by the wallet birthday and the fallback start height. + fn extend_range( + &self, + pool: &ShieldedProtocol, + range: &Range, + required_subtree_indices: BTreeSet, + fallback_start_height: Option, + birthday_height: Option, + ) -> Result>, Error> { + // we'll either have both min and max bounds, or we'll have neither + let subtree_index_bounds = required_subtree_indices + .iter() + .min() + .zip(required_subtree_indices.iter().max()); + + let shard_end = |index| -> Result<_, Error> { + match pool { + ShieldedProtocol::Sapling => Ok(self + .sapling_tree_shard_end_heights + .get(&Address::from_parts(SAPLING_SHARD_HEIGHT.into(), index)) + .cloned()), + ShieldedProtocol::Orchard => { + #[cfg(feature = "orchard")] + { + Ok(self + .orchard_tree_shard_end_heights + .get(&Address::from_parts(ORCHARD_SHARD_HEIGHT.into(), index)) + .cloned()) + } + #[cfg(not(feature = "orchard"))] + panic!("Unsupported pool") + } + } + }; + + // If no notes belonging to the wallet were found, we don't need to extend the scanning + // range suggestions to include the associated subtrees, and our bounds are just the + // scanned range. Otherwise, ensure that all shard ranges starting from the wallet + // birthday are included. + subtree_index_bounds + .map(|(min_idx, max_idx)| { + let range_min = if *min_idx > 0 { + // get the block height of the end of the previous shard + shard_end(*min_idx - 1)? + } else { + // our lower bound is going to be the fallback height + fallback_start_height + }; + + // bound the minimum to the wallet birthday + let range_min = + range_min.map(|h| birthday_height.map_or(h, |b| std::cmp::max(b, h))); + + // Get the block height for the end of the current shard, and make it an + // exclusive end bound. + let range_max = shard_end(*max_idx)?.map(|end| end + 1); + + Ok(Range { + start: range.start.min(range_min.unwrap_or(range.start)), + end: range.end.max(range_max.unwrap_or(range.end)), + }) + }) + .transpose() + } + + pub(crate) fn get_sent_notes(&self) -> &SentNoteTable { + &self.sent_notes + } + + pub(crate) fn sapling_scan_progress( + &self, + birthday_height: &BlockHeight, + fully_scanned_height: &BlockHeight, + chain_tip_height: &BlockHeight, + ) -> Result>, Error> { + if fully_scanned_height == chain_tip_height { + let outputs_sum = self + .blocks + .iter() + .filter(|(height, _)| height >= &birthday_height) + .fold(0, |sum, (_, block)| { + sum + block.sapling_output_count.unwrap_or(0) + }); + Ok(Some(Ratio::new(outputs_sum as u64, outputs_sum as u64))) + } else { + // Get the starting note commitment tree size from the wallet birthday, or failing that + // from the blocks table. + let start_size = self + .accounts + .iter() + .filter_map(|(_, account)| { + if account.birthday().height() == *birthday_height { + Some(account.birthday().sapling_frontier().tree_size()) + } else { + None + } + }) + .next() + .or_else(|| { + self.blocks + .iter() + .filter(|(height, _)| height <= &birthday_height) + .map(|(_, block)| { + (block.sapling_commitment_tree_size.unwrap_or(0) + - block.sapling_output_count.unwrap_or(0)) + as u64 + }) + .max() + }); + + // Compute the total blocks scanned so far above the starting height + let scanned_count = self + .blocks + .iter() + .filter(|(height, _)| height > &birthday_height) + .fold(0_u64, |acc, (_, block)| { + acc + block.sapling_output_count.unwrap_or(0) as u64 + }); + + // We don't have complete information on how many outputs will exist in the shard at + // the chain tip without having scanned the chain tip block, so we overestimate by + // computing the maximum possible number of notes directly from the shard indices. + // + // TODO: it would be nice to be able to reliably have the size of the commitment tree + // at the chain tip without having to have scanned that block. + + let shard_index_iter = self + .sapling_tree_shard_end_heights + .iter() + .filter(|(_, height)| height > &birthday_height) + .map(|(address, _)| address.index()); + + let min_idx = shard_index_iter.clone().min().unwrap_or(0); + let max_idx = shard_index_iter.max().unwrap_or(0); + + let max_tree_size = Some(min_idx << SAPLING_SHARD_HEIGHT); + let min_tree_size = Some((max_idx + 1) << SAPLING_SHARD_HEIGHT); + + Ok(start_size.or(min_tree_size).zip(max_tree_size).map( + |(min_tree_size, max_tree_size)| { + Ratio::new(scanned_count, max_tree_size - min_tree_size) + }, + )) + } + } + + #[cfg(feature = "orchard")] + pub(crate) fn orchard_scan_progress( + &self, + birthday_height: &BlockHeight, + fully_scanned_height: &BlockHeight, + chain_tip_height: &BlockHeight, + ) -> Result>, Error> { + if fully_scanned_height == chain_tip_height { + let outputs_sum = self + .blocks + .iter() + .filter(|(height, _)| height >= &birthday_height) + .fold(0, |sum, (_, block)| { + sum + block.orchard_action_count.unwrap_or(0) + }); + Ok(Some(Ratio::new(outputs_sum as u64, outputs_sum as u64))) + } else { + // Get the starting note commitment tree size from the wallet birthday, or failing that + // from the blocks table. + let start_size = self + .accounts + .iter() + .filter_map(|(_, account)| { + if account.birthday().height() == *birthday_height { + Some(account.birthday().sapling_frontier().tree_size()) + } else { + None + } + }) + .next() + .or_else(|| { + self.blocks + .iter() + .filter(|(height, _)| height <= &birthday_height) + .map(|(_, block)| { + (block.orchard_commitment_tree_size.unwrap_or(0) + - block.orchard_action_count.unwrap_or(0)) + as u64 + }) + .max() + }); + + // Compute the total blocks scanned so far above the starting height + let scanned_count = self + .blocks + .iter() + .filter(|(height, _)| height > &birthday_height) + .fold(0_u64, |acc, (_, block)| { + acc + block.orchard_action_count.unwrap_or(0) as u64 + }); + + // We don't have complete information on how many outputs will exist in the shard at + // the chain tip without having scanned the chain tip block, so we overestimate by + // computing the maximum possible number of notes directly from the shard indices. + // + // TODO: it would be nice to be able to reliably have the size of the commitment tree + // at the chain tip without having to have scanned that block. + + let shard_index_iter = self + .orchard_tree_shard_end_heights + .iter() + .filter(|(_, height)| height > &birthday_height) + .map(|(address, _)| address.index()); + + let min_idx = shard_index_iter.clone().min().unwrap_or(0); + let max_idx = shard_index_iter.max().unwrap_or(0); + + let max_tree_size = Some(min_idx << ORCHARD_SHARD_HEIGHT); + let min_tree_size = Some((max_idx + 1) << ORCHARD_SHARD_HEIGHT); + + Ok(start_size.or(min_tree_size).zip(max_tree_size).map( + |(min_tree_size, max_tree_size)| { + Ratio::new(scanned_count, max_tree_size - min_tree_size) + }, + )) + } + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn find_account_for_transparent_address( + &self, + address: &TransparentAddress, + ) -> Result, Error> { + self.accounts.find_account_for_transparent_address(address) + } + + pub(crate) fn mark_transparent_output_spent( + &mut self, + spent_in_tx: &TxId, + outpoint: &OutPoint, + ) -> Result { + // TODO: Remove it from the search queue + + self.transparent_received_output_spends + .insert(outpoint.clone(), *spent_in_tx); + + // TODO: Check if this is an update and therefore we need to add something to transparent_spend_map + + Ok(false) + } + + #[cfg(feature = "transparent-inputs")] + pub(crate) fn put_transparent_output( + &mut self, + output: &WalletTransparentOutput, + receiving_account: &AccountId, + known_unspent: bool, + ) -> Result { + use transparent::ReceivedTransparentOutput; + + let address = output.recipient_address(); + // get the block height of the block that mined the output only if we have it in the block table + // otherwise return None + let block = output + .mined_height() + .and_then(|h| self.blocks.get(&h).map(|b| b.height)); + let txid = TxId::from_bytes(output.outpoint().hash().to_vec().try_into().unwrap()); + + // insert a new tx into the transactions table for the one that spent this output. If there is already one then do an update + self.tx_table + .put_tx_partial(&txid, &block, output.mined_height()); + + // look for a spent_height for this output by querying transparent_received_output_spends. + // If there isn't one then return None (this is an unspent output) + // otherwise return the height found by joining on the tx table + let spent_height = self + .transparent_received_output_spends + .get(output.outpoint()) + .and_then(|txid| { + self.tx_table + .tx_status(txid) + .and_then(|status| match status { + TransactionStatus::Mined(height) => Some(height), + _ => None, + }) + }); + + // The max observed unspent height is either the spending transaction's mined height - 1, or + // the current chain tip height if the UTXO was received via a path that confirmed that it was + // unspent, such as by querying the UTXO set of the network. + let max_observed_unspent = match spent_height { + Some(h) => Some(h - 1), + None => { + if known_unspent { + self.chain_height()? + } else { + None + } + } + }; + + // insert into transparent_received_outputs table. Update if it exists + match self + .transparent_received_outputs + .entry(output.outpoint().clone()) + { + Entry::Occupied(mut entry) => { + entry.get_mut().transaction_id = txid; + entry.get_mut().address = *address; + entry.get_mut().account_id = *receiving_account; + entry.get_mut().txout = output.txout().clone(); + } + Entry::Vacant(entry) => { + entry.insert(ReceivedTransparentOutput::new( + txid, + *receiving_account, + *address, + output.txout().clone(), + max_observed_unspent.unwrap_or(BlockHeight::from(0)), + )); + } + } + + // look in transparent_spend_map for a record of the output already having been spent, then mark it as spent using the + // stored reference to the spending transaction. + if self + .transparent_spend_map + .contains(&txid, output.outpoint()) + { + self.mark_transparent_output_spent(&txid, output.outpoint())?; + } + + Ok(output.outpoint().clone()) + } +} diff --git a/zcash_client_memory/src/types/memory_wallet/serialization.rs b/zcash_client_memory/src/types/memory_wallet/serialization.rs new file mode 100644 index 000000000..89a8e41b1 --- /dev/null +++ b/zcash_client_memory/src/types/memory_wallet/serialization.rs @@ -0,0 +1,433 @@ +use bytes::{Buf, BufMut}; +use consensus::Parameters; +use prost::Message; + +use super::*; +use crate::error::Result; +use crate::proto::memwallet as proto; +use crate::read_optional; +use crate::wallet_commitment_trees::serialization::{tree_from_protobuf, tree_to_protobuf}; + +impl MemoryWalletDb

{ + /// Encode a memory wallet db as a protobuf byte buffer + /// Always uses the latest version of the wire protocol + pub fn encode(&self, buf: &mut B) -> Result<()> { + let proto_wallet: proto::MemoryWallet = self.into(); + proto_wallet.encode(buf)?; + Ok(()) + } + + /// Create a mew memory wallet db from a protobuf encoded byte buffer with version awareness + pub fn decode_new(buf: B, params: P, max_checkpoints: usize) -> Result { + let proto_wallet = proto::MemoryWallet::decode(buf)?; + Self::new_from_proto(proto_wallet, params, max_checkpoints) + } + + /// Build a memory wallet db from protobuf type with version awareness + pub fn new_from_proto( + proto_wallet: proto::MemoryWallet, + params: P, + max_checkpoints: usize, + ) -> Result { + match proto_wallet.version { + 1 => Self::new_from_proto_v1(proto_wallet, params, max_checkpoints), + _ => Err(Error::UnsupportedProtoVersion(1, proto_wallet.version)), + } + } + + fn new_from_proto_v1( + proto_wallet: proto::MemoryWallet, + params: P, + max_checkpoints: usize, + ) -> Result { + if proto_wallet.version != 1 { + return Err(Error::UnsupportedProtoVersion(1, proto_wallet.version)); + } + + let mut wallet = MemoryWalletDb::new(params, max_checkpoints); + + wallet.accounts = { + let proto_accounts = read_optional!(proto_wallet, accounts)?; + let accounts = proto_accounts + .accounts + .into_iter() + .map(|proto_account| { + let id = proto_account.account_id; + let account = Account::try_from(proto_account)?; + Ok((AccountId::from(id), account)) + }) + .collect::>()?; + Ok::(Accounts { + accounts, + nonce: proto_accounts.account_nonce, + }) + }?; + + wallet.blocks = proto_wallet + .blocks + .into_iter() + .map(|proto_block| { + Ok(( + proto_block.height.into(), + MemoryWalletBlock::try_from(proto_block)?, + )) + }) + .collect::>()?; + + wallet.tx_table = TransactionTable( + proto_wallet + .tx_table + .into_iter() + .map(|proto_tx| { + let txid = read_optional!(proto_tx, tx_id)?; + let tx = read_optional!(proto_tx, tx_entry)?; + Ok((txid.try_into()?, tx.try_into()?)) + }) + .collect::>()?, + ); + + wallet.received_notes = ReceivedNoteTable( + proto_wallet + .received_note_table + .into_iter() + .map(ReceivedNote::try_from) + .collect::>()?, + ); + + wallet.received_note_spends = ReceievedNoteSpends( + proto_wallet + .received_note_spends + .into_iter() + .map(|proto_spend| { + let note_id = read_optional!(proto_spend, note_id)?; + let tx_id = read_optional!(proto_spend, tx_id)?; + Ok((note_id.try_into()?, tx_id.try_into()?)) + }) + .collect::>()?, + ); + + wallet.nullifiers = NullifierMap( + proto_wallet + .nullifiers + .into_iter() + .map(|proto_nullifier| { + let block_height = proto_nullifier.block_height.into(); + let tx_index = proto_nullifier.tx_index; + let nullifier = read_optional!(proto_nullifier, nullifier)?.try_into()?; + Ok((nullifier, (block_height, tx_index))) + }) + .collect::>()?, + ); + + wallet.sent_notes = SentNoteTable( + proto_wallet + .sent_notes + .into_iter() + .map(|proto_sent_note| { + let sent_note_id = read_optional!(proto_sent_note, sent_note_id)?; + let sent_note = read_optional!(proto_sent_note, sent_note)?; + Ok((sent_note_id.try_into()?, SentNote::try_from(sent_note)?)) + }) + .collect::>()?, + ); + + wallet.tx_locator = TxLocatorMap( + proto_wallet + .tx_locator + .into_iter() + .map(|proto_locator| { + let block_height = proto_locator.block_height.into(); + let tx_index = proto_locator.tx_index; + let tx_id = read_optional!(proto_locator, tx_id)?.try_into()?; + Ok(((block_height, tx_index), tx_id)) + }) + .collect::>()?, + ); + + wallet.scan_queue = ScanQueue( + proto_wallet + .scan_queue + .into_iter() + .map(|item| item.into()) + .collect(), + ); + + wallet.sapling_tree = + tree_from_protobuf(read_optional!(proto_wallet, sapling_tree)?, 100, 16.into())?; + + wallet.sapling_tree_shard_end_heights = proto_wallet + .sapling_tree_shard_end_heights + .into_iter() + .map(|proto_end_height| { + let address = Address::from_parts( + Level::from(u8::try_from(proto_end_height.level)?), + proto_end_height.index, + ); + let height = proto_end_height.block_height.into(); + Ok((address, height)) + }) + .collect::>()?; + + #[cfg(feature = "orchard")] + { + wallet.orchard_tree = + tree_from_protobuf(read_optional!(proto_wallet, orchard_tree)?, 100, 16.into())?; + }; + + #[cfg(feature = "orchard")] + { + wallet.orchard_tree_shard_end_heights = proto_wallet + .orchard_tree_shard_end_heights + .into_iter() + .map(|proto_end_height| { + let address = Address::from_parts( + Level::from(u8::try_from(proto_end_height.level)?), + proto_end_height.index, + ); + let height = proto_end_height.block_height.into(); + Ok((address, height)) + }) + .collect::>()?; + }; + + wallet.transparent_received_outputs = TransparentReceivedOutputs( + proto_wallet + .transparent_received_outputs + .into_iter() + .map(|proto_output| { + let outpoint = read_optional!(proto_output, outpoint)?; + let output = read_optional!(proto_output, output)?.try_into()?; + Ok((OutPoint::try_from(outpoint)?, output)) + }) + .collect::>()?, + ); + + wallet.transparent_received_output_spends = TransparentReceivedOutputSpends( + proto_wallet + .transparent_received_output_spends + .into_iter() + .map(|proto_spend| { + let outpoint = read_optional!(proto_spend, outpoint)?; + let txid = read_optional!(proto_spend, tx_id)?.try_into()?; + Ok((OutPoint::try_from(outpoint)?, txid)) + }) + .collect::>()?, + ); + + wallet.transparent_spend_map = TransparentSpendCache( + proto_wallet + .transparent_spend_map + .into_iter() + .map(|proto_spend| { + let txid = read_optional!(proto_spend, tx_id)?.try_into()?; + let outpoint = read_optional!(proto_spend, outpoint)?; + Ok((txid, OutPoint::try_from(outpoint)?)) + }) + .collect::>()?, + ); + + wallet.transaction_data_request_queue = TransactionDataRequestQueue( + proto_wallet + .transaction_data_requests + .into_iter() + .map(TryInto::try_into) + .collect::>()?, + ); + + Ok(wallet) + } +} + +impl From<&TxId> for proto::TxId { + fn from(txid: &TxId) -> Self { + proto::TxId { + hash: txid.as_ref().to_vec(), + } + } +} + +impl From for proto::TxId { + fn from(txid: TxId) -> Self { + proto::TxId { + hash: txid.as_ref().to_vec(), + } + } +} + +impl TryFrom for TxId { + type Error = Error; + + fn try_from(txid: proto::TxId) -> Result { + Ok(TxId::from_bytes(txid.hash.try_into()?)) + } +} + +impl From<&MemoryWalletDb

> for proto::MemoryWallet { + fn from(wallet: &MemoryWalletDb

) -> Self { + Self { + version: 1, + accounts: Some(proto::Accounts { + accounts: wallet + .accounts + .accounts + .clone() + .into_values() + .map(proto::Account::from) + .collect(), + account_nonce: wallet.accounts.nonce, + }), + + blocks: wallet + .blocks + .clone() + .into_values() + .map(proto::WalletBlock::from) + .collect(), + + tx_table: wallet + .tx_table + .0 + .clone() + .into_iter() + .map(|(txid, tx)| proto::TransactionTableRecord { + tx_id: Some(txid.into()), + tx_entry: Some(tx.into()), + }) + .collect(), + + received_note_table: wallet + .received_notes + .iter() + .map(|note| proto::ReceivedNote::from(note.clone())) + .collect(), + + received_note_spends: wallet + .received_note_spends + .0 + .clone() + .into_iter() + .map(|(note_id, tx_id)| proto::ReceivedNoteSpendRecord { + note_id: Some(note_id.into()), + tx_id: Some(tx_id.into()), + }) + .collect(), + + nullifiers: wallet + .nullifiers + .0 + .clone() + .into_iter() + .map(|(nullifier, (height, tx_index))| proto::NullifierRecord { + block_height: height.into(), + tx_index, + nullifier: Some(nullifier.into()), + }) + .collect(), + + sent_notes: wallet + .sent_notes + .0 + .clone() + .into_iter() + .map(|(id, note)| proto::SentNoteRecord { + sent_note_id: Some(id.into()), + sent_note: Some(proto::SentNote::from(note)), + }) + .collect(), + + tx_locator: wallet + .tx_locator + .0 + .clone() + .into_iter() + .map(|((height, tx_index), txid)| proto::TxLocatorRecord { + block_height: height.into(), + tx_index, + tx_id: Some(txid.into()), + }) + .collect(), + + scan_queue: wallet + .scan_queue + .iter() + .map(|r| proto::ScanQueueRecord::from(*r)) + .collect(), + + sapling_tree: tree_to_protobuf(&wallet.sapling_tree).unwrap(), + sapling_tree_shard_end_heights: wallet + .sapling_tree_shard_end_heights + .clone() + .into_iter() + .map(|(address, height)| proto::TreeEndHeightsRecord { + level: address.level().into(), + index: address.index(), + block_height: height.into(), + }) + .collect(), + + #[cfg(feature = "orchard")] + orchard_tree: tree_to_protobuf(&wallet.orchard_tree).unwrap(), + #[cfg(not(feature = "orchard"))] + orchard_tree: None, + + #[cfg(feature = "orchard")] + orchard_tree_shard_end_heights: wallet + .orchard_tree_shard_end_heights + .clone() + .into_iter() + .map(|(address, height)| proto::TreeEndHeightsRecord { + level: address.level().into(), + index: address.index(), + block_height: height.into(), + }) + .collect(), + #[cfg(not(feature = "orchard"))] + orchard_tree_shard_end_heights: Vec::new(), + + transparent_received_outputs: wallet + .transparent_received_outputs + .0 + .clone() + .into_iter() + .map( + |(outpoint, output)| proto::TransparentReceivedOutputRecord { + outpoint: Some(proto::OutPoint::from(outpoint)), + output: Some(proto::ReceivedTransparentOutput::from(output)), + }, + ) + .collect(), + + transparent_received_output_spends: wallet + .transparent_received_output_spends + .0 + .clone() + .into_iter() + .map( + |(outpoint, txid)| proto::TransparentReceivedOutputSpendRecord { + outpoint: Some(proto::OutPoint::from(outpoint)), + tx_id: Some(txid.into()), + }, + ) + .collect(), + + transparent_spend_map: wallet + .transparent_spend_map + .0 + .clone() + .into_iter() + .map(|(txid, outpoint)| proto::TransparentSpendCacheRecord { + tx_id: Some(txid.into()), + outpoint: Some(proto::OutPoint::from(outpoint)), + }) + .collect(), + + transaction_data_requests: wallet + .transaction_data_request_queue + .0 + .clone() + .into_iter() + .map(Into::into) + .collect(), + } + } +} diff --git a/zcash_client_memory/src/types/mod.rs b/zcash_client_memory/src/types/mod.rs new file mode 100644 index 000000000..c9308aff8 --- /dev/null +++ b/zcash_client_memory/src/types/mod.rs @@ -0,0 +1,17 @@ +pub(crate) mod account; +pub(crate) mod block; +pub(crate) mod data_requests; +pub(crate) mod memory_wallet; +pub(crate) mod notes; +pub(crate) mod nullifier; +pub(crate) mod scanning; +pub(crate) mod transaction; +pub(crate) mod transparent; + +pub(crate) use account::*; +pub(crate) use block::*; +pub(crate) use data_requests::*; +pub use memory_wallet::*; +pub(crate) use notes::*; +pub(crate) use nullifier::*; +pub(crate) use transaction::*; diff --git a/zcash_client_memory/src/types/notes/mod.rs b/zcash_client_memory/src/types/notes/mod.rs new file mode 100644 index 000000000..003f9bc2a --- /dev/null +++ b/zcash_client_memory/src/types/notes/mod.rs @@ -0,0 +1,160 @@ +mod received; +mod sent; + +pub(crate) use received::{ + to_spendable_notes, ReceievedNoteSpends, ReceivedNote, ReceivedNoteTable, +}; +#[cfg(test)] +pub(crate) use sent::SentNoteId; +pub(crate) use sent::{SentNote, SentNoteTable}; + +mod serialization { + use crate::error::Error; + use crate::proto::memwallet::{self as proto}; + use crate::read_optional; + use jubjub::Fr; + use zcash_client_backend::wallet::{Note, NoteId}; + + impl From for proto::NoteId { + fn from(note_id: NoteId) -> Self { + Self { + tx_id: Some(note_id.txid().into()), + pool: match note_id.protocol() { + zcash_protocol::ShieldedProtocol::Sapling => { + proto::PoolType::ShieldedSapling.into() + } + #[cfg(feature = "orchard")] + zcash_protocol::ShieldedProtocol::Orchard => { + proto::PoolType::ShieldedOrchard.into() + } + #[cfg(not(feature = "orchard"))] + zcash_protocol::ShieldedProtocol::Orchard => panic!("Attempting to deserialize orchard supporting wallet using library built without orchard feature"), + }, + output_index: note_id.output_index() as u32, + } + } + } + + impl TryFrom for NoteId { + type Error = Error; + fn try_from(note_id: proto::NoteId) -> Result { + Ok(Self::new( + read_optional!(note_id.clone(), tx_id)?.try_into()?, + match note_id.pool() { + proto::PoolType::ShieldedSapling => zcash_protocol::ShieldedProtocol::Sapling, + #[cfg(feature = "orchard")] + proto::PoolType::ShieldedOrchard => zcash_protocol::ShieldedProtocol::Orchard, + _ => panic!("invalid pool"), + }, + note_id.output_index.try_into()?, + )) + } + } + + impl From for proto::Note { + fn from(note: Note) -> Self { + match note { + Note::Sapling(note) => Self { + protocol: proto::ShieldedProtocol::Sapling.into(), + recipient: note.recipient().to_bytes().to_vec(), + value: note.value().inner(), + rseed: match note.rseed() { + sapling::Rseed::AfterZip212(inner) => Some(proto::RSeed { + rseed_type: Some(proto::RSeedType::AfterZip212 as i32), + payload: inner.to_vec(), + }), + sapling::Rseed::BeforeZip212(inner) => Some(proto::RSeed { + rseed_type: Some(proto::RSeedType::BeforeZip212 as i32), + payload: inner.to_bytes().to_vec(), + }), + }, + rho: None, + }, + #[cfg(feature = "orchard")] + Note::Orchard(note) => Self { + protocol: proto::ShieldedProtocol::Orchard.into(), + recipient: note.recipient().to_raw_address_bytes().to_vec(), + value: note.value().inner(), + rseed: Some(proto::RSeed { + rseed_type: None, + payload: note.rseed().as_bytes().to_vec(), + }), + rho: Some(note.rho().to_bytes().to_vec()), + }, + } + } + } + + impl From for Note { + fn from(note: proto::Note) -> Self { + match note.protocol() { + proto::ShieldedProtocol::Sapling => { + let recipient = + sapling::PaymentAddress::from_bytes(¬e.recipient.try_into().unwrap()) + .unwrap(); + let value = sapling::value::NoteValue::from_raw(note.value); + let rseed = match note.rseed { + Some(proto::RSeed { + rseed_type: Some(0), + payload, + }) => sapling::Rseed::BeforeZip212( + Fr::from_bytes(&payload.try_into().unwrap()).unwrap(), + ), + Some(proto::RSeed { + rseed_type: Some(1), + payload, + }) => sapling::Rseed::AfterZip212(payload.try_into().unwrap()), + _ => panic!("rseed is required"), + }; + Self::Sapling(sapling::Note::from_parts(recipient, value, rseed)) + } + #[cfg(feature = "orchard")] + proto::ShieldedProtocol::Orchard => { + let recipient = orchard::Address::from_raw_address_bytes( + ¬e.recipient.try_into().unwrap(), + ) + .unwrap(); + let value = orchard::value::NoteValue::from_raw(note.value); + let rho = + orchard::note::Rho::from_bytes(¬e.rho.unwrap().try_into().unwrap()) + .unwrap(); + let rseed = orchard::note::RandomSeed::from_bytes( + note.rseed.unwrap().payload.try_into().unwrap(), + &rho, + ) + .unwrap(); + Self::Orchard(orchard::Note::from_parts(recipient, value, rho, rseed).unwrap()) + } + #[cfg(not(feature = "orchard"))] + _ => panic!("invalid protocol"), + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + use crate::proto::memwallet as proto; + use pretty_assertions::assert_eq; + + #[test] + fn test_note_roundtrip() { + let note = Note::Sapling(sapling::note::Note::from_parts( + sapling::PaymentAddress::from_bytes(&[ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x8e, + 0x11, 0x9d, 0x72, 0x99, 0x2b, 0x56, 0x0d, 0x26, 0x50, 0xff, 0xe0, 0xbe, 0x7f, + 0x35, 0x42, 0xfd, 0x97, 0x00, 0x3c, 0xb7, 0xcc, 0x3a, 0xbf, 0xf8, 0x1a, 0x7f, + 0x90, 0x37, 0xf3, 0xea, + ]) + .unwrap(), + sapling::value::NoteValue::from_raw(99), + sapling::Rseed::AfterZip212([0; 32]), + )); + + let proto_note: proto::Note = note.clone().into(); + let recovered: Note = proto_note.into(); + + assert_eq!(note, recovered); + } + } +} diff --git a/zcash_client_memory/src/types/notes/received.rs b/zcash_client_memory/src/types/notes/received.rs new file mode 100644 index 000000000..d7fe6ecd8 --- /dev/null +++ b/zcash_client_memory/src/types/notes/received.rs @@ -0,0 +1,411 @@ +use incrementalmerkletree::Position; + +use std::collections::BTreeSet; +use std::{ + collections::BTreeMap, + ops::{Deref, DerefMut}, +}; +use zip32::Scope; + +use zcash_primitives::transaction::TxId; +use zcash_protocol::{memo::Memo, PoolType, ShieldedProtocol::Sapling}; + +use zcash_client_backend::{ + data_api::{SentTransactionOutput, SpendableNotes}, + wallet::{Note, NoteId, Recipient, WalletSaplingOutput}, +}; + +use crate::AccountId; + +#[cfg(feature = "orchard")] +use { + zcash_client_backend::wallet::WalletOrchardOutput, zcash_protocol::ShieldedProtocol::Orchard, +}; + +use crate::{error::Error, Nullifier}; + +/// Keeps track of notes that are spent in which transaction +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ReceievedNoteSpends(pub(crate) BTreeMap); + +impl ReceievedNoteSpends { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + pub fn insert_spend(&mut self, note_id: NoteId, txid: TxId) -> Option { + self.0.insert(note_id, txid) + } + pub fn get(&self, note_id: &NoteId) -> Option<&TxId> { + self.0.get(note_id) + } +} + +impl Deref for ReceievedNoteSpends { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// A note that has been received by the wallet +/// TODO: Instead of Vec, perhaps we should identify by some unique ID +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ReceivedNoteTable(pub(crate) Vec); + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ReceivedNote { + // Uniquely identifies this note + pub(crate) note_id: NoteId, + pub(crate) txid: TxId, + // output_index: sapling, action_index: orchard + pub(crate) output_index: u32, + pub(crate) account_id: AccountId, + //sapling: (diversifier, value, rcm) orchard: (diversifier, value, rho, rseed) + pub(crate) note: Note, + pub(crate) nf: Option, + pub(crate) is_change: bool, + pub(crate) memo: Memo, + pub(crate) commitment_tree_position: Option, + pub(crate) recipient_key_scope: Option, +} +impl ReceivedNote { + pub fn pool(&self) -> PoolType { + match self.note { + Note::Sapling { .. } => PoolType::SAPLING, + #[cfg(feature = "orchard")] + Note::Orchard { .. } => PoolType::ORCHARD, + } + } + pub fn account_id(&self) -> AccountId { + self.account_id + } + pub fn nullifier(&self) -> Option<&Nullifier> { + self.nf.as_ref() + } + pub fn txid(&self) -> TxId { + self.txid + } + pub fn note_id(&self) -> NoteId { + self.note_id + } + pub fn from_sent_tx_output( + txid: TxId, + output: &SentTransactionOutput, + ) -> Result { + match output.recipient() { + Recipient::InternalAccount { + receiving_account, + note: Note::Sapling(note), + .. + } => Ok(ReceivedNote { + note_id: NoteId::new(txid, Sapling, output.output_index() as u16), + txid, + output_index: output.output_index() as u32, + account_id: *receiving_account, + note: Note::Sapling(note.clone()), + nf: None, + is_change: true, + memo: output + .memo() + .map(Memo::try_from) + .transpose()? + .expect("expected a memo for a non-transparent output"), + commitment_tree_position: None, + recipient_key_scope: Some(Scope::Internal), + }), + #[cfg(feature = "orchard")] + Recipient::InternalAccount { + receiving_account, + note: Note::Orchard(note), + .. + } => Ok(ReceivedNote { + note_id: NoteId::new(txid, Orchard, output.output_index() as u16), + txid, + output_index: output.output_index() as u32, + account_id: *receiving_account, + note: Note::Orchard(*note), + nf: None, + is_change: true, + memo: output + .memo() + .map(Memo::try_from) + .transpose()? + .expect("expected a memo for a non-transparent output"), + commitment_tree_position: None, + recipient_key_scope: Some(Scope::Internal), + }), + _ => Err(Error::Other( + "Recipient is not an internal shielded account".to_owned(), + )), + } + } + pub fn from_wallet_sapling_output( + note_id: NoteId, + output: &WalletSaplingOutput, + ) -> Self { + ReceivedNote { + note_id, + txid: *note_id.txid(), + output_index: output.index() as u32, + account_id: *output.account_id(), + note: Note::Sapling(output.note().clone()), + nf: output.nf().map(|nf| Nullifier::Sapling(*nf)), + is_change: output.is_change(), + memo: Memo::Empty, + commitment_tree_position: Some(output.note_commitment_tree_position()), + recipient_key_scope: output.recipient_key_scope(), + } + } + #[cfg(feature = "orchard")] + pub fn from_wallet_orchard_output( + note_id: NoteId, + output: &WalletOrchardOutput, + ) -> Self { + ReceivedNote { + note_id, + txid: *note_id.txid(), + output_index: output.index() as u32, + account_id: *output.account_id(), + note: Note::Orchard(*output.note()), + nf: output.nf().map(|nf| Nullifier::Orchard(*nf)), + is_change: output.is_change(), + memo: Memo::Empty, + commitment_tree_position: Some(output.note_commitment_tree_position()), + recipient_key_scope: output.recipient_key_scope(), + } + } +} + +impl From + for zcash_client_backend::wallet::ReceivedNote +{ + fn from(value: ReceivedNote) -> Self { + zcash_client_backend::wallet::ReceivedNote::from_parts( + value.note_id, + value.txid, + value.output_index.try_into().unwrap(), + value.note, + value.recipient_key_scope.unwrap(), + value.commitment_tree_position.unwrap(), + ) + } +} + +impl ReceivedNoteTable { + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn get_sapling_nullifiers( + &self, + ) -> impl Iterator + '_ { + self.0.iter().filter_map(|entry| { + if let Some(Nullifier::Sapling(nf)) = entry.nullifier() { + Some((entry.account_id(), entry.txid(), *nf)) + } else { + None + } + }) + } + #[cfg(feature = "orchard")] + pub fn get_orchard_nullifiers( + &self, + ) -> impl Iterator + '_ { + self.0.iter().filter_map(|entry| { + if let Some(Nullifier::Orchard(nf)) = entry.nullifier() { + Some((entry.account_id(), entry.txid(), *nf)) + } else { + None + } + }) + } + + pub fn insert_received_note(&mut self, note: ReceivedNote) { + // ensure note_id is unique. + // follow upsert rules to update the note if it already exists + let is_absent = self + .0 + .iter_mut() + .find(|n| n.note_id == note.note_id) + .map(|n| { + n.nf = note.nf.or(n.nf); + n.is_change = note.is_change || n.is_change; + n.commitment_tree_position = + note.commitment_tree_position.or(n.commitment_tree_position); + }) + .is_none(); + + if is_absent { + self.0.push(note); + } + } + + #[cfg(feature = "orchard")] + pub fn detect_orchard_spending_accounts<'a>( + &self, + nfs: impl Iterator, + ) -> Result, Error> { + let mut acc = BTreeSet::new(); + let nfs = nfs.collect::>(); + for (nf, id) in self.0.iter().filter_map(|n| match (n.nf, n.account_id) { + (Some(Nullifier::Orchard(nf)), account_id) => Some((nf, account_id)), + _ => None, + }) { + if nfs.contains(&&nf) { + acc.insert(id); + } + } + Ok(acc) + } + + pub fn detect_sapling_spending_accounts<'a>( + &self, + nfs: impl Iterator, + ) -> Result, Error> { + let mut acc = BTreeSet::new(); + let nfs = nfs.collect::>(); + for (nf, id) in self.0.iter().filter_map(|n| match (n.nf, n.account_id) { + (Some(Nullifier::Sapling(nf)), account_id) => Some((nf, account_id)), + _ => None, + }) { + if nfs.contains(&&nf) { + acc.insert(id); + } + } + Ok(acc) + } +} + +// We deref to slice so that we can reuse the slice impls +impl Deref for ReceivedNoteTable { + type Target = [ReceivedNote]; + + fn deref(&self) -> &Self::Target { + &self.0[..] + } +} +impl DerefMut for ReceivedNoteTable { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0[..] + } +} + +pub(crate) fn to_spendable_notes( + sapling_received_notes: &[&ReceivedNote], + #[cfg(feature = "orchard")] orchard_received_notes: &[&ReceivedNote], +) -> Result, Error> { + let sapling = sapling_received_notes + .iter() + .map(|note| { + #[allow(irrefutable_let_patterns)] + if let Note::Sapling(inner) = ¬e.note { + Ok(zcash_client_backend::wallet::ReceivedNote::from_parts( + note.note_id, + note.txid(), + note.output_index.try_into().unwrap(), // this overflow can never happen or else the chain is broken + inner.clone(), + note.recipient_key_scope + .ok_or(Error::Missing("recipient key scope".into()))?, + note.commitment_tree_position + .ok_or(Error::Missing("commitment tree position".into()))?, + )) + } else { + Err(Error::Other("Note is not a sapling note".to_owned())) + } + }) + .collect::, _>>()?; + + #[cfg(feature = "orchard")] + let orchard = orchard_received_notes + .iter() + .map(|note| { + if let Note::Orchard(inner) = ¬e.note { + Ok(zcash_client_backend::wallet::ReceivedNote::from_parts( + note.note_id, + note.txid(), + note.output_index.try_into().unwrap(), // this overflow can never happen or else the chain is broken + *inner, + note.recipient_key_scope + .ok_or(Error::Missing("recipient key scope".into()))?, + note.commitment_tree_position + .ok_or(Error::Missing("commitment tree position".into()))?, + )) + } else { + Err(Error::Other("Note is not an orchard note".to_owned())) + } + }) + .collect::, _>>()?; + + Ok(SpendableNotes::new( + sapling, + #[cfg(feature = "orchard")] + orchard, + )) +} + +#[derive(PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum SentNoteId { + Shielded(NoteId), +} + +impl From for SentNoteId { + fn from(note_id: NoteId) -> Self { + SentNoteId::Shielded(note_id) + } +} + +impl From<&NoteId> for SentNoteId { + fn from(note_id: &NoteId) -> Self { + SentNoteId::Shielded(*note_id) + } +} + +mod serialization { + use super::*; + use crate::{proto::memwallet as proto, read_optional}; + + impl From for proto::ReceivedNote { + fn from(value: ReceivedNote) -> Self { + Self { + note_id: Some(value.note_id.into()), + tx_id: Some(value.txid.into()), + output_index: value.output_index, + account_id: *value.account_id, + note: Some(value.note.into()), + nullifier: value.nf.map(|nf| nf.into()), + is_change: value.is_change, + memo: value.memo.encode().as_array().to_vec(), + commitment_tree_position: value.commitment_tree_position.map(|pos| pos.into()), + recipient_key_scope: match value.recipient_key_scope { + Some(Scope::Internal) => Some(proto::Scope::Internal as i32), + Some(Scope::External) => Some(proto::Scope::External as i32), + None => None, + }, + } + } + } + + impl TryFrom for ReceivedNote { + type Error = Error; + + fn try_from(value: proto::ReceivedNote) -> Result { + Ok(Self { + note_id: read_optional!(value, note_id)?.try_into()?, + txid: read_optional!(value, tx_id)?.try_into()?, + output_index: value.output_index, + account_id: value.account_id.into(), + note: read_optional!(value, note)?.into(), + nf: value.nullifier.map(|nf| nf.try_into()).transpose()?, + is_change: value.is_change, + memo: Memo::from_bytes(&value.memo)?, + commitment_tree_position: value.commitment_tree_position.map(|pos| pos.into()), + recipient_key_scope: match value.recipient_key_scope { + Some(0) => Some(Scope::Internal), + Some(1) => Some(Scope::External), + _ => None, + }, + }) + } + } +} diff --git a/zcash_client_memory/src/types/notes/sent.rs b/zcash_client_memory/src/types/notes/sent.rs new file mode 100644 index 000000000..2d4661eca --- /dev/null +++ b/zcash_client_memory/src/types/notes/sent.rs @@ -0,0 +1,371 @@ +use std::{collections::BTreeMap, ops::Deref}; + +use zcash_primitives::transaction::{components::OutPoint, TxId}; +use zcash_protocol::{memo::Memo, value::Zatoshis, PoolType, ShieldedProtocol::Sapling}; + +use zcash_client_backend::{ + data_api::{SentTransaction, SentTransactionOutput}, + wallet::{Note, NoteId, Recipient}, +}; + +use crate::AccountId; + +#[cfg(feature = "orchard")] +use zcash_protocol::ShieldedProtocol::Orchard; + +#[derive(PartialEq, PartialOrd, Eq, Ord, Debug, Clone)] +pub enum SentNoteId { + Shielded(NoteId), + Transparent { txid: TxId, output_index: u32 }, +} + +impl From for SentNoteId { + fn from(note_id: NoteId) -> Self { + SentNoteId::Shielded(note_id) + } +} + +impl From<&NoteId> for SentNoteId { + fn from(note_id: &NoteId) -> Self { + SentNoteId::Shielded(*note_id) + } +} + +impl SentNoteId { + pub fn txid(&self) -> &TxId { + match self { + SentNoteId::Shielded(note_id) => note_id.txid(), + SentNoteId::Transparent { txid, .. } => txid, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SentNoteTable(pub(crate) BTreeMap); + +impl SentNoteTable { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + + pub fn insert_sent_output( + &mut self, + tx: &SentTransaction, + output: &SentTransactionOutput, + ) { + let pool_type = match output.recipient() { + Recipient::External(_, pool_type) => *pool_type, + Recipient::EphemeralTransparent { .. } => PoolType::Transparent, + Recipient::InternalAccount { note, .. } => PoolType::Shielded(note.protocol()), + }; + match pool_type { + PoolType::Transparent => { + // we kind of are in a tricky spot here since NoteId cannot represent a transparent note.. + // just make it a sapling one for now until we figure out a better way to represent this + let note_id = SentNoteId::Transparent { + txid: tx.tx().txid(), + output_index: output.output_index().try_into().unwrap(), + }; + self.0.insert( + note_id, + SentNote { + from_account_id: *tx.account_id(), + to: output.recipient().clone(), + value: output.value(), + memo: Memo::Empty, // transparent notes don't have memos + }, + ); + } + PoolType::Shielded(protocol) => { + let note_id = NoteId::new( + tx.tx().txid(), + protocol, + output.output_index().try_into().unwrap(), + ); + self.0.insert( + note_id.into(), + SentNote { + from_account_id: *tx.account_id(), + to: output.recipient().clone(), + value: output.value(), + memo: output.memo().map(|m| Memo::try_from(m).unwrap()).unwrap(), + }, + ); + } + } + } + + pub fn put_sent_output( + &mut self, + txid: TxId, + from_account_id: AccountId, + output: &SentTransactionOutput, + ) { + let pool_type = match output.recipient() { + Recipient::External(_, pool_type) => *pool_type, + Recipient::EphemeralTransparent { .. } => PoolType::Transparent, + Recipient::InternalAccount { note, .. } => PoolType::Shielded(note.protocol()), + }; + match pool_type { + PoolType::Transparent => { + // we kind of are in a tricky spot here since NoteId cannot represent a transparent note.. + // just make it a sapling one for now until we figure out a better way to represent this + let note_id = SentNoteId::Transparent { + txid, + output_index: output.output_index().try_into().unwrap(), + }; + self.0.insert( + note_id, + SentNote { + from_account_id, + to: output.recipient().clone(), + value: output.value(), + memo: Memo::Empty, // transparent notes don't have memos + }, + ); + } + PoolType::Shielded(protocol) => { + let note_id = + NoteId::new(txid, protocol, output.output_index().try_into().unwrap()); + self.0.insert( + note_id.into(), + SentNote { + from_account_id, + to: output.recipient().clone(), + value: output.value(), + memo: output.memo().map(|m| Memo::try_from(m).unwrap()).unwrap(), + }, + ); + } + } + } + + pub fn get_sent_note(&self, note_id: &NoteId) -> Option<&SentNote> { + self.0.get(¬e_id.into()) + } +} + +impl Deref for SentNoteTable { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SentNote { + pub(crate) from_account_id: AccountId, + pub(crate) to: Recipient, + pub(crate) value: Zatoshis, + pub(crate) memo: Memo, +} + +mod serialization { + use super::*; + use crate::{error::Error, proto::memwallet as proto, read_optional}; + use zcash_address::ZcashAddress; + use zcash_keys::encoding::AddressCodec; + use zcash_primitives::{ + consensus::Network::MainNetwork as EncodingParams, legacy::TransparentAddress, + }; + use zcash_protocol::ShieldedProtocol; + + impl From for proto::SentNote { + fn from(note: SentNote) -> Self { + Self { + from_account_id: *note.from_account_id, + to: Some(note.to.into()), + value: note.value.into(), + memo: note.memo.encode().as_array().to_vec(), + } + } + } + + impl TryFrom for SentNote { + type Error = crate::Error; + + fn try_from(note: proto::SentNote) -> Result { + Ok(Self { + from_account_id: note.from_account_id.into(), + to: read_optional!(note, to)?.try_into()?, + value: Zatoshis::from_u64(note.value)?, + memo: Memo::from_bytes(¬e.memo)?, + }) + } + } + + impl From for proto::NoteId { + fn from(note_id: SentNoteId) -> Self { + match note_id { + SentNoteId::Shielded(note_id) => proto::NoteId { + tx_id: Some(note_id.txid().into()), + output_index: note_id.output_index().into(), + pool: match note_id.protocol() { + ShieldedProtocol::Sapling => proto::PoolType::ShieldedSapling as i32, + ShieldedProtocol::Orchard => proto::PoolType::ShieldedOrchard as i32, + }, + }, + SentNoteId::Transparent { txid, output_index } => proto::NoteId { + tx_id: Some(txid.into()), + output_index, + pool: proto::PoolType::Transparent as i32, + }, + } + } + } + + impl TryFrom for SentNoteId { + type Error = Error; + + fn try_from(note_id: proto::NoteId) -> Result { + Ok(match note_id.pool() { + proto::PoolType::ShieldedSapling => SentNoteId::Shielded(NoteId::new( + read_optional!(note_id, tx_id)?.try_into()?, + Sapling, + note_id.output_index.try_into()?, + )), + #[cfg(feature = "orchard")] + proto::PoolType::ShieldedOrchard => SentNoteId::Shielded(NoteId::new( + read_optional!(note_id, tx_id)?.try_into()?, + Orchard, + note_id.output_index.try_into()?, + )), + #[cfg(not(feature = "orchard"))] + proto::PoolType::ShieldedOrchard => return Err(Error::OrchardNotEnabled), + proto::PoolType::Transparent => SentNoteId::Transparent { + txid: read_optional!(note_id, tx_id)?.try_into()?, + output_index: note_id.output_index, + }, + }) + } + } + + impl From for proto::OutPoint { + fn from(outpoint: OutPoint) -> Self { + Self { + hash: outpoint.txid().as_ref().to_vec(), + n: outpoint.n(), + } + } + } + + impl TryFrom for OutPoint { + type Error = Error; + + fn try_from(outpoint: proto::OutPoint) -> Result { + Ok(Self::new(outpoint.hash.try_into()?, outpoint.n)) + } + } + + impl From> for proto::Recipient { + fn from(recipient: Recipient) -> Self { + match recipient { + Recipient::External(address, pool_type) => proto::Recipient { + recipient_type: proto::RecipientType::ExternalRecipient as i32, + + address: Some(address.to_string()), + pool_type: Some(match pool_type { + PoolType::Transparent => proto::PoolType::Transparent, + PoolType::Shielded(Sapling) => proto::PoolType::ShieldedSapling, + #[cfg(feature = "orchard")] + PoolType::Shielded(Orchard) => proto::PoolType::ShieldedOrchard, + #[cfg(not(feature = "orchard"))] + _ => panic!("Orchard not enabled"), + } as i32), + + account_id: None, + outpoint_metadata: None, + note: None, + }, + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + } => proto::Recipient { + recipient_type: proto::RecipientType::EphemeralTransparent as i32, + + address: Some(ephemeral_address.encode(&EncodingParams)), + pool_type: Some(proto::PoolType::Transparent as i32), + + account_id: Some(*receiving_account), + outpoint_metadata: Some(outpoint_metadata.into()), + note: None, + }, + Recipient::InternalAccount { + receiving_account, + external_address, + note, + } => proto::Recipient { + recipient_type: proto::RecipientType::InternalAccount as i32, + + address: external_address.map(|a| a.to_string()), + pool_type: None, + + account_id: Some(*receiving_account), + outpoint_metadata: None, + note: Some(note.into()), + }, + } + } + } + + impl TryFrom for Recipient { + type Error = Error; + fn try_from(recipient: proto::Recipient) -> Result { + Ok(match recipient.recipient_type() { + proto::RecipientType::ExternalRecipient => { + let address_str = read_optional!(recipient.clone(), address)?; + let address = ZcashAddress::try_from_encoded(&address_str)?; + Recipient::External( + address, + match recipient.pool_type() { + proto::PoolType::Transparent => PoolType::Transparent, + proto::PoolType::ShieldedSapling => PoolType::Shielded(Sapling), + #[cfg(feature = "orchard")] + proto::PoolType::ShieldedOrchard => PoolType::Shielded(Orchard), + #[cfg(not(feature = "orchard"))] + proto::PoolType::ShieldedOrchard => { + return Err(Error::OrchardNotEnabled) + } + }, + ) + } + proto::RecipientType::EphemeralTransparent => Recipient::EphemeralTransparent { + receiving_account: read_optional!(recipient, account_id)?.into(), + ephemeral_address: TransparentAddress::decode( + &EncodingParams, + &read_optional!(recipient, address)?, + )?, + outpoint_metadata: read_optional!(recipient, outpoint_metadata)?.try_into()?, + }, + proto::RecipientType::InternalAccount => Recipient::InternalAccount { + receiving_account: read_optional!(recipient, account_id)?.into(), + external_address: recipient.address.map(|a| a.parse()).transpose()?, + note: read_optional!(recipient, note)?.into(), + }, + }) + } + } + + #[cfg(test)] + mod tests { + use super::*; + use crate::proto::memwallet as proto; + use zcash_primitives::transaction::components::OutPoint; + use zcash_protocol::ShieldedProtocol; + + #[test] + fn proto_roundtrip_recipient() { + let recipient = Recipient::::External( + ZcashAddress::try_from_encoded("uregtest1a7mkafdn9c87xywjnyup65uker8tx3y72r9f6elcfm6uh263c9s6smcw6xm5m8k8eythcreuyqktp9z7mtpcd6jsm5xw7skgdcfjx84z").unwrap(), + PoolType::Shielded(ShieldedProtocol::Sapling), + ); + let proto = proto::Recipient::from(recipient.clone()); + let recipient2 = + Recipient::::try_from(proto.clone()).unwrap(); + let proto2 = proto::Recipient::from(recipient2.clone()); + assert_eq!(proto, proto2); + } + } +} diff --git a/zcash_client_memory/src/types/nullifier.rs b/zcash_client_memory/src/types/nullifier.rs new file mode 100644 index 000000000..53671393a --- /dev/null +++ b/zcash_client_memory/src/types/nullifier.rs @@ -0,0 +1,98 @@ +use std::{collections::BTreeMap, ops::Deref}; + +use zcash_primitives::consensus::BlockHeight; +use zcash_protocol::PoolType; + +/// Maps a nullifier to the block height and transaction index (NOT txid!) where it was spent. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct NullifierMap(pub(crate) BTreeMap); + +impl NullifierMap { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + pub fn insert(&mut self, height: BlockHeight, index: u32, nullifier: Nullifier) { + self.0.insert(nullifier, (height, index)); + } + + pub fn get(&self, nullifier: &Nullifier) -> Option<&(BlockHeight, u32)> { + self.0.get(nullifier) + } +} + +impl Deref for NullifierMap { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Nullifier { + Sapling(sapling::Nullifier), + #[cfg(feature = "orchard")] + Orchard(orchard::note::Nullifier), +} + +impl Nullifier { + pub(crate) fn _pool(&self) -> PoolType { + match self { + Nullifier::Sapling(_) => PoolType::SAPLING, + #[cfg(feature = "orchard")] + Nullifier::Orchard(_) => PoolType::ORCHARD, + } + } +} + +impl From for Nullifier { + fn from(n: sapling::Nullifier) -> Self { + Nullifier::Sapling(n) + } +} + +#[cfg(feature = "orchard")] +impl From for Nullifier { + fn from(n: orchard::note::Nullifier) -> Self { + Nullifier::Orchard(n) + } +} + +mod serialization { + use super::*; + use crate::{proto::memwallet as proto, Error}; + + impl From for proto::Nullifier { + fn from(nullifier: Nullifier) -> Self { + match nullifier { + Nullifier::Sapling(n) => Self { + protocol: proto::ShieldedProtocol::Sapling.into(), + nullifier: n.to_vec(), + }, + #[cfg(feature = "orchard")] + Nullifier::Orchard(n) => Self { + protocol: proto::ShieldedProtocol::Orchard.into(), + nullifier: n.to_bytes().to_vec(), + }, + } + } + } + + impl TryFrom for Nullifier { + type Error = Error; + + fn try_from(nullifier: proto::Nullifier) -> Result { + Ok(match nullifier.protocol() { + proto::ShieldedProtocol::Sapling => { + Nullifier::Sapling(sapling::Nullifier::from_slice(&nullifier.nullifier)?) + } + #[cfg(feature = "orchard")] + proto::ShieldedProtocol::Orchard => Nullifier::Orchard( + orchard::note::Nullifier::from_bytes(&nullifier.nullifier.try_into()?) + .into_option() + .ok_or(Error::CorruptedData("Invalid Orchard nullifier".into()))?, + ), + }) + } + } +} diff --git a/zcash_client_memory/src/types/scanning.rs b/zcash_client_memory/src/types/scanning.rs new file mode 100644 index 000000000..799ab63b8 --- /dev/null +++ b/zcash_client_memory/src/types/scanning.rs @@ -0,0 +1,202 @@ +use std::ops::{Deref, DerefMut, Range}; + +use zcash_client_backend::data_api::scanning::{ + spanning_tree::SpanningTree, ScanPriority, ScanRange, +}; +use zcash_primitives::consensus::BlockHeight; + +use crate::error::Error; + +/// A queue of scanning ranges. Contains the start and end heights of each range, along with the +/// priority of scanning that range. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ScanQueue(pub(crate) Vec<(BlockHeight, BlockHeight, ScanPriority)>); + +impl ScanQueue { + pub(crate) fn new() -> Self { + ScanQueue(Vec::new()) + } + + pub(crate) fn suggest_scan_ranges(&self, min_priority: ScanPriority) -> Vec { + let mut priorities: Vec<_> = self + .0 + .iter() + .filter(|(_, _, p)| *p >= min_priority) + .collect(); + priorities.sort_by(|(_, _, a), (_, _, b)| b.cmp(a)); + + priorities + .into_iter() + .map(|(start, end, priority)| { + let range = Range { + start: *start, + end: *end, + }; + ScanRange::from_parts(range, *priority) + }) + .collect() + } + fn insert_queue_entries<'a>( + &mut self, + entries: impl Iterator, + ) -> Result<(), Error> { + for entry in entries { + if entry.block_range().start >= entry.block_range().end { + return Err(Error::InvalidScanRange( + entry.block_range().start, + entry.block_range().end, + "start must be less than end".to_string(), + )); + } + + for (start, end, _) in &self.0 { + if *start == entry.block_range().start || *end == entry.block_range().end { + return Err(Error::InvalidScanRange( + entry.block_range().start, + entry.block_range().end, + "at least part of range is already covered by another range".to_string(), + )); + } + } + + self.0.push(( + entry.block_range().start, + entry.block_range().end, + entry.priority(), + )); + } + Ok(()) + } + pub(crate) fn replace_queue_entries( + &mut self, + query_range: &Range, + entries: impl Iterator, + force_rescans: bool, + ) -> Result<(), Error> { + let (to_create, to_delete_ends) = { + let mut q_ranges: Vec<_> = self + .0 + .iter() + .filter(|(start, end, _)| { + // Ignore ranges that do not overlap and are not adjacent to the query range. + !(start > &query_range.end || &query_range.start > end) + }) + .collect(); + q_ranges.sort_by(|(_, end_a, _), (_, end_b, _)| end_a.cmp(end_b)); + + // Iterate over the ranges in the scan queue that overlap the range that we have + // identified as needing to be fully scanned. For each such range add it to the + // spanning tree (these should all be nonoverlapping ranges, but we might coalesce + // some in the process). + let mut to_create: Option = None; + let mut to_delete_ends: Vec = vec![]; + + let q_ranges = q_ranges.into_iter(); + for (start, end, priority) in q_ranges { + let entry = ScanRange::from_parts( + Range { + start: *start, + end: *end, + }, + *priority, + ); + to_delete_ends.push(entry.block_range().end); + to_create = if let Some(cur) = to_create { + Some(cur.insert(entry, force_rescans)) + } else { + Some(SpanningTree::Leaf(entry)) + }; + } + + // Update the tree that we read from the database, or if we didn't find any ranges + // start with the scanned range. + for entry in entries { + to_create = if let Some(cur) = to_create { + Some(cur.insert(entry, force_rescans)) + } else { + Some(SpanningTree::Leaf(entry)) + }; + } + (to_create, to_delete_ends) + }; + + if let Some(tree) = to_create { + self.0.retain(|(_, block_range_end, _)| { + // if the block_range_end is equal to any in to_delete_ends, remove it + !to_delete_ends.contains(block_range_end) + }); + let scan_ranges = tree.into_vec(); + self.insert_queue_entries(scan_ranges.iter())?; + } + Ok(()) + } + + pub fn delete_starts_greater_than_equal_to(&mut self, height: BlockHeight) { + self.0.retain(|(start, _, _)| *start < height); + } + + pub fn truncate_ends_to(&mut self, height: BlockHeight) { + self.0.iter_mut().for_each(|(_, end, _)| { + if *end > height { + *end = height; + } + }); + } +} + +// We deref to slice so that we can reuse the slice impls +impl Deref for ScanQueue { + type Target = [(BlockHeight, BlockHeight, ScanPriority)]; + + fn deref(&self) -> &Self::Target { + &self.0[..] + } +} +impl DerefMut for ScanQueue { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0[..] + } +} + +mod serialization { + use super::*; + use crate::proto::memwallet as proto; + + impl From<(BlockHeight, BlockHeight, ScanPriority)> for proto::ScanQueueRecord { + fn from( + (start_height, end_height, priority): (BlockHeight, BlockHeight, ScanPriority), + ) -> Self { + Self { + start_height: start_height.into(), + end_height: end_height.into(), + priority: match priority { + ScanPriority::Ignored => proto::ScanPriority::Ignored as i32, + ScanPriority::Scanned => proto::ScanPriority::Scanned as i32, + ScanPriority::Historic => proto::ScanPriority::Historic as i32, + ScanPriority::OpenAdjacent => proto::ScanPriority::OpenAdjacent as i32, + ScanPriority::FoundNote => proto::ScanPriority::FoundNote as i32, + ScanPriority::ChainTip => proto::ScanPriority::ChainTip as i32, + ScanPriority::Verify => proto::ScanPriority::Verify as i32, + }, + } + } + } + + impl From for (BlockHeight, BlockHeight, ScanPriority) { + fn from(record: proto::ScanQueueRecord) -> Self { + ( + record.start_height.into(), + record.end_height.into(), + match record.priority() { + proto::ScanPriority::Ignored => ScanPriority::Ignored, + proto::ScanPriority::Scanned => ScanPriority::Scanned, + proto::ScanPriority::Historic => ScanPriority::Historic, + proto::ScanPriority::OpenAdjacent => ScanPriority::OpenAdjacent, + proto::ScanPriority::FoundNote => ScanPriority::FoundNote, + proto::ScanPriority::ChainTip => ScanPriority::ChainTip, + proto::ScanPriority::Verify => ScanPriority::Verify, + }, + ) + } + } +} diff --git a/zcash_client_memory/src/types/transaction.rs b/zcash_client_memory/src/types/transaction.rs new file mode 100644 index 000000000..063957341 --- /dev/null +++ b/zcash_client_memory/src/types/transaction.rs @@ -0,0 +1,315 @@ +use std::{ + collections::{btree_map::Entry, BTreeMap}, + ops::Deref, +}; + +use zcash_client_backend::{data_api::TransactionStatus, wallet::WalletTx}; +use zcash_primitives::{ + consensus::BlockHeight, + transaction::{Transaction, TxId}, +}; +use zcash_protocol::value::Zatoshis; + +use crate::error::Error; +use crate::AccountId; + +/// Maps a block height and transaction index to a transaction ID. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct TxLocatorMap(pub(crate) BTreeMap<(BlockHeight, u32), TxId>); + +impl Deref for TxLocatorMap { + type Target = BTreeMap<(BlockHeight, u32), TxId>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// A table of received notes. Corresponds to sapling_received_notes and orchard_received_notes tables. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct TransactionEntry { + // created: String, + /// mined_height is rolled into into a txn status + tx_status: TransactionStatus, + block: Option, + tx_index: Option, + expiry_height: Option, + raw: Option>, + fee: Option, + /// - `target_height`: stores the target height for which the transaction was constructed, if + /// known. This will ordinarily be null for transactions discovered via chain scanning; it + /// will only be set for transactions created using this wallet specifically, and not any + /// other wallet that uses the same seed (including previous installations of the same + /// wallet application.) + _target_height: Option, +} +impl TransactionEntry { + pub fn new_from_tx_meta(tx_meta: WalletTx, height: BlockHeight) -> Self { + Self { + tx_status: TransactionStatus::Mined(height), + tx_index: Some(tx_meta.block_index() as u32), + block: Some(height), + expiry_height: None, + raw: None, + fee: None, + _target_height: None, + } + } + pub(crate) fn expiry_height(&self) -> Option { + self.expiry_height + } + pub(crate) fn status(&self) -> TransactionStatus { + self.tx_status + } + + pub(crate) fn mined_height(&self) -> Option { + match self.tx_status { + TransactionStatus::Mined(height) => Some(height), + _ => None, + } + } + + #[cfg(test)] + pub(crate) fn fee(&self) -> Option { + self.fee + } + + pub(crate) fn raw(&self) -> Option<&[u8]> { + self.raw.as_deref() + } + + pub(crate) fn is_mined_or_unexpired_at(&self, height: BlockHeight) -> bool { + match self.tx_status { + TransactionStatus::Mined(tx_height) => tx_height <= height, + TransactionStatus::NotInMainChain => self + .expiry_height + .map_or(false, |expiry_height| expiry_height > height), + _ => false, + } + } +} + +#[derive(Debug, PartialEq)] +pub(crate) struct TransactionTable(pub(crate) BTreeMap); + +impl TransactionTable { + pub(crate) fn new() -> Self { + Self(BTreeMap::new()) + } + + /// Returns transaction status for a given transaction ID. None if the transaction is not known. + pub(crate) fn tx_status(&self, txid: &TxId) -> Option { + self.0.get(txid).map(|entry| entry.tx_status) + } + + pub(crate) fn _get_transaction(&self, txid: TxId) -> Option<&TransactionEntry> { + self.0.get(&txid) + } + + pub(crate) fn get_by_height_and_index( + &self, + height: BlockHeight, + index: u32, + ) -> Option<&TransactionEntry> { + self.0 + .values() + .find(|entry| entry.block == Some(height) && entry.tx_index == Some(index)) + } + + /// Inserts information about a MINED transaction that was observed to + /// contain a note related to this wallet + pub(crate) fn put_tx_meta(&mut self, tx_meta: WalletTx, height: BlockHeight) { + match self.0.entry(tx_meta.txid()) { + Entry::Occupied(mut entry) => { + entry.get_mut().tx_index = Some(tx_meta.block_index() as u32); + entry.get_mut().tx_status = TransactionStatus::Mined(height); + } + Entry::Vacant(entry) => { + entry.insert(TransactionEntry::new_from_tx_meta(tx_meta, height)); + } + } + } + + #[cfg(feature = "transparent-inputs")] + /// Insert partial transaction data ontained from a received transparent output + /// Will update an existing transaction if it already exists with new date (e.g. will replace Nones with newer Some value) + pub(crate) fn put_tx_partial( + &mut self, + txid: &TxId, + block: &Option, + mined_height: Option, + ) { + match self.0.entry(*txid) { + Entry::Occupied(mut entry) => { + entry.get_mut().tx_status = mined_height + .map(TransactionStatus::Mined) + .unwrap_or(TransactionStatus::NotInMainChain); + // replace the block if it's not already set + entry.get_mut().block = (*block).or(entry.get().block); + } + Entry::Vacant(entry) => { + entry.insert(TransactionEntry { + tx_status: mined_height + .map(TransactionStatus::Mined) + .unwrap_or(TransactionStatus::NotInMainChain), + block: *block, + tx_index: None, + expiry_height: None, + raw: None, + fee: None, + _target_height: None, + }); + } + } + } + + /// Inserts full transaction data + pub(crate) fn put_tx_data( + &mut self, + tx: &Transaction, + fee: Option, + target_height: Option, + ) { + match self.0.entry(tx.txid()) { + Entry::Occupied(mut entry) => { + entry.get_mut().fee = fee; + entry.get_mut().expiry_height = Some(tx.expiry_height()); + + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + entry.get_mut().raw = Some(raw); + } + Entry::Vacant(entry) => { + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + entry.insert(TransactionEntry { + tx_status: TransactionStatus::NotInMainChain, + tx_index: None, + block: None, + expiry_height: Some(tx.expiry_height()), + raw: Some(raw), + fee, + _target_height: target_height, + }); + } + } + } + + pub(crate) fn set_transaction_status( + &mut self, + txid: &TxId, + status: TransactionStatus, + ) -> Result<(), Error> { + if let Some(entry) = self.0.get_mut(txid) { + entry.tx_status = status; + Ok(()) + } else { + Err(Error::TransactionNotFound(*txid)) + } + } + + pub(crate) fn unmine_transactions_greater_than(&mut self, height: BlockHeight) { + self.0.iter_mut().for_each(|(_, entry)| { + if let TransactionStatus::Mined(tx_height) = entry.tx_status { + if tx_height > height { + entry.tx_status = TransactionStatus::NotInMainChain; + entry.block = None; + entry.tx_index = None; + } + } + }); + } +} + +impl TransactionTable { + pub(crate) fn get(&self, txid: &TxId) -> Option<&TransactionEntry> { + self.0.get(txid) + } + + pub(crate) fn _get_mut(&mut self, txid: &TxId) -> Option<&mut TransactionEntry> { + self.0.get_mut(txid) + } + + pub(crate) fn _remove(&mut self, txid: &TxId) -> Option { + self.0.remove(txid) + } +} + +impl Deref for TransactionTable { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl TxLocatorMap { + pub(crate) fn new() -> Self { + Self(BTreeMap::new()) + } + pub(crate) fn _insert(&mut self, height: BlockHeight, index: u32, txid: TxId) { + self.0.insert((height, index), txid); + } + + pub(crate) fn get(&self, height: BlockHeight, index: u32) -> Option<&TxId> { + self.0.get(&(height, index)) + } + pub(crate) fn entry(&mut self, k: (BlockHeight, u32)) -> Entry<(BlockHeight, u32), TxId> { + self.0.entry(k) + } +} + +mod serialization { + use super::*; + use crate::{proto::memwallet as proto, read_optional}; + + impl From for proto::TransactionEntry { + fn from(entry: TransactionEntry) -> Self { + Self { + tx_status: match entry.tx_status { + TransactionStatus::TxidNotRecognized => { + proto::TransactionStatus::TxidNotRecognized.into() + } + TransactionStatus::NotInMainChain => { + proto::TransactionStatus::NotInMainChain.into() + } + TransactionStatus::Mined(_) => proto::TransactionStatus::Mined.into(), + }, + block: entry.block.map(Into::into), + tx_index: entry.tx_index, + expiry_height: entry.expiry_height.map(Into::into), + raw_tx: entry.raw, + fee: entry.fee.map(Into::into), + target_height: entry._target_height.map(Into::into), + mined_height: match entry.tx_status { + TransactionStatus::Mined(height) => Some(height.into()), + _ => None, + }, + } + } + } + + impl TryFrom for TransactionEntry { + type Error = Error; + + fn try_from(entry: proto::TransactionEntry) -> Result { + Ok(Self { + tx_status: match entry.tx_status() { + proto::TransactionStatus::TxidNotRecognized => { + TransactionStatus::TxidNotRecognized + } + proto::TransactionStatus::NotInMainChain => TransactionStatus::NotInMainChain, + proto::TransactionStatus::Mined => { + TransactionStatus::Mined(read_optional!(entry, mined_height)?.into()) + } + }, + block: entry.block.map(Into::into), + tx_index: entry.tx_index.map(Into::into), + expiry_height: entry.expiry_height.map(Into::into), + raw: entry.raw_tx, + fee: entry.fee.map(|fee| fee.try_into()).transpose()?, + _target_height: entry.target_height.map(Into::into), + }) + } + } +} diff --git a/zcash_client_memory/src/types/transparent.rs b/zcash_client_memory/src/types/transparent.rs new file mode 100644 index 000000000..65055fcd2 --- /dev/null +++ b/zcash_client_memory/src/types/transparent.rs @@ -0,0 +1,210 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::{Deref, DerefMut}, +}; + +use zcash_client_backend::wallet::WalletTransparentOutput; +use zcash_primitives::{ + legacy::TransparentAddress, + transaction::{ + components::{OutPoint, TxOut}, + TxId, + }, +}; +use zcash_protocol::consensus::BlockHeight; + +use super::AccountId; +use crate::Error; + +/// Stores the transparent outputs received by the wallet. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct TransparentReceivedOutputs(pub(crate) BTreeMap); + +impl TransparentReceivedOutputs { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + + pub fn get(&self, outpoint: &OutPoint) -> Option<&ReceivedTransparentOutput> { + self.0.get(outpoint) + } + + pub fn detect_spending_accounts<'a>( + &self, + spent: impl Iterator, + ) -> Result, Error> { + let mut acc = BTreeSet::new(); + for prevout in spent { + if let Some(output) = self.0.get(prevout) { + acc.insert(output.account_id); + } + } + Ok(acc) + } +} + +impl Deref for TransparentReceivedOutputs { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TransparentReceivedOutputs { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// A junction table between received transparent outputs and the transactions that spend them. +#[derive(Debug, Default, PartialEq)] +pub struct TransparentReceivedOutputSpends(pub(crate) BTreeMap); + +impl TransparentReceivedOutputSpends { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + + pub fn get(&self, outpoint: &OutPoint) -> Option<&TxId> { + self.0.get(outpoint) + } + + pub fn insert(&mut self, outpoint: OutPoint, txid: TxId) { + self.0.insert(outpoint, txid); + } +} + +impl Deref for TransparentReceivedOutputSpends { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +// transparent_received_outputs +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReceivedTransparentOutput { + // Reference to the transaction in which this TXO was created + pub(crate) transaction_id: TxId, + // The account that controls spend authority for this TXO + pub(crate) account_id: AccountId, + // The address to which this TXO was sent + pub(crate) address: TransparentAddress, + // script, value_zat + pub(crate) txout: TxOut, + /// The maximum block height at which this TXO was either + /// observed to be a member of the UTXO set at the start of the block, or observed + /// to be an output of a transaction mined in the block. This is intended to be used to + /// determine when the TXO is no longer a part of the UTXO set, in the case that the + /// transaction that spends it is not detected by the wallet. + pub(crate) max_observed_unspent_height: Option, +} + +impl ReceivedTransparentOutput { + pub fn new( + transaction_id: TxId, + account_id: AccountId, + address: TransparentAddress, + txout: TxOut, + max_observed_unspent_height: BlockHeight, + ) -> Self { + Self { + transaction_id, + account_id, + address, + txout, + max_observed_unspent_height: Some(max_observed_unspent_height), + } + } + + pub fn to_wallet_transparent_output( + &self, + outpoint: &OutPoint, + mined_height: Option, + ) -> Option { + WalletTransparentOutput::from_parts(outpoint.clone(), self.txout.clone(), mined_height) + } +} + +/// A cache of the relationship between a transaction and the prevout data of its +/// transparent inputs. +/// +/// Output may be attempted to be spent in multiple transactions, even though only one will ever be mined +/// which is why can cannot just rely on TransparentReceivedOutputSpends or implement this as as map +#[derive(Debug, Default, PartialEq)] +pub struct TransparentSpendCache(pub(crate) BTreeSet<(TxId, OutPoint)>); + +impl TransparentSpendCache { + pub fn new() -> Self { + Self(BTreeSet::new()) + } + + /// Get all the outpoints for a given transaction ID. + pub fn contains(&self, txid: &TxId, outpoint: &OutPoint) -> bool { + self.0.contains(&(*txid, outpoint.clone())) + } +} + +impl Deref for TransparentSpendCache { + type Target = BTreeSet<(TxId, OutPoint)>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +mod serialization { + use super::*; + use crate::{proto::memwallet as proto, read_optional}; + use zcash_keys::encoding::AddressCodec; + use zcash_primitives::{consensus::Network::MainNetwork as EncodingParams, legacy::Script}; + use zcash_protocol::value::Zatoshis; + + impl From for proto::ReceivedTransparentOutput { + fn from(output: ReceivedTransparentOutput) -> Self { + Self { + transaction_id: output.transaction_id.as_ref().to_vec(), + account_id: *output.account_id, + address: output.address.encode(&EncodingParams), + txout: Some(output.txout.into()), + max_observed_unspent_height: output.max_observed_unspent_height.map(|h| h.into()), + } + } + } + + impl TryFrom for ReceivedTransparentOutput { + type Error = crate::Error; + + fn try_from(output: proto::ReceivedTransparentOutput) -> Result { + Ok(Self { + transaction_id: TxId::from_bytes(output.transaction_id.clone().try_into()?), + account_id: output.account_id.into(), + address: TransparentAddress::decode(&EncodingParams, &output.address)?, + txout: read_optional!(output, txout)?.try_into()?, + max_observed_unspent_height: output.max_observed_unspent_height.map(|h| h.into()), + }) + } + } + + impl From for proto::TxOut { + fn from(txout: TxOut) -> Self { + Self { + script: txout.script_pubkey.0, + value: txout.value.into(), + } + } + } + + impl TryFrom for TxOut { + type Error = crate::Error; + + fn try_from(txout: proto::TxOut) -> Result { + Ok(Self { + script_pubkey: Script(txout.script), + value: Zatoshis::try_from(txout.value)?, + }) + } + } +} diff --git a/zcash_client_memory/src/wallet_commitment_trees.rs b/zcash_client_memory/src/wallet_commitment_trees.rs new file mode 100644 index 000000000..cab38786e --- /dev/null +++ b/zcash_client_memory/src/wallet_commitment_trees.rs @@ -0,0 +1,203 @@ +use std::convert::Infallible; + +use incrementalmerkletree::Address; +use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree}; +#[cfg(feature = "orchard")] +use zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT; +use zcash_client_backend::data_api::{ + chain::CommitmentTreeRoot, WalletCommitmentTrees, SAPLING_SHARD_HEIGHT, +}; +use zcash_primitives::consensus::BlockHeight; +use zcash_protocol::consensus; + +use crate::MemoryWalletDb; + +impl WalletCommitmentTrees for MemoryWalletDb

{ + type Error = Infallible; + type SaplingShardStore<'a> = MemoryShardStore; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + tracing::debug!("with_sapling_tree_mut"); + callback(&mut self.sapling_tree) + } + + fn put_sapling_subtree_roots( + &mut self, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError> { + tracing::debug!("put_sapling_subtree_roots"); + self.with_sapling_tree_mut(|t| { + for (root, i) in roots.iter().zip(0u64..) { + let root_addr = Address::from_parts(SAPLING_SHARD_HEIGHT.into(), start_index + i); + t.insert(root_addr, *root.root_hash())?; + } + Ok::<_, ShardTreeError>(()) + })?; + + // store the end block heights for each shard as well + for (root, i) in roots.iter().zip(0u64..) { + let root_addr = Address::from_parts(SAPLING_SHARD_HEIGHT.into(), start_index + i); + self.sapling_tree_shard_end_heights + .insert(root_addr, root.subtree_end_height()); + } + + Ok(()) + } + + #[cfg(feature = "orchard")] + type OrchardShardStore<'a> = MemoryShardStore; + + #[cfg(feature = "orchard")] + fn with_orchard_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::OrchardShardStore<'a>, + { ORCHARD_SHARD_HEIGHT * 2 }, + ORCHARD_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + tracing::debug!("with_orchard_tree_mut"); + callback(&mut self.orchard_tree) + } + + /// Adds a sequence of note commitment tree subtree roots to the data store. + #[cfg(feature = "orchard")] + fn put_orchard_subtree_roots( + &mut self, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError> { + tracing::debug!("put_orchard_subtree_roots"); + self.with_orchard_tree_mut(|t| { + for (root, i) in roots.iter().zip(0u64..) { + let root_addr = Address::from_parts(ORCHARD_SHARD_HEIGHT.into(), start_index + i); + t.insert(root_addr, *root.root_hash())?; + } + Ok::<_, ShardTreeError>(()) + })?; + + // store the end block heights for each shard as well + for (root, i) in roots.iter().zip(0u64..) { + let root_addr = Address::from_parts(SAPLING_SHARD_HEIGHT.into(), start_index + i); + self.orchard_tree_shard_end_heights + .insert(root_addr, root.subtree_end_height()); + } + + Ok(()) + } +} + +pub(crate) mod serialization { + use std::io::Cursor; + + use incrementalmerkletree::{Address, Level}; + use shardtree::{ + store::{memory::MemoryShardStore, Checkpoint, ShardStore}, + LocatedPrunableTree, ShardTree, + }; + use zcash_client_backend::serialization::shardtree::{read_shard, write_shard}; + use zcash_protocol::consensus::BlockHeight; + + use crate::{proto::memwallet as proto, Error}; + + pub(crate) fn tree_to_protobuf< + H: Clone + + incrementalmerkletree::Hashable + + PartialEq + + zcash_primitives::merkle_tree::HashSer, + const DEPTH: u8, + const SHARD_HEIGHT: u8, + >( + tree: &ShardTree, DEPTH, SHARD_HEIGHT>, + ) -> Result, Error> { + use crate::proto::memwallet::{ShardTree, TreeCheckpoint, TreeShard}; + + let mut cap_bytes = Vec::new(); + write_shard(&mut cap_bytes, &tree.store().get_cap()?)?; + + let shards = tree + .store() + .get_shard_roots()? + .iter() + .map(|shard_root| { + let shard = tree.store().get_shard(*shard_root)?.unwrap(); + + let mut shard_data = Vec::new(); + write_shard(&mut shard_data, shard.root())?; + + Ok(TreeShard { + shard_index: shard_root.index(), + shard_data, + }) + }) + .collect::, Error>>()?; + + let mut checkpoints = Vec::new(); + tree.store() + .for_each_checkpoint(usize::MAX, |id, checkpoint| { + checkpoints.push(TreeCheckpoint { + checkpoint_id: (*id).into(), + position: match checkpoint.tree_state() { + shardtree::store::TreeState::Empty => 0, + shardtree::store::TreeState::AtPosition(position) => position.into(), + }, + }); + Ok(()) + }) + .ok(); + + Ok(Some(ShardTree { + cap: cap_bytes, + shards, + checkpoints, + })) + } + + pub(crate) fn tree_from_protobuf< + H: Clone + + incrementalmerkletree::Hashable + + PartialEq + + zcash_primitives::merkle_tree::HashSer, + const DEPTH: u8, + const SHARD_HEIGHT: u8, + >( + proto_tree: proto::ShardTree, + max_checkpoints: usize, + shard_root_level: Level, + ) -> Result, DEPTH, SHARD_HEIGHT>, Error> { + let mut tree = ShardTree::new(MemoryShardStore::empty(), max_checkpoints); + + let cap = read_shard(Cursor::new(&proto_tree.cap))?; + tree.store_mut().put_cap(cap)?; + + for proto_shard in proto_tree.shards { + let shard_root = Address::from_parts(shard_root_level, proto_shard.shard_index); + let shard_tree = read_shard(&mut Cursor::new(proto_shard.shard_data))?; + let shard = LocatedPrunableTree::from_parts(shard_root, shard_tree); + tree.store_mut().put_shard(shard)?; + } + + for proto_checkpoint in proto_tree.checkpoints { + tree.store_mut().add_checkpoint( + BlockHeight::from(proto_checkpoint.checkpoint_id), + Checkpoint::at_position(proto_checkpoint.position.into()), + )?; + } + + Ok(tree) + } +} diff --git a/zcash_client_memory/src/wallet_read.rs b/zcash_client_memory/src/wallet_read.rs new file mode 100644 index 000000000..27c7d29ec --- /dev/null +++ b/zcash_client_memory/src/wallet_read.rs @@ -0,0 +1,793 @@ +use std::{ + collections::{hash_map::Entry, HashMap}, + num::NonZeroU32, + ops::Add, + ops::Range, +}; + +use nonempty::NonEmpty; +use secrecy::{ExposeSecret, SecretVec}; +use shardtree::store::ShardStore as _; +use zcash_client_backend::data_api::{ + scanning::ScanRange, BlockMetadata, NullifierQuery, WalletRead, WalletSummary, +}; +use zcash_client_backend::{ + address::UnifiedAddress, + data_api::{ + scanning::ScanPriority, Account as _, AccountBalance, AccountSource, Balance, Progress, + Ratio, SeedRelevance, TransactionDataRequest, TransactionStatus, + }, + keys::{UnifiedAddressRequest, UnifiedFullViewingKey, UnifiedSpendingKey}, + wallet::NoteId, + PoolType, +}; +use zcash_keys::keys::UnifiedIncomingViewingKey; +use zcash_primitives::{ + block::BlockHash, + consensus::BlockHeight, + transaction::{Transaction, TransactionData, TxId}, +}; +use zcash_protocol::{ + consensus::{self, BranchId}, + memo::Memo, + value::Zatoshis, +}; +use zip32::fingerprint::SeedFingerprint; +use zip32::Scope; + +#[cfg(feature = "transparent-inputs")] +use { + zcash_client_backend::wallet::TransparentAddressMetadata, + zcash_primitives::legacy::TransparentAddress, +}; + +use crate::{error::Error, Account, AccountId, MemoryWalletBlock, MemoryWalletDb, Nullifier}; + +impl WalletRead for MemoryWalletDb

{ + type Error = Error; + type AccountId = AccountId; + type Account = Account; + + fn get_account_ids(&self) -> Result, Self::Error> { + tracing::debug!("get_account_ids"); + Ok(self.accounts.account_ids().copied().collect()) + } + + fn get_account( + &self, + account_id: Self::AccountId, + ) -> Result, Self::Error> { + tracing::debug!("get_account: {:?}", account_id); + Ok(self.accounts.get(account_id).cloned()) + } + + fn get_derived_account( + &self, + seed: &SeedFingerprint, + account_id: zip32::AccountId, + ) -> Result, Self::Error> { + tracing::debug!("get_derived_account: {:?}, {:?}", seed, account_id); + Ok(self + .accounts + .iter() + .find_map(|(_id, acct)| match acct.kind() { + AccountSource::Derived { + seed_fingerprint, + account_index, + .. + } => { + if seed_fingerprint == seed && account_index == &account_id { + Some(acct.clone()) + } else { + None + } + } + AccountSource::Imported { purpose: _, .. } => None, + })) + } + + fn validate_seed( + &self, + account_id: Self::AccountId, + seed: &SecretVec, + ) -> Result { + tracing::debug!("validate_seed: {:?}", account_id); + if let Some(account) = self.get_account(account_id)? { + if let AccountSource::Derived { + seed_fingerprint, + account_index, + .. + } = account.source() + { + seed_matches_derived_account( + &self.params, + seed, + &seed_fingerprint, + *account_index, + &account.uivk(), + ) + } else { + Err(Error::UnknownZip32Derivation) + } + } else { + // Missing account is documented to return false. + Ok(false) + } + } + + fn seed_relevance_to_derived_accounts( + &self, + seed: &SecretVec, + ) -> Result, Self::Error> { + tracing::debug!("seed_relevance_to_derived_accounts"); + let mut has_accounts = false; + let mut has_derived = false; + let mut relevant_account_ids = vec![]; + + for account_id in self.get_account_ids()? { + has_accounts = true; + let account = self.get_account(account_id)?.expect("account ID exists"); + + // If the account is imported, the seed _might_ be relevant, but the only + // way we could determine that is by brute-forcing the ZIP 32 account + // index space, which we're not going to do. The method name indicates to + // the caller that we only check derived accounts. + if let AccountSource::Derived { + seed_fingerprint, + account_index, + .. + } = account.source() + { + has_derived = true; + + if seed_matches_derived_account( + &self.params, + seed, + &seed_fingerprint, + *account_index, + &account.uivk(), + )? { + // The seed is relevant to this account. + relevant_account_ids.push(account_id); + } + } + } + + Ok( + if let Some(account_ids) = NonEmpty::from_vec(relevant_account_ids) { + SeedRelevance::Relevant { account_ids } + } else if has_derived { + SeedRelevance::NotRelevant + } else if has_accounts { + SeedRelevance::NoDerivedAccounts + } else { + SeedRelevance::NoAccounts + }, + ) + } + + fn get_account_for_ufvk( + &self, + ufvk: &UnifiedFullViewingKey, + ) -> Result, Self::Error> { + tracing::debug!("get_account_for_ufvk"); + let ufvk_req = + UnifiedAddressRequest::all().expect("At least one protocol should be enabled"); + Ok(self.accounts.iter().find_map(|(_id, acct)| { + if acct.ufvk()?.default_address(ufvk_req).unwrap() + == ufvk.default_address(ufvk_req).unwrap() + { + Some(acct.clone()) + } else { + None + } + })) + } + + fn get_current_address( + &self, + account: Self::AccountId, + ) -> Result, Self::Error> { + tracing::debug!("get_current_address: {:?}", account); + Ok(self + .get_account(account)? + .map(|account| Account::current_address(&account)) + .transpose()? + .map(|(addr, _)| addr.clone())) + } + + fn get_account_birthday(&self, account: Self::AccountId) -> Result { + tracing::debug!("get_account_birthday: {:?}", account); + self.accounts + .get(account) + .map(|account| account.birthday().height()) + .ok_or(Error::AccountUnknown(account)) + } + + fn get_wallet_birthday(&self) -> Result, Self::Error> { + tracing::debug!("get_wallet_birthday"); + Ok(self + .accounts + .iter() + .map(|(_id, account)| account.birthday().height()) + .min()) + } + + fn get_wallet_summary( + &self, + min_confirmations: u32, + ) -> Result>, Self::Error> { + tracing::debug!("get_wallet_summary"); + let chain_tip_height = match self.chain_height()? { + Some(height) => height, + None => return Ok(None), + }; + let birthday_height = self + .get_wallet_birthday()? + .expect("If a scan range exists, we know the wallet birthday."); + + let fully_scanned_height = self + .block_fully_scanned()? + .map_or(birthday_height - 1, |m| m.block_height()); + + let mut account_balances = self + .accounts + .iter() + .map(|(_id, account)| (account.account_id(), AccountBalance::ZERO)) + .collect::>(); + + for note in self.get_received_notes().iter() { + // don't count spent notes + if self.note_is_spent(note, min_confirmations)? { + continue; + } + // don't count notes in unscanned ranges + let unscanned_ranges = self.unscanned_ranges(); + for (_, _, start_position, end_position_exclusive) in unscanned_ranges { + if note.commitment_tree_position >= start_position + && note.commitment_tree_position < end_position_exclusive + { + continue; // note is in an unscanned range. Skip it + } + } + // don't count notes in unmined transactions or that have expired + if let Ok(Some(note_tx)) = self.get_transaction(note.txid) { + if note_tx.expiry_height() + < self.summary_height(min_confirmations)?.unwrap_or(0.into()) + { + continue; + } + } + + let account_id = note.account_id(); + // if this is the first note for this account add a new balance record + if let Entry::Vacant(entry) = account_balances.entry(account_id) { + entry.insert(AccountBalance::ZERO); + } + let account_balance = account_balances + .get_mut(&account_id) + .expect("Account balance should exist"); + + // Given a note update the balance for the account + // This includes determining if it is change, spendable, etc + let update_balance_with_note = |b: &mut Balance| -> Result<(), Error> { + match ( + self.note_is_pending(note, min_confirmations)?, + note.is_change, + ) { + (true, true) => b.add_pending_change_value(note.note.value()), + (true, false) => b.add_pending_spendable_value(note.note.value()), + (false, _) => b.add_spendable_value(note.note.value()), + }?; + Ok(()) + }; + + match note.pool() { + PoolType::SAPLING => { + account_balance.with_sapling_balance_mut(update_balance_with_note)?; + } + PoolType::ORCHARD => { + account_balance.with_orchard_balance_mut(update_balance_with_note)?; + } + _ => unimplemented!("Unknown pool type"), + } + } + + #[cfg(feature = "transparent-inputs")] + for (account, balance) in account_balances.iter_mut() { + let transparent_balances = + self.get_transparent_balances(*account, fully_scanned_height)?; + for (_, value) in transparent_balances { + balance.add_unshielded_value(value)?; + } + } + + let next_sapling_subtree_index = self + .sapling_tree + .store() + .last_shard()? + .map(|s| s.root_addr().index()) + .unwrap_or(0); + + #[cfg(feature = "orchard")] + let next_orchard_subtree_index = self + .orchard_tree + .store() + .last_shard()? + .map(|s| s.root_addr().index()) + .unwrap_or(0); + + // Treat Sapling and Orchard outputs as having the same cost to scan. + let sapling_scan_progress = + self.sapling_scan_progress(&birthday_height, &fully_scanned_height, &chain_tip_height)?; + #[cfg(feature = "orchard")] + let orchard_scan_progress = + self.orchard_scan_progress(&birthday_height, &fully_scanned_height, &chain_tip_height)?; + #[cfg(not(feature = "orchard"))] + let orchard_scan_progress: Option> = None; + + let scan_progress = sapling_scan_progress + .zip(orchard_scan_progress) + .map(|(s, o)| { + Ratio::new( + s.numerator() + o.numerator(), + s.denominator() + o.denominator(), + ) + }) + .or(sapling_scan_progress) + .or(orchard_scan_progress); + + // TODO: This won't work + let scan_progress = Progress::new(scan_progress.unwrap(), Some(Ratio::new(0, 0))); + + let summary = WalletSummary::new( + account_balances, + chain_tip_height, + fully_scanned_height, + scan_progress, + next_sapling_subtree_index, + #[cfg(feature = "orchard")] + next_orchard_subtree_index, + ); + Ok(Some(summary)) + } + + fn chain_height(&self) -> Result, Self::Error> { + tracing::debug!("chain_height"); + Ok(self + .scan_queue + .iter() + .max_by(|(_, end_a, _), (_, end_b, _)| end_a.cmp(end_b)) + .map(|(_, end, _)| end.saturating_sub(1))) + } + + fn get_block_hash(&self, block_height: BlockHeight) -> Result, Self::Error> { + tracing::debug!("get_block_hash: {:?}", block_height); + Ok(self.blocks.iter().find_map(|b| { + if b.0 == &block_height { + Some(b.1.hash) + } else { + None + } + })) + } + + fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error> { + tracing::debug!("block_metadata: {:?}", height); + Ok(self.blocks.get(&height).map(|block| { + let MemoryWalletBlock { + height, + hash, + sapling_commitment_tree_size, + #[cfg(feature = "orchard")] + orchard_commitment_tree_size, + .. + } = block; + // TODO: Deal with legacy sapling trees + BlockMetadata::from_parts( + *height, + *hash, + *sapling_commitment_tree_size, + #[cfg(feature = "orchard")] + *orchard_commitment_tree_size, + ) + })) + } + + fn block_fully_scanned(&self) -> Result, Self::Error> { + tracing::debug!("block_fully_scanned"); + if let Some(birthday_height) = self.get_wallet_birthday()? { + // We assume that the only way we get a contiguous range of block heights in the `blocks` table + // starting with the birthday block, is if all scanning operations have been performed on those + // blocks. This holds because the `blocks` table is only altered by `WalletDb::put_blocks` via + // `put_block`, and the effective combination of intra-range linear scanning and the nullifier + // map ensures that we discover all wallet-related information within the contiguous range. + // + // We also assume that every contiguous range of block heights in the `blocks` table has a + // single matching entry in the `scan_queue` table with priority "Scanned". This requires no + // bugs in the scan queue update logic, which we have had before. However, a bug here would + // mean that we return a more conservative fully-scanned height, which likely just causes a + // performance regression. + // + // The fully-scanned height is therefore the last height that falls within the first range in + // the scan queue with priority "Scanned". + // SQL query problems. + + let mut scanned_ranges: Vec<_> = self + .scan_queue + .iter() + .filter(|(_, _, p)| p == &ScanPriority::Scanned) + .collect(); + scanned_ranges.sort_by(|(start_a, _, _), (start_b, _, _)| start_a.cmp(start_b)); + if let Some(fully_scanned_height) = scanned_ranges.first().and_then( + |(block_range_start, block_range_end, _priority)| { + // If the start of the earliest scanned range is greater than + // the birthday height, then there is an unscanned range between + // the wallet birthday and that range, so there is no fully + // scanned height. + if *block_range_start <= birthday_height { + // Scan ranges are end-exclusive. + Some(*block_range_end - 1) + } else { + None + } + }, + ) { + self.block_metadata(fully_scanned_height) + } else { + Ok(None) + } + } else { + Ok(None) + } + } + + fn get_max_height_hash(&self) -> Result, Self::Error> { + tracing::debug!("get_max_height_hash"); + Ok(self + .blocks + .last_key_value() + .map(|(height, block)| (*height, block.hash))) + } + + fn block_max_scanned(&self) -> Result, Self::Error> { + tracing::debug!("block_max_scanned"); + Ok(self + .blocks + .last_key_value() + .map(|(height, _)| self.block_metadata(*height)) + .transpose()? + .flatten()) + } + + fn suggest_scan_ranges(&self) -> Result, Self::Error> { + tracing::debug!("suggest_scan_ranges"); + Ok(self.scan_queue.suggest_scan_ranges(ScanPriority::Historic)) + } + + fn get_target_and_anchor_heights( + &self, + min_confirmations: NonZeroU32, + ) -> Result, Self::Error> { + if let Some(chain_tip_height) = self.chain_height()? { + let sapling_anchor_height = + self.get_sapling_max_checkpointed_height(chain_tip_height, min_confirmations)?; + + #[cfg(feature = "orchard")] + let orchard_anchor_height = + self.get_orchard_max_checkpointed_height(chain_tip_height, min_confirmations)?; + #[cfg(not(feature = "orchard"))] + let orchard_anchor_height: Option = None; + + let anchor_height = sapling_anchor_height + .zip(orchard_anchor_height) + .map(|(s, o)| std::cmp::min(s, o)) + .or(sapling_anchor_height) + .or(orchard_anchor_height); + + Ok(anchor_height.map(|h| (chain_tip_height + 1, h))) + } else { + Ok(None) + } + } + + fn get_tx_height(&self, txid: TxId) -> Result, Self::Error> { + tracing::debug!("get_tx_height: {:?}", txid); + if let Some(TransactionStatus::Mined(height)) = self.tx_table.tx_status(&txid) { + Ok(Some(height)) + } else { + Ok(None) + } + } + + fn get_unified_full_viewing_keys( + &self, + ) -> Result, Self::Error> { + tracing::debug!("get_unified_full_viewing_keys"); + Ok(self + .accounts + .iter() + .filter_map(|(_id, account)| account.ufvk().map(|ufvk| (account.id(), ufvk.clone()))) + .collect()) + } + + fn get_memo(&self, id_note: NoteId) -> Result, Self::Error> { + tracing::debug!("get_memo: {:?}", id_note); + // look in both the received and sent notes + Ok(self + .get_received_note(id_note) + .map(|note| note.memo.clone()) + .or_else(|| { + self.sent_notes + .get_sent_note(&id_note) + .map(|note| note.memo.clone()) + })) + } + + fn get_transaction(&self, txid: TxId) -> Result, Self::Error> { + tracing::debug!("get_transaction: {:?}", txid); + self.tx_table + .get(&txid) + .map(|tx| (tx.status(), tx.expiry_height(), tx.raw())) + .map(|(status, expiry_height, raw)| { + let raw = raw.ok_or_else(|| { + Self::Error::CorruptedData("Transaction raw data not found".to_string()) + })?; + + // We need to provide a consensus branch ID so that pre-v5 `Transaction` structs + // (which don't commit directly to one) can store it internally. + // - If the transaction is mined, we use the block height to get the correct one. + // - If the transaction is unmined and has a cached non-zero expiry height, we use + // that (relying on the invariant that a transaction can't be mined across a network + // upgrade boundary, so the expiry height must be in the same epoch). + // - Otherwise, we use a placeholder for the initial transaction parse (as the + // consensus branch ID is not used there), and then either use its non-zero expiry + // height or return an error. + if let TransactionStatus::Mined(height) = status { + return Ok(Transaction::read( + raw, + BranchId::for_height(&self.params, height), + )?); + } + if let Some(height) = expiry_height.filter(|h| h > &BlockHeight::from(0)) { + return Ok(Transaction::read( + raw, + BranchId::for_height(&self.params, height), + )?); + } + + let tx_data = Transaction::read(raw, BranchId::Sprout) + .map_err(Self::Error::from)? + .into_data(); + + let expiry_height = tx_data.expiry_height(); + if expiry_height > BlockHeight::from(0) { + Ok(TransactionData::from_parts( + tx_data.version(), + BranchId::for_height(&self.params, expiry_height), + tx_data.lock_time(), + expiry_height, + tx_data.transparent_bundle().cloned(), + tx_data.sprout_bundle().cloned(), + tx_data.sapling_bundle().cloned(), + tx_data.orchard_bundle().cloned(), + ) + .freeze()?) + } else { + Err(Self::Error::CorruptedData( + "Consensus branch ID not known, cannot parse this transaction until it is mined" + .to_string(), + )) + } + }) + .transpose() + } + + fn get_sapling_nullifiers( + &self, + query: NullifierQuery, + ) -> Result, Self::Error> { + tracing::debug!("get_sapling_nullifiers"); + let nullifiers = self.received_notes.get_sapling_nullifiers(); + Ok(match query { + NullifierQuery::All => nullifiers + .map(|(account_id, _, nf)| (account_id, nf)) + .collect(), + NullifierQuery::Unspent => nullifiers + .filter_map(|(account_id, _, nf)| { + // find any tx we know of that spends this nullifier and if so require that it is unmined or expired + if let Some((height, tx_index)) = self.nullifiers.get(&Nullifier::Sapling(nf)) { + if let Some(spending_tx) = + self.tx_table.get_by_height_and_index(*height, *tx_index) + { + if matches!(spending_tx.status(), TransactionStatus::Mined(_)) + || spending_tx.expiry_height().is_none() + { + None + } else { + Some((account_id, nf)) + } + } else { + None + } + } else { + Some((account_id, nf)) + } + }) + .collect(), + }) + } + + #[cfg(feature = "orchard")] + fn get_orchard_nullifiers( + &self, + query: NullifierQuery, + ) -> Result, Self::Error> { + tracing::debug!("get_orchard_nullifiers"); + let nullifiers = self.received_notes.get_orchard_nullifiers(); + Ok(match query { + NullifierQuery::All => nullifiers + .map(|(account_id, _, nf)| (account_id, nf)) + .collect(), + NullifierQuery::Unspent => nullifiers + .filter_map(|(account_id, _, nf)| { + // find any tx we know of that spends this nullifier and if so require that it is unmined or expired + if let Some((height, tx_index)) = self.nullifiers.get(&Nullifier::Orchard(nf)) { + if let Some(spending_tx) = + self.tx_table.get_by_height_and_index(*height, *tx_index) + { + if matches!(spending_tx.status(), TransactionStatus::Mined(_)) + || spending_tx.expiry_height().is_none() + { + None + } else { + Some((account_id, nf)) + } + } else { + None + } + } else { + Some((account_id, nf)) + } + }) + .collect(), + }) + } + #[cfg(feature = "transparent-inputs")] + fn get_known_ephemeral_addresses( + &self, + account_id: Self::AccountId, + index_range: Option>, + ) -> Result, Self::Error> { + Ok(self + .accounts + .get(account_id) + .map(Account::ephemeral_addresses) + .unwrap_or_else(|| Ok(vec![]))? + .into_iter() + .filter(|(_addr, meta)| { + index_range + .as_ref() + .map(|range| range.contains(&meta.address_index().index())) + .unwrap_or(true) + }) + .collect::>()) + } + + #[cfg(feature = "transparent-inputs")] + fn get_transparent_receivers( + &self, + account_id: Self::AccountId, + ) -> Result>, Self::Error> { + let account = self + .get_account(account_id)? + .ok_or(Error::AccountUnknown(account_id))?; + + let t_addresses = account + .addresses() + .iter() + .filter_map(|(diversifier_index, ua)| { + ua.transparent().map(|ta| { + let metadata = + zcash_primitives::legacy::keys::NonHardenedChildIndex::from_index( + (*diversifier_index).try_into().unwrap(), + ) + .map(|i| TransparentAddressMetadata::new(Scope::External.into(), i)); + (*ta, metadata) + }) + }) + .collect(); + Ok(t_addresses) + } + + /// Returns a mapping from each transparent receiver associated with the specified account + /// to its not-yet-shielded UTXO balance, including only the effects of transactions mined + /// at a block height less than or equal to `summary_height`. + /// + /// Only non-ephemeral transparent receivers with a non-zero balance at the summary height + /// will be included. + #[cfg(feature = "transparent-inputs")] + fn get_transparent_balances( + &self, + account_id: Self::AccountId, + summary_height: BlockHeight, + ) -> Result, Self::Error> { + tracing::debug!("get_transparent_balances"); + + let mut balances = HashMap::new(); + + for (outpoint, txo) in self.transparent_received_outputs.iter().filter(|(_, txo)| { + if let Ok(Some(txo_account_id)) = + self.find_account_for_transparent_address(&txo.address) + { + txo_account_id == account_id + } else { + false + } + }) { + let tx = self + .tx_table + .get(&txo.transaction_id) + .ok_or(Error::TransactionNotFound(txo.transaction_id))?; + if tx.is_mined_or_unexpired_at(summary_height) + && self.utxo_is_spendable(outpoint, summary_height, 0)? + { + let address = txo.address; + let balance = balances.entry(address).or_insert(Zatoshis::ZERO); + *balance = balance.add(txo.txout.value).expect("balance overflow"); + } + } + + Ok(balances) + } + + fn transaction_data_requests(&self) -> Result, Self::Error> { + tracing::debug!("transaction_data_requests"); + Ok(self + .transaction_data_request_queue + .iter() + .cloned() + .collect()) + } +} + +/// Copied from zcash_client_sqlite::wallet::seed_matches_derived_account +fn seed_matches_derived_account( + params: &P, + seed: &SecretVec, + seed_fingerprint: &SeedFingerprint, + account_index: zip32::AccountId, + uivk: &UnifiedIncomingViewingKey, +) -> Result { + let seed_fingerprint_match = + &SeedFingerprint::from_seed(seed.expose_secret()).ok_or_else(|| { + Error::BadAccountData("Seed must be between 32 and 252 bytes in length.".to_owned()) + })? == seed_fingerprint; + + // Keys are not comparable with `Eq`, but addresses are, so we derive what should + // be equivalent addresses for each key and use those to check for key equality. + let uivk_match = + match UnifiedSpendingKey::from_seed(params, &seed.expose_secret()[..], account_index) { + // If we can't derive a USK from the given seed with the account's ZIP 32 + // account index, then we immediately know the UIVK won't match because wallet + // accounts are required to have a known UIVK. + Err(_) => false, + Ok(usk) => { + UnifiedAddressRequest::all().map_or(Ok::<_, Error>(false), |ua_request| { + Ok(usk + .to_unified_full_viewing_key() + .default_address(ua_request)? + == uivk.default_address(ua_request)?) + })? + } + }; + + if seed_fingerprint_match != uivk_match { + // If these mismatch, it suggests database corruption. + Err(Error::CorruptedData(format!( + "Seed fingerprint match: {seed_fingerprint_match}, uivk match: {uivk_match}" + ))) + } else { + Ok(seed_fingerprint_match && uivk_match) + } +} diff --git a/zcash_client_memory/src/wallet_write.rs b/zcash_client_memory/src/wallet_write.rs new file mode 100644 index 000000000..f8ba1d1ca --- /dev/null +++ b/zcash_client_memory/src/wallet_write.rs @@ -0,0 +1,1269 @@ +use std::cmp::{max, min}; +use std::{ + collections::{BTreeSet, HashMap}, + ops::Range, +}; + +use incrementalmerkletree::{Marking, Position, Retention}; +use rayon::prelude::*; +use secrecy::ExposeSecret; +use secrecy::SecretVec; +use shardtree::store::ShardStore; +use zcash_client_backend::{ + address::UnifiedAddress, + data_api::{ + chain::ChainState, + scanning::{ScanPriority, ScanRange}, + AccountPurpose, AccountSource, TransactionStatus, WalletCommitmentTrees as _, + SAPLING_SHARD_HEIGHT, + }, + data_api::{ + AccountBirthday, DecryptedTransaction, ScannedBlock, SentTransaction, + SentTransactionOutput, WalletRead, WalletWrite, + }, + keys::{UnifiedAddressRequest, UnifiedFullViewingKey, UnifiedSpendingKey}, + wallet::{NoteId, Recipient, WalletTransparentOutput}, + TransferType, +}; +use zcash_primitives::{ + consensus::BlockHeight, + transaction::{ + components::{OutPoint, TxOut}, + TxId, + }, +}; +use zcash_protocol::{ + consensus::{self, NetworkUpgrade}, + PoolType, + ShieldedProtocol::{self, Sapling}, +}; +use zip32::fingerprint::SeedFingerprint; +#[cfg(feature = "orchard")] +use { + shardtree::error::ShardTreeError, std::collections::BTreeMap, + zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT, +}; + +use crate::{ + error::Error, MemoryWalletBlock, MemoryWalletDb, Nullifier, ReceivedNote, PRUNING_DEPTH, + VERIFY_LOOKAHEAD, +}; + +#[cfg(feature = "orchard")] +use zcash_protocol::ShieldedProtocol::Orchard; + +#[cfg(feature = "transparent-inputs")] +use { + zcash_client_backend::wallet::TransparentAddressMetadata, + zcash_primitives::legacy::TransparentAddress, +}; + +impl WalletWrite for MemoryWalletDb

{ + type UtxoRef = OutPoint; + + fn create_account( + &mut self, + account_name: &str, + seed: &SecretVec, + birthday: &AccountBirthday, + key_source: Option<&str>, + ) -> Result<(Self::AccountId, UnifiedSpendingKey), Self::Error> { + if cfg!(not(test)) { + unimplemented!( + "Memwallet does not support adding accounts from seed phrases. + Instead derive the ufvk in the calling code and import it using `import_account_ufvk`" + ) + } else { + let seed_fingerprint = SeedFingerprint::from_seed(seed.expose_secret()) + .ok_or_else(|| Self::Error::InvalidSeedLength)?; + let account_index = self + .max_zip32_account_index(&seed_fingerprint)? + .map(|a| a.next().ok_or_else(|| Self::Error::AccountOutOfRange)) + .transpose()? + .unwrap_or(zip32::AccountId::ZERO); + + let usk = + UnifiedSpendingKey::from_seed(&self.params, seed.expose_secret(), account_index)?; + let ufvk = usk.to_unified_full_viewing_key(); + + let (id, _account) = self.add_account( + account_name, + AccountSource::Derived { + seed_fingerprint, + account_index, + key_source: key_source.map(|s| s.to_string()), + }, + ufvk, + birthday.clone(), + )?; + + Ok((id, usk)) + } + } + + fn get_next_available_address( + &mut self, + account: Self::AccountId, + request: UnifiedAddressRequest, + ) -> Result, Self::Error> { + tracing::debug!("get_next_available_address"); + self.accounts + .get_mut(account) + .map(|account| account.next_available_address(request)) + .transpose() + .map(|a| a.flatten()) + } + + fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error> { + tracing::debug!("update_chain_tip"); + // If the caller provided a chain tip that is before Sapling activation, do nothing. + let sapling_activation = match self.params.activation_height(NetworkUpgrade::Sapling) { + Some(h) if h <= tip_height => h, + _ => return Ok(()), + }; + + let max_scanned = self.block_height_extrema().map(|range| *range.end()); + let wallet_birthday = self.get_wallet_birthday()?; + + // If the chain tip is below the prior max scanned height, then the caller has caught + // the chain in the middle of a reorg. Do nothing; the caller will continue using the + // old scan ranges and either: + // - encounter an error trying to fetch the blocks (and thus trigger the same handling + // logic as if this happened with the old linear scanning code); or + // - encounter a discontinuity error in `scan_cached_blocks`, at which point they will + // call `WalletDb::truncate_to_height` as part of their reorg handling which will + // resolve the problem. + // + // We don't check the shard height, as normal usage would have the caller update the + // shard state prior to this call, so it is possible and expected to be in a situation + // where we should update the tip-related scan ranges but not the shard-related ones. + match max_scanned { + Some(h) if tip_height < h => return Ok(()), + _ => (), + }; + + // `ScanRange` uses an exclusive upper bound. + let chain_end = tip_height + 1; + + let sapling_shard_tip = self.sapling_tip_shard_end_height(); + // TODO: Handle orchard case as well. See zcash_client_sqlite scanning.rs update_chain_tip + let min_shard_tip = sapling_shard_tip; + + // Create a scanning range for the fragment of the last shard leading up to new tip. + // We set a lower bound at the wallet birthday (if known), because account creation + // requires specifying a tree frontier that ensures we don't need tree information + // prior to the birthday. + let tip_shard_entry = min_shard_tip.filter(|h| h < &chain_end).map(|h| { + let min_to_scan = wallet_birthday.filter(|b| b > &h).unwrap_or(h); + ScanRange::from_parts(min_to_scan..chain_end, ScanPriority::ChainTip) + }); + + // Create scan ranges to either validate potentially invalid blocks at the wallet's + // view of the chain tip, or connect the prior tip to the new tip. + let tip_entry = max_scanned.map_or_else( + || { + // No blocks have been scanned, so we need to anchor the start of the new scan + // range to something else. + wallet_birthday.map_or_else( + // We don't have a wallet birthday, which means we have no accounts yet. + // We can therefore ignore all blocks up to the chain tip. + || ScanRange::from_parts(sapling_activation..chain_end, ScanPriority::Ignored), + // We have a wallet birthday, so mark all blocks between that and the + // chain tip as `Historic` (performing wallet recovery). + |wallet_birthday| { + ScanRange::from_parts(wallet_birthday..chain_end, ScanPriority::Historic) + }, + ) + }, + |max_scanned| { + // The scan range starts at the block after the max scanned height. Since + // `scan_cached_blocks` retrieves the metadata for the block being connected to + // (if it exists), the connectivity of the scan range to the max scanned block + // will always be checked if relevant. + let min_unscanned = max_scanned + 1; + + // If we don't have shard metadata, this means we're doing linear scanning, so + // create a scan range from the prior tip to the current tip with `Historic` + // priority. + if tip_shard_entry.is_none() { + ScanRange::from_parts(min_unscanned..chain_end, ScanPriority::Historic) + } else { + // Determine the height to which we expect new blocks retrieved from the + // block source to be stable and not subject to being reorg'ed. + let stable_height = tip_height.saturating_sub(PRUNING_DEPTH); + + // If the wallet's max scanned height is above the stable height, + // prioritize the range between it and the new tip as `ChainTip`. + if max_scanned > stable_height { + // We are in the steady-state case, where a wallet is close to the + // chain tip and just needs to catch up. + // + // This overlaps the `tip_shard_entry` range and so will be coalesced + // with it. + ScanRange::from_parts(min_unscanned..chain_end, ScanPriority::ChainTip) + } else { + // In this case, the max scanned height is considered stable relative + // to the chain tip. However, it may be stable or unstable relative to + // the prior chain tip, which we could determine by looking up the + // prior chain tip height from the scan queue. For simplicity we merge + // these two cases together, and proceed as though the max scanned + // block is unstable relative to the prior chain tip. + // + // To confirm its stability, prioritize the `VERIFY_LOOKAHEAD` blocks + // above the max scanned height as `Verify`: + // + // - We use `Verify` to ensure that a connectivity check is performed, + // along with any required rewinds, before any `ChainTip` ranges + // (from this or any prior `update_chain_tip` call) are scanned. + // + // - We prioritize `VERIFY_LOOKAHEAD` blocks because this is expected + // to be 12.5 minutes, within which it is reasonable for a user to + // have potentially received a transaction (if they opened their + // wallet to provide an address to someone else, or spent their own + // funds creating a change output), without necessarily having left + // their wallet open long enough for the transaction to be mined and + // the corresponding block to be scanned. + // + // - We limit the range to at most the stable region, to prevent any + // `Verify` ranges from being susceptible to reorgs, and potentially + // interfering with subsequent `Verify` ranges defined by future + // calls to `update_chain_tip`. Any gap between `stable_height` and + // `shard_start_height` will be filled by the scan range merging + // logic with a `Historic` range. + // + // If `max_scanned == stable_height` then this is a zero-length range. + // In this case, any non-empty `(stable_height+1)..shard_start_height` + // will be marked `Historic`, minimising the prioritised blocks at the + // chain tip and allowing for other ranges (for example, `FoundNote`) + // to take priority. + ScanRange::from_parts( + min_unscanned + ..std::cmp::min( + stable_height + 1, + min_unscanned + VERIFY_LOOKAHEAD, + ), + ScanPriority::Verify, + ) + } + } + }, + ); + if let Some(entry) = &tip_shard_entry { + tracing::debug!("{} will update latest shard", entry); + } + tracing::debug!("{} will connect prior scanned state to new tip", tip_entry); + + let query_range = match tip_shard_entry.as_ref() { + Some(se) => Range { + start: std::cmp::min(se.block_range().start, tip_entry.block_range().start), + end: std::cmp::max(se.block_range().end, tip_entry.block_range().end), + }, + None => tip_entry.block_range().clone(), + }; + + self.scan_queue.replace_queue_entries( + &query_range, + tip_shard_entry.into_iter().chain(Some(tip_entry)), + false, + )?; + Ok(()) + } + + /// Adds a sequence of blocks to the data store. + /// + /// Assumes blocks will be here in order. + fn put_blocks( + &mut self, + from_state: &ChainState, + blocks: Vec>, + ) -> Result<(), Self::Error> { + tracing::debug!("put_blocks"); + let mut last_scanned_height = None; + struct BlockPositions { + height: BlockHeight, + sapling_start_position: Position, + #[cfg(feature = "orchard")] + orchard_start_position: Position, + } + let start_positions = blocks.first().map(|block| BlockPositions { + height: block.height(), + sapling_start_position: Position::from( + u64::from(block.sapling().final_tree_size()) + - u64::try_from(block.sapling().commitments().len()).unwrap(), + ), + #[cfg(feature = "orchard")] + orchard_start_position: Position::from( + u64::from(block.orchard().final_tree_size()) + - u64::try_from(block.orchard().commitments().len()).unwrap(), + ), + }); + + let mut sapling_commitments = vec![]; + #[cfg(feature = "orchard")] + let mut orchard_commitments = vec![]; + let mut note_positions = vec![]; + for block in blocks.into_iter() { + let mut transactions = HashMap::new(); + let mut memos = HashMap::new(); + if last_scanned_height + .iter() + .any(|prev| block.height() != *prev + 1) + { + return Err(Error::NonSequentialBlocks); + } + + for transaction in block.transactions().iter() { + let txid = transaction.txid(); + + // Mark the Sapling nullifiers of the spent notes as spent in the `sapling_spends` map. + for spend in transaction.sapling_spends() { + println!( + "marking note {:?} as spent in transaction {:?}", + spend.nf(), + txid + ); + self.mark_sapling_note_spent(*spend.nf(), txid)?; + } + + // Mark the Orchard nullifiers of the spent notes as spent in the `orchard_spends` map. + #[cfg(feature = "orchard")] + for spend in transaction.orchard_spends() { + self.mark_orchard_note_spent(*spend.nf(), txid)?; + } + + for output in transaction.sapling_outputs() { + // Insert the memo into the `memos` map. + let note_id = NoteId::new( + txid, + Sapling, + u16::try_from(output.index()) + .expect("output indices are representable as u16"), + ); + if let Ok(Some(memo)) = self.get_memo(note_id) { + memos.insert(note_id, memo.encode()); + } + // Check whether this note was spent in a later block range that + // we previously scanned. + let spent_in = output + .nf() + .and_then(|nf| self.nullifiers.get(&Nullifier::Sapling(*nf))) + .and_then(|(height, tx_idx)| self.tx_locator.get(*height, *tx_idx)) + .copied(); + + self.insert_received_sapling_note(note_id, output, spent_in); + } + + #[cfg(feature = "orchard")] + for output in transaction.orchard_outputs().iter() { + // Insert the memo into the `memos` map. + let note_id = NoteId::new( + txid, + Orchard, + u16::try_from(output.index()) + .expect("output indices are representable as u16"), + ); + if let Ok(Some(memo)) = self.get_memo(note_id) { + memos.insert(note_id, memo.encode()); + } + // Check whether this note was spent in a later block range that + // we previously scanned. + let spent_in = output + .nf() + .and_then(|nf| self.nullifiers.get(&Nullifier::Orchard(*nf))) + .and_then(|(height, tx_idx)| self.tx_locator.get(*height, *tx_idx)) + .copied(); + + self.insert_received_orchard_note(note_id, output, spent_in) + } + + transactions.insert(txid, transaction.clone()); + } + + // Insert the new nullifiers from this block into the nullifier map + self.insert_sapling_nullifier_map(block.height(), block.sapling().nullifier_map())?; + #[cfg(feature = "orchard")] + self.insert_orchard_nullifier_map(block.height(), block.orchard().nullifier_map())?; + note_positions.extend(block.transactions().iter().flat_map(|wtx| { + let iter = wtx.sapling_outputs().iter().map(|out| { + ( + ShieldedProtocol::Sapling, + out.note_commitment_tree_position(), + ) + }); + #[cfg(feature = "orchard")] + let iter = iter.chain(wtx.orchard_outputs().iter().map(|out| { + ( + ShieldedProtocol::Orchard, + out.note_commitment_tree_position(), + ) + })); + + iter + })); + + let memory_block = MemoryWalletBlock { + height: block.height(), + hash: block.block_hash(), + block_time: block.block_time(), + _transactions: transactions.keys().cloned().collect(), + _memos: memos, + sapling_commitment_tree_size: Some(block.sapling().final_tree_size()), + sapling_output_count: Some(block.sapling().commitments().len().try_into().unwrap()), + #[cfg(feature = "orchard")] + orchard_commitment_tree_size: Some(block.orchard().final_tree_size()), + #[cfg(feature = "orchard")] + orchard_action_count: Some(block.orchard().commitments().len().try_into().unwrap()), + }; + + // Insert transaction metadata into the transaction table + transactions + .into_iter() + .for_each(|(_id, tx)| self.tx_table.put_tx_meta(tx, block.height())); + + // Insert the block into the block map + self.blocks.insert(block.height(), memory_block); + last_scanned_height = Some(block.height()); + + let block_commitments = block.into_commitments(); + sapling_commitments.extend(block_commitments.sapling.into_iter().map(Some)); + #[cfg(feature = "orchard")] + orchard_commitments.extend(block_commitments.orchard.into_iter().map(Some)); + } + + if let Some((start_positions, last_scanned_height)) = + start_positions.zip(last_scanned_height) + { + // Create subtrees from the note commitments in parallel. + const CHUNK_SIZE: usize = 1024; + let sapling_subtrees = sapling_commitments + .par_chunks_mut(CHUNK_SIZE) + .enumerate() + .filter_map(|(i, chunk)| { + let start = start_positions.sapling_start_position + (i * CHUNK_SIZE) as u64; + let end = start + chunk.len() as u64; + + shardtree::LocatedTree::from_iter( + start..end, + SAPLING_SHARD_HEIGHT.into(), + chunk.iter_mut().map(|n| n.take().expect("always Some")), + ) + }) + .map(|res| (res.subtree, res.checkpoints)) + .collect::>(); + + #[cfg(feature = "orchard")] + let orchard_subtrees = orchard_commitments + .par_chunks_mut(CHUNK_SIZE) + .enumerate() + .filter_map(|(i, chunk)| { + let start = start_positions.orchard_start_position + (i * CHUNK_SIZE) as u64; + let end = start + chunk.len() as u64; + + shardtree::LocatedTree::from_iter( + start..end, + ORCHARD_SHARD_HEIGHT.into(), + chunk.iter_mut().map(|n| n.take().expect("always Some")), + ) + }) + .map(|res| (res.subtree, res.checkpoints)) + .collect::>(); + + // Collect the complete set of Sapling checkpoints + #[cfg(feature = "orchard")] + let sapling_checkpoint_positions: BTreeMap = sapling_subtrees + .iter() + .flat_map(|(_, checkpoints)| checkpoints.iter()) + .map(|(k, v)| (*k, *v)) + .collect(); + + #[cfg(feature = "orchard")] + let orchard_checkpoint_positions: BTreeMap = orchard_subtrees + .iter() + .flat_map(|(_, checkpoints)| checkpoints.iter()) + .map(|(k, v)| (*k, *v)) + .collect(); + + #[cfg(feature = "orchard")] + let (missing_sapling_checkpoints, missing_orchard_checkpoints) = ( + ensure_checkpoints( + orchard_checkpoint_positions.keys(), + &sapling_checkpoint_positions, + from_state.final_sapling_tree(), + ), + ensure_checkpoints( + sapling_checkpoint_positions.keys(), + &orchard_checkpoint_positions, + from_state.final_orchard_tree(), + ), + ); + + // Update the Sapling note commitment tree with all newly read note commitments + { + let mut sapling_subtrees_iter = sapling_subtrees.into_iter(); + self.with_sapling_tree_mut::<_, _, Self::Error>(|sapling_tree| { + sapling_tree.insert_frontier( + from_state.final_sapling_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + marking: Marking::Reference, + }, + )?; + + for (tree, checkpoints) in &mut sapling_subtrees_iter { + sapling_tree.insert_tree(tree, checkpoints)?; + } + + // Ensure we have a Sapling checkpoint for each checkpointed Orchard block height. + // We skip all checkpoints below the minimum retained checkpoint in the + // Sapling tree, because branches below this height may be pruned. + #[cfg(feature = "orchard")] + { + let min_checkpoint_height = sapling_tree + .store() + .min_checkpoint_id() + .map_err(ShardTreeError::Storage)? + .expect("At least one checkpoint was inserted (by insert_frontier)"); + + for (height, checkpoint) in &missing_sapling_checkpoints { + if *height > min_checkpoint_height { + sapling_tree + .store_mut() + .add_checkpoint(*height, checkpoint.clone()) + .map_err(ShardTreeError::Storage)?; + } + } + } + + Ok(()) + })?; + } + + // Update the Orchard note commitment tree with all newly read note commitments + #[cfg(feature = "orchard")] + { + let mut orchard_subtrees = orchard_subtrees.into_iter(); + self.with_orchard_tree_mut::<_, _, Self::Error>(|orchard_tree| { + orchard_tree.insert_frontier( + from_state.final_orchard_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + marking: Marking::Reference, + }, + )?; + + for (tree, checkpoints) in &mut orchard_subtrees { + orchard_tree.insert_tree(tree, checkpoints)?; + } + + // Ensure we have an Orchard checkpoint for each checkpointed Sapling block height. + // We skip all checkpoints below the minimum retained checkpoint in the + // Orchard tree, because branches below this height may be pruned. + { + let min_checkpoint_height = orchard_tree + .store() + .min_checkpoint_id() + .map_err(ShardTreeError::Storage)? + .expect("At least one checkpoint was inserted (by insert_frontier)"); + + for (height, checkpoint) in &missing_orchard_checkpoints { + if *height > min_checkpoint_height { + orchard_tree + .store_mut() + .add_checkpoint(*height, checkpoint.clone()) + .map_err(ShardTreeError::Storage)?; + } + } + } + Ok(()) + })?; + } + + self.scan_complete( + Range { + start: start_positions.height, + end: last_scanned_height + 1, + }, + ¬e_positions, + )?; + } + + Ok(()) + } + + /// Adds a transparent UTXO received by the wallet to the data store. + fn put_received_transparent_utxo( + &mut self, + output: &WalletTransparentOutput, + ) -> Result { + tracing::debug!("put_received_transparent_utxo"); + #[cfg(feature = "transparent-inputs")] + { + let address = output.recipient_address(); + if let Some(account_id) = self.find_account_for_transparent_address(address)? { + self.put_transparent_output(output, &account_id, false) + } else { + Err(Error::AddressNotRecognized(*address)) + } + } + #[cfg(not(feature = "transparent-inputs"))] + panic!( + "The wallet must be compiled with the transparent-inputs feature to use this method." + ); + } + + fn store_decrypted_tx( + &mut self, + d_tx: DecryptedTransaction, + ) -> Result<(), Self::Error> { + tracing::debug!("store_decrypted_tx"); + self.tx_table.put_tx_data(d_tx.tx(), None, None); + if let Some(height) = d_tx.mined_height() { + self.set_transaction_status(d_tx.tx().txid(), TransactionStatus::Mined(height))? + } + + let funding_accounts = self.get_funding_accounts(d_tx.tx())?; + // TODO(#1305): Correctly track accounts that fund each transaction output. + let funding_account = funding_accounts.iter().next().copied(); + if funding_accounts.len() > 1 { + tracing::warn!( + "More than one wallet account detected as funding transaction {:?}, selecting {:?}", + d_tx.tx().txid(), + funding_account.unwrap() + ) + } + + // A flag used to determine whether it is necessary to query for transactions that + // provided transparent inputs to this transaction, in order to be able to correctly + // recover transparent transaction history. + #[cfg(feature = "transparent-inputs")] + let mut tx_has_wallet_outputs = false; + + for output in d_tx.sapling_outputs() { + #[cfg(feature = "transparent-inputs")] + { + tx_has_wallet_outputs = true; + } + + match output.transfer_type() { + TransferType::Outgoing => { + let recipient = { + let receiver = Receiver::Sapling(output.note().recipient()); + let wallet_address = self + .accounts + .get(*output.account()) + .map(|acc| { + acc.select_receiving_address(self.params.network_type(), &receiver) + }) + .transpose()? + .flatten() + .unwrap_or_else(|| { + receiver.to_zcash_address(self.params.network_type()) + }); + + Recipient::External(wallet_address, PoolType::SAPLING) + }; + + let sent_tx_output = SentTransactionOutput::from_parts( + output.index(), + recipient, + output.note_value(), + Some(output.memo().clone()), + ); + self.sent_notes.put_sent_output( + d_tx.tx().txid(), + *output.account(), + &sent_tx_output, + ); + } + TransferType::WalletInternal => { + let recipient = Recipient::InternalAccount { + receiving_account: *output.account(), + external_address: None, + note: Note::Sapling(output.note().clone()), + }; + let sent_tx_output = SentTransactionOutput::from_parts( + output.index(), + recipient, + output.note_value(), + Some(output.memo().clone()), + ); + + self.received_notes + .insert_received_note(ReceivedNote::from_sent_tx_output( + d_tx.tx().txid(), + &sent_tx_output, + )?); + + self.sent_notes.put_sent_output( + d_tx.tx().txid(), + *output.account(), + &sent_tx_output, + ); + } + TransferType::Incoming => { + todo!("store decrypted tx sapling incoming") + } + } + } + + #[cfg(feature = "orchard")] + for output in d_tx.orchard_outputs() { + #[cfg(feature = "transparent-inputs")] + { + tx_has_wallet_outputs = true; + } + match output.transfer_type() { + TransferType::Outgoing => { + let recipient = { + let receiver = Receiver::Orchard(output.note().recipient()); + let wallet_address = self + .accounts + .get(*output.account()) + .map(|acc| { + acc.select_receiving_address(self.params.network_type(), &receiver) + }) + .transpose()? + .flatten() + .unwrap_or_else(|| { + receiver.to_zcash_address(self.params.network_type()) + }); + + Recipient::External(wallet_address, PoolType::ORCHARD) + }; + + let sent_tx_output = SentTransactionOutput::from_parts( + output.index(), + recipient, + output.note_value(), + Some(output.memo().clone()), + ); + self.sent_notes.put_sent_output( + d_tx.tx().txid(), + *output.account(), + &sent_tx_output, + ); + } + TransferType::WalletInternal => { + let recipient = Recipient::InternalAccount { + receiving_account: *output.account(), + external_address: None, + note: Note::Orchard(*output.note()), + }; + let sent_tx_output = SentTransactionOutput::from_parts( + output.index(), + recipient, + output.note_value(), + Some(output.memo().clone()), + ); + + self.received_notes + .insert_received_note(ReceivedNote::from_sent_tx_output( + d_tx.tx().txid(), + &sent_tx_output, + )?); + + self.sent_notes.put_sent_output( + d_tx.tx().txid(), + *output.account(), + &sent_tx_output, + ); + } + TransferType::Incoming => { + todo!("store decrypted tx orchard incoming") + } + } + } + + // If any of the utxos spent in the transaction are ours, mark them as spent. + #[cfg(feature = "transparent-inputs")] + for txin in d_tx + .tx() + .transparent_bundle() + .iter() + .flat_map(|b| b.vin.iter()) + { + self.mark_transparent_output_spent(&d_tx.tx().txid(), &txin.prevout)?; + } + + // This `if` is just an optimization for cases where we would do nothing in the loop. + if funding_account.is_some() || cfg!(feature = "transparent-inputs") { + for (output_index, txout) in d_tx + .tx() + .transparent_bundle() + .iter() + .flat_map(|b| b.vout.iter()) + .enumerate() + { + if let Some(address) = txout.recipient_address() { + tracing::debug!( + "{:?} output {} has recipient {}", + d_tx.tx().txid(), + output_index, + address.encode(self.params()) + ); + + // The transaction is not necessarily mined yet, but we want to record + // that an output to the address was seen in this tx anyway. This will + // advance the gap regardless of whether it is mined, but an output in + // an unmined transaction won't advance the range of safe indices. + #[cfg(feature = "transparent-inputs")] + self.accounts + .mark_ephemeral_address_as_seen(&address, d_tx.tx().txid())?; + + // If the output belongs to the wallet, add it to `transparent_received_outputs`. + #[cfg(feature = "transparent-inputs")] + if let Some(account_id) = self + .accounts + .find_account_for_transparent_address(&address)? + { + tracing::debug!( + "{:?} output {} belongs to account {:?}", + d_tx.tx().txid(), + output_index, + account_id + ); + let wallet_transparent_output = WalletTransparentOutput::from_parts( + OutPoint::new( + d_tx.tx().txid().into(), + u32::try_from(output_index).unwrap(), + ), + txout.clone(), + d_tx.mined_height(), + ) + .unwrap(); + self.put_transparent_output( + &wallet_transparent_output, + &account_id, + false, + )?; + + // Since the wallet created the transparent output, we need to ensure + // that any transparent inputs belonging to the wallet will be + // discovered. + tx_has_wallet_outputs = true; + } else { + tracing::debug!( + "Address {} is not recognized as belonging to any of our accounts.", + address.encode(self.params()) + ); + } + + // If a transaction we observe contains spends from our wallet, we will + // store its transparent outputs in the same way they would be stored by + // create_spend_to_address. + if let Some(account_id) = funding_account { + let receiver = Receiver::Transparent(address); + + #[cfg(feature = "transparent-inputs")] + let recipient_addr = self + .accounts + .get(account_id) + .map(|acc| { + acc.select_receiving_address(self.params.network_type(), &receiver) + }) + .transpose()? + .flatten() + .unwrap_or_else(|| { + receiver.to_zcash_address(self.params.network_type()) + }); + + #[cfg(not(feature = "transparent-inputs"))] + let recipient_addr = receiver.to_zcash_address(self.params.network_type()); + + let recipient = Recipient::External(recipient_addr, PoolType::TRANSPARENT); + + let sent_tx_output = SentTransactionOutput::from_parts( + output_index, + recipient, + txout.value, + None, + ); + self.sent_notes.put_sent_output( + d_tx.tx().txid(), + account_id, + &sent_tx_output, + ); + // Even though we know the funding account, we don't know that we have + // information for all of the transparent inputs to the transaction. + #[cfg(feature = "transparent-inputs")] + { + tx_has_wallet_outputs = true; + } + } + } else { + tracing::warn!( + "Unable to determine recipient address for tx {:?} output {}", + d_tx.tx().txid(), + output_index + ); + } + } + } + + // If the transaction has outputs that belong to the wallet as well as transparent + // inputs, we may need to download the transactions corresponding to the transparent + // prevout references to determine whether the transaction was created (at least in + // part) by this wallet. + #[cfg(feature = "transparent-inputs")] + if tx_has_wallet_outputs && d_tx.tx().transparent_bundle().is_some() { + // queue the transparent inputs for enhancement + self.transaction_data_request_queue + .queue_status_retrieval(&d_tx.tx().txid()); + } + + #[cfg(feature = "transparent-inputs")] + { + let detectable_via_scanning = d_tx.tx().sapling_bundle().is_some(); + #[cfg(feature = "orchard")] + let detectable_via_scanning = + detectable_via_scanning | d_tx.tx().orchard_bundle().is_some(); + + if d_tx.mined_height().is_none() && !detectable_via_scanning { + self.transaction_data_request_queue + .queue_status_retrieval(&d_tx.tx().txid()); + } + } + Ok(()) + } + + /// Truncates the database to the given height. + /// + /// If the requested height is greater than or equal to the height of the last scanned + /// block, this function does nothing. + /// + /// This should only be executed inside a transactional context. + fn truncate_to_height(&mut self, max_height: BlockHeight) -> Result { + let truncation_height = { + // This is the intersection of all the checkpoint heights from the sapling and orchard tree. + let mut checkpoint_heights = BTreeSet::new(); + self.sapling_tree.store().for_each_checkpoint( + self.sapling_tree.store().checkpoint_count()?, + |height, _| { + checkpoint_heights.insert(u32::from(*height)); + Ok(()) + }, + )?; + #[cfg(feature = "orchard")] + { + let mut orchard_checkpoint_heights = BTreeSet::new(); + self.orchard_tree.store().for_each_checkpoint( + self.orchard_tree.store().checkpoint_count()?, + |height, _| { + orchard_checkpoint_heights.insert(u32::from(*height)); + Ok(()) + }, + )?; + + checkpoint_heights = checkpoint_heights + .intersection(&orchard_checkpoint_heights) + .copied() + .collect(); + } + // All the checkpoints that are greater than the truncation height + let over = checkpoint_heights.split_off(&(u32::from(max_height + 1))); + if let Some(height) = checkpoint_heights.last().copied() { + Ok(BlockHeight::from(height)) + } else { + // If there are no checkpoints that are less than or equal to the truncation height + // then we can't truncate the tree. + Err(Error::RequestedRewindInvalid( + over.first().copied().map(Into::into), + max_height, + )) + } + }?; + + // Recall where we synced up to previously. + let last_scanned_height = self.blocks.keys().max().copied().unwrap_or_else(|| { + self.params + .activation_height(NetworkUpgrade::Sapling) + .expect("Sapling activation height must be available.") + - 1 + }); + + // Delete from the scanning queue any range with a start height greater than the + // truncation height, and then truncate any remaining range by setting the end + // equal to the truncation height + 1. This sets our view of the chain tip back + // to the retained height. + self.scan_queue + .delete_starts_greater_than_equal_to(truncation_height + 1); + self.scan_queue.truncate_ends_to(truncation_height + 1); + + // Mark transparent utxos as un-mined. Since the TXO is now not mined, it would ideally be + // considered to have been returned to the mempool; it _might_ be spendable in this state, but + // we must also set its max_observed_unspent_height field to NULL because the transaction may + // be rendered entirely invalid by a reorg that alters anchor(s) used in constructing shielded + // spends in the transaction. + self.transparent_received_outputs + .iter_mut() + .for_each(|(_, txo)| { + if let Some(mined_height) = self + .tx_table + .get(&txo.transaction_id) + .and_then(|tx| tx.mined_height()) + { + if mined_height <= truncation_height { + txo.max_observed_unspent_height = Some(truncation_height) + } else { + txo.max_observed_unspent_height = None + } + } + }); + + // Un-mine transactions. This must be done outside of the last_scanned_height check because + // transaction entries may be created as a consequence of receiving transparent TXOs. + self.tx_table + .unmine_transactions_greater_than(truncation_height); + + // If we're removing scanned blocks, we need to truncate the note commitment tree and remove + // affected block records from the database. + if truncation_height < last_scanned_height { + self.with_sapling_tree_mut(|tree| { + tree.truncate_to_checkpoint(&truncation_height).map(|_| ()) + })?; + #[cfg(feature = "orchard")] + self.with_orchard_tree_mut(|tree| { + tree.truncate_to_checkpoint(&truncation_height).map(|_| ()) + })?; + + // Do not delete sent notes; this can contain data that is not recoverable + // from the chain. Wallets must continue to operate correctly in the + // presence of stale sent notes that link to unmined transactions. + // Also, do not delete received notes; they may contain memo data that is + // not recoverable; balance APIs must ensure that un-mined received notes + // do not count towards spendability or transaction balalnce. + + // Now that they aren't depended on, delete un-mined blocks. + self.blocks.retain(|height, _| *height <= truncation_height); + + // Delete from the nullifier map any entries with a locator referencing a block + // height greater than the truncation height. + // Willem: We don't need to do this I think.. + } + Ok(truncation_height) + } + + fn import_account_hd( + &mut self, + _account_name: &str, + _seed: &SecretVec, + _account_index: zip32::AccountId, + _birthday: &AccountBirthday, + _key_source: Option<&str>, + ) -> Result<(Self::Account, UnifiedSpendingKey), Self::Error> { + unimplemented!( + "Memwallet does not support adding accounts from seed phrases. +Instead derive the ufvk in the calling code and import it using `import_account_ufvk`" + ) + } + + fn import_account_ufvk( + &mut self, + account_name: &str, + unified_key: &UnifiedFullViewingKey, + birthday: &AccountBirthday, + purpose: AccountPurpose, + key_source: Option<&str>, + ) -> Result { + tracing::debug!("import_account_ufvk"); + let (_id, account) = self.add_account( + account_name, + AccountSource::Imported { + purpose, + key_source: key_source.map(str::to_owned), + }, + unified_key.to_owned(), + birthday.clone(), + )?; + Ok(account) + } + + fn store_transactions_to_be_sent( + &mut self, + transactions: &[SentTransaction], + ) -> Result<(), Self::Error> { + tracing::debug!("store_transactions_to_be_sent"); + for sent_tx in transactions { + self.tx_table.put_tx_data( + sent_tx.tx(), + Some(sent_tx.fee_amount()), + Some(sent_tx.target_height()), + ); + let mut detectable_via_scanning = false; + // Mark sapling notes as spent + if let Some(bundle) = sent_tx.tx().sapling_bundle() { + detectable_via_scanning = true; + for spend in bundle.shielded_spends() { + self.mark_sapling_note_spent(*spend.nullifier(), sent_tx.tx().txid())?; + } + } + // Mark orchard notes as spent + if let Some(_bundle) = sent_tx.tx().orchard_bundle() { + #[cfg(feature = "orchard")] + { + detectable_via_scanning = true; + for action in _bundle.actions() { + match self.mark_orchard_note_spent(*action.nullifier(), sent_tx.tx().txid()) + { + Ok(()) => {} + Err(Error::NoteNotFound) => { + // This is expected as some of the actions will be new outputs we don't have notes for + // The ones we do recognize will be marked as spent + } + Err(e) => return Err(e), + } + } + } + + #[cfg(not(feature = "orchard"))] + panic!("Sent a transaction with Orchard Actions without `orchard` enabled?"); + } + // Mark transparent UTXOs as spent + #[cfg(feature = "transparent-inputs")] + for utxo_outpoint in sent_tx.utxos_spent() { + self.mark_transparent_output_spent(&sent_tx.tx().txid(), utxo_outpoint)?; + } + + for output in sent_tx.outputs() { + self.sent_notes.insert_sent_output(sent_tx, output); + + match output.recipient() { + Recipient::InternalAccount { .. } => { + self.received_notes.insert_received_note( + ReceivedNote::from_sent_tx_output(sent_tx.tx().txid(), output)?, + ); + } + #[cfg(feature = "transparent-inputs")] + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint_metadata, + } => { + let txo = WalletTransparentOutput::from_parts( + outpoint_metadata.clone(), + TxOut { + value: output.value(), + script_pubkey: ephemeral_address.script(), + }, + None, + ) + .unwrap(); + self.put_transparent_output(&txo, receiving_account, true)?; + if let Some(account) = self.accounts.get_mut(*receiving_account) { + account.mark_ephemeral_address_as_used( + ephemeral_address, + sent_tx.tx().txid(), + )? + } + } + _ => {} + } + } + + // Add the transaction to the set to be queried for transaction status. This is only necessary + // at present for fully transparent transactions, because any transaction with a shielded + // component will be detected via ordinary chain scanning and/or nullifier checking. + if !detectable_via_scanning { + self.transaction_data_request_queue + .queue_status_retrieval(&sent_tx.tx().txid()); + } + } + + Ok(()) + } + + fn set_transaction_status( + &mut self, + txid: TxId, + status: TransactionStatus, + ) -> Result<(), Self::Error> { + tracing::debug!("set_transaction_status"); + self.tx_table.set_transaction_status(&txid, status) + } + + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + account_id: Self::AccountId, + n: usize, + ) -> Result, Self::Error> { + // TODO: We need to implement first_unsafe_index to make sure we dont violate gap invarient + let first_unsafe = self.first_unsafe_index(account_id)?; + if let Some(account) = self.accounts.get_mut(account_id) { + let first_unreserved = account.first_unreserved_index()?; + + let allocation = range_from(first_unreserved, u32::try_from(n).unwrap()); + + if allocation.len() < n { + return Err(AddressGenerationError::DiversifierSpaceExhausted.into()); + } + if allocation.end > first_unsafe { + return Err(Error::ReachedGapLimit( + account_id, + max(first_unreserved, first_unsafe), + )); + } + let _reserved = account.reserve_until(allocation.end)?; + self.get_known_ephemeral_addresses(account_id, Some(allocation)) + } else { + Err(Self::Error::AccountUnknown(account_id)) + } + } +} + +#[cfg(feature = "transparent-inputs")] +fn range_from(i: u32, n: u32) -> Range { + let first = min(1 << 31, i); + let last = min(1 << 31, i.saturating_add(n)); + first..last +} + +use zcash_client_backend::wallet::Note; +use zcash_keys::address::Receiver; +use zcash_keys::encoding::AddressCodec; +use zcash_keys::keys::AddressGenerationError; +#[cfg(feature = "orchard")] +use {incrementalmerkletree::frontier::Frontier, shardtree::store::Checkpoint}; + +#[cfg(feature = "orchard")] +fn ensure_checkpoints<'a, H, I: Iterator, const DEPTH: u8>( + // An iterator of checkpoints heights for which we wish to ensure that + // checkpoints exists. + ensure_heights: I, + // The map of checkpoint positions from which we will draw note commitment tree + // position information for the newly created checkpoints. + existing_checkpoint_positions: &BTreeMap, + // The frontier whose position will be used for an inserted checkpoint when + // there is no preceding checkpoint in existing_checkpoint_positions. + state_final_tree: &Frontier, +) -> Vec<(BlockHeight, Checkpoint)> { + ensure_heights + .flat_map(|ensure_height| { + existing_checkpoint_positions + .range::(..=*ensure_height) + .last() + .map_or_else( + || { + Some(( + *ensure_height, + state_final_tree + .value() + .map_or_else(Checkpoint::tree_empty, |t| { + Checkpoint::at_position(t.position()) + }), + )) + }, + |(existing_checkpoint_height, position)| { + if *existing_checkpoint_height < *ensure_height { + Some((*ensure_height, Checkpoint::at_position(*position))) + } else { + // The checkpoint already exists, so we don't need to + // do anything. + None + } + }, + ) + .into_iter() + }) + .collect::>() +} diff --git a/zcash_keys/src/address.rs b/zcash_keys/src/address.rs index 665c03a34..674719a5e 100644 --- a/zcash_keys/src/address.rs +++ b/zcash_keys/src/address.rs @@ -173,7 +173,7 @@ impl UnifiedAddress { &self.unknown } - fn to_address(&self, net: NetworkType) -> ZcashAddress { + pub fn to_address(&self, net: NetworkType) -> ZcashAddress { let items = self .unknown .iter()