From c67186139d39a98ae1bebed1fbe630f5030bb315 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 2 Dec 2024 08:36:24 -0700 Subject: [PATCH] WIP: Add handling for transparent gap limit. --- zcash_client_backend/CHANGELOG.md | 16 +- zcash_client_backend/src/data_api.rs | 40 +- zcash_client_backend/src/data_api/testing.rs | 11 + .../src/data_api/testing/pool.rs | 52 +- zcash_client_backend/src/sync.rs | 5 +- zcash_client_sqlite/CHANGELOG.md | 3 + zcash_client_sqlite/src/error.rs | 34 +- zcash_client_sqlite/src/lib.rs | 233 +++++-- zcash_client_sqlite/src/testing/db.rs | 1 + zcash_client_sqlite/src/testing/pool.rs | 6 +- zcash_client_sqlite/src/wallet.rs | 373 ++++++++--- zcash_client_sqlite/src/wallet/db.rs | 161 ++--- zcash_client_sqlite/src/wallet/init.rs | 17 +- .../src/wallet/init/migrations.rs | 8 +- .../init/migrations/add_utxo_account.rs | 22 +- .../init/migrations/ephemeral_addresses.rs | 78 ++- .../migrations/fix_bad_change_flagging.rs | 5 +- .../init/migrations/receiving_key_scopes.rs | 23 +- .../transparent_gap_limit_handling.rs | 492 ++++++++++++++ zcash_client_sqlite/src/wallet/orchard.rs | 84 ++- zcash_client_sqlite/src/wallet/sapling.rs | 87 ++- zcash_client_sqlite/src/wallet/transparent.rs | 607 ++++++++++++++---- .../src/wallet/transparent/ephemeral.rs | 476 ++------------ zcash_keys/src/address.rs | 12 + zcash_transparent/CHANGELOG.md | 3 + zcash_transparent/src/keys.rs | 53 +- 26 files changed, 2010 insertions(+), 892 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index ba5a86f54c..5f4a2cc00c 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -13,9 +13,19 @@ and this library adheres to Rust's notion of - The `Recipient::External` variant is now a structured variant. - The `Recipient::EphemeralTransparent` variant is now only available if `zcash_client_backend` is built using the `transparent-inputs` feature flag. -- `zcash_client_backend::data_api::WalletRead::get_known_ephemeral_addresses` - now takes a `Range` as its - argument instead of a `Range` +- `zcash_client_backend::data_api::WalletRead`: + - `get_transparent_receivers` now takes additional `include_change` and + `include_ephemeral` arguments. + - `get_known_ephemeral_addresses` now takes a + `Range` as its argument + instead of a `Range` +- `zcash_client_backend::data_api::WalletWrite` has an added method + `get_address_for_index` + +### Removed +- `zcash_client_backend::data_api::GAP_LIMIT` gap limits are now configured + based upon the key scope that they're associated with; there is no longer a + globally applicable gap limit. ## [0.16.0] - 2024-12-16 diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 63943ab3f5..3a4d3ca8a1 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -67,7 +67,7 @@ use incrementalmerkletree::{frontier::Frontier, Retention}; use nonempty::NonEmpty; use secrecy::SecretVec; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; -use zip32::fingerprint::SeedFingerprint; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use self::{ chain::{ChainState, CommitmentTreeRoot}, @@ -131,10 +131,6 @@ pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; #[cfg(feature = "orchard")] pub const ORCHARD_SHARD_HEIGHT: u8 = { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 } / 2; -/// The number of ephemeral addresses that can be safely reserved without observing any -/// of them to be mined. This is the same as the gap limit in Bitcoin. -pub const GAP_LIMIT: u32 = 20; - /// An enumeration of constraints that can be applied when querying for nullifiers for notes /// belonging to the wallet. pub enum NullifierQuery { @@ -1386,6 +1382,8 @@ pub trait WalletRead { fn get_transparent_receivers( &self, _account: Self::AccountId, + _include_change: bool, + _include_ephemeral: bool, ) -> Result>, Self::Error> { Ok(HashMap::new()) } @@ -1410,7 +1408,7 @@ pub trait WalletRead { /// This is equivalent to (but may be implemented more efficiently than): /// ```compile_fail /// Ok( - /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + /// if let Some(result) = self.get_transparent_receivers(account, true, true)?.get(address) { /// result.clone() /// } else { /// self.get_known_ephemeral_addresses(account, None)? @@ -1431,7 +1429,10 @@ pub trait WalletRead { ) -> Result, Self::Error> { // This should be overridden. Ok( - if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + if let Some(result) = self + .get_transparent_receivers(account, true, true)? + .get(address) + { result.clone() } else { self.get_known_ephemeral_addresses(account, None)? @@ -2377,9 +2378,10 @@ pub trait WalletWrite: WalletRead { key_source: Option<&str>, ) -> Result; - /// Generates and persists the next available diversified address for the specified account, - /// given the current addresses known to the wallet. If the `request` parameter is `None`, - /// an address should be generated using all of the available receivers for the account's UFVK. + /// Generates, persists, and marks as exposed the next available diversified address for the + /// specified account, given the current addresses known to the wallet. If the `request` + /// parameter is `None`, an address should be generated using all of the available receivers + /// for the account's UFVK. /// /// Returns `Ok(None)` if the account identifier does not correspond to a known /// account. @@ -2389,6 +2391,24 @@ pub trait WalletWrite: WalletRead { request: Option, ) -> Result, Self::Error>; + /// Generates, persists, and marks as exposed a diversified address for the specified account + /// at the provided diversifier index. If the `request` parameter is `None`, an address should + /// be generated using all of the available receivers for the account's UFVK. + /// + /// In the case that the diversifier index is outside of the range of valid transparent address + /// indexes, no transparent receiver should be generated in the resulting unified address. If a + /// transparent receiver is specifically requested for such a diversifier index, + /// implementations of this method should return an error. + /// + /// Address generation should fail if a transparent receiver would be generated that violates + /// the backend's internally configured gap limit for HD-seed-based recovery. + fn get_address_for_index( + &mut self, + account: Self::AccountId, + diversifier_index: DiversifierIndex, + request: Option, + ) -> Result, Self::Error>; + /// Updates the wallet's view of the blockchain. /// /// This method is used to provide the wallet with information about the state of the diff --git a/zcash_client_backend/src/data_api/testing.rs b/zcash_client_backend/src/data_api/testing.rs index 15e70b664c..d742d5d25d 100644 --- a/zcash_client_backend/src/data_api/testing.rs +++ b/zcash_client_backend/src/data_api/testing.rs @@ -2613,6 +2613,8 @@ impl WalletRead for MockWalletDb { fn get_transparent_receivers( &self, _account: Self::AccountId, + _include_change: bool, + _include_ephemeral: bool, ) -> Result>, Self::Error> { Ok(HashMap::new()) } @@ -2703,6 +2705,15 @@ impl WalletWrite for MockWalletDb { Ok(None) } + fn get_address_for_index( + &mut self, + _account: Self::AccountId, + _diversifier_index: DiversifierIndex, + _request: Option, + ) -> Result, Self::Error> { + Ok(None) + } + #[allow(clippy::type_complexity)] fn put_blocks( &mut self, diff --git a/zcash_client_backend/src/data_api/testing/pool.rs b/zcash_client_backend/src/data_api/testing/pool.rs index 0e21621924..70f7d29b85 100644 --- a/zcash_client_backend/src/data_api/testing/pool.rs +++ b/zcash_client_backend/src/data_api/testing/pool.rs @@ -87,6 +87,11 @@ use crate::PoolType; #[cfg(feature = "pczt")] use pczt::roles::{prover::Prover, signer::Signer}; +/// The number of ephemeral addresses that can be safely reserved without observing any +/// of them to be mined. +#[cfg(feature = "transparent-inputs")] +pub const EPHEMERAL_ADDR_GAP_LIMIT: u32 = 3; + /// Trait that exposes the pool-specific types and operations necessary to run the /// single-shielded-pool tests on a given pool. /// @@ -511,7 +516,7 @@ pub fn send_multi_step_proposed_transfer( { use zcash_primitives::transaction::components::transparent::builder::TransparentSigningSet; - use crate::data_api::{OutputOfSentTx, GAP_LIMIT}; + use crate::data_api::OutputOfSentTx; let mut st = TestBuilder::new() .with_data_store_factory(ds_factory) @@ -536,24 +541,31 @@ pub fn send_multi_step_proposed_transfer( .block_height(), h ); - assert_eq!(st.get_spendable_balance(account_id, 1), value); h }; let value = NonNegativeAmount::const_from_u64(100000); let transfer_amount = NonNegativeAmount::const_from_u64(50000); - let run_test = |st: &mut TestState<_, DSF::DataStore, _>, expected_index| { + let run_test = |st: &mut TestState<_, DSF::DataStore, _>, expected_index, prior_balance| { // Add funds to the wallet. add_funds(st, value); + let initial_balance: Option = prior_balance + value; + assert_eq!( + st.get_spendable_balance(account_id, 1), + initial_balance.unwrap() + ); let expected_step0_fee = (zip317::MARGINAL_FEE * 3u64).unwrap(); let expected_step1_fee = zip317::MINIMUM_FEE; let expected_ephemeral = (transfer_amount + expected_step1_fee).unwrap(); let expected_step0_change = - (value - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); + (initial_balance - expected_ephemeral - expected_step0_fee) + .expect("sufficient funds"); assert!(expected_step0_change.is_positive()); + let total_sent = (expected_step0_fee + expected_step1_fee + transfer_amount).unwrap(); + // Generate a ZIP 320 proposal, sending to the wallet's default transparent address // expressed as a TEX address. let tex_addr = match default_addr { @@ -599,6 +611,12 @@ pub fn send_multi_step_proposed_transfer( assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); let txids = create_proposed_result.unwrap(); + // Mine the created transactions. + for txid in txids.iter() { + let (h, _) = st.generate_next_block_including(*txid); + st.scan_cached_blocks(h, 1); + } + // Check that there are sent outputs with the correct values. let confirmed_sent: Vec> = txids .iter() @@ -662,12 +680,15 @@ pub fn send_multi_step_proposed_transfer( -ZatBalance::from(expected_ephemeral), ); - (ephemeral_address.unwrap().0, txids) + let ending_balance = st.get_spendable_balance(account_id, 1); + assert_eq!(initial_balance - total_sent, ending_balance.into()); + + (ephemeral_address.unwrap().0, txids, ending_balance) }; // Each transfer should use a different ephemeral address. - let (ephemeral0, txids0) = run_test(&mut st, 0); - let (ephemeral1, txids1) = run_test(&mut st, 1); + let (ephemeral0, txids0, bal_0) = run_test(&mut st, 0, NonNegativeAmount::ZERO); + let (ephemeral1, txids1, bal_1) = run_test(&mut st, 1, bal_0); assert_ne!(ephemeral0, ephemeral1); let height = add_funds(&mut st, value); @@ -708,7 +729,7 @@ pub fn send_multi_step_proposed_transfer( .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(known_addrs.len(), (GAP_LIMIT as usize) + 2); + assert_eq!(known_addrs.len(), (EPHEMERAL_ADDR_GAP_LIMIT as usize) + 2); // Check that the addresses are all distinct. let known_set: HashSet<_> = known_addrs.iter().map(|(addr, _)| addr).collect(); @@ -802,7 +823,10 @@ pub fn send_multi_step_proposed_transfer( .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(new_known_addrs.len(), (GAP_LIMIT as usize) + 11); + assert_eq!( + new_known_addrs.len(), + (EPHEMERAL_ADDR_GAP_LIMIT as usize) + 11 + ); assert!(new_known_addrs.starts_with(&known_addrs)); let reservation_should_succeed = |st: &mut TestState<_, DSF::DataStore, _>, n| { @@ -836,7 +860,10 @@ pub fn send_multi_step_proposed_transfer( ), ) .unwrap(); - assert_eq!(newer_known_addrs.len(), (GAP_LIMIT as usize) + 12 - 5); + assert_eq!( + newer_known_addrs.len(), + (EPHEMERAL_ADDR_GAP_LIMIT as usize) + 12 - 5 + ); assert!(newer_known_addrs.starts_with(&new_known_addrs[5..])); // None of the five transactions created above (two from each proposal and the @@ -898,7 +925,10 @@ pub fn send_multi_step_proposed_transfer( .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(newest_known_addrs.len(), (GAP_LIMIT as usize) + 31); + assert_eq!( + newest_known_addrs.len(), + (EPHEMERAL_ADDR_GAP_LIMIT as usize) + 31 + ); assert!(newest_known_addrs.starts_with(&known_addrs)); assert!(newest_known_addrs[5..].starts_with(&newer_known_addrs)); } diff --git a/zcash_client_backend/src/sync.rs b/zcash_client_backend/src/sync.rs index 2ddce5fbec..f7a544a4ff 100644 --- a/zcash_client_backend/src/sync.rs +++ b/zcash_client_backend/src/sync.rs @@ -135,7 +135,7 @@ where "Refreshing UTXOs for {:?} from height {}", account_id, start_height, ); - refresh_utxos(params, client, db_data, account_id, start_height).await?; + refresh_utxos(params, client, db_data, account_id, start_height, false).await?; } // 5) Get the suggested scan ranges from the wallet database @@ -498,6 +498,7 @@ async fn refresh_utxos( db_data: &mut DbT, account_id: DbT::AccountId, start_height: BlockHeight, + include_ephemeral: bool, ) -> Result<(), Error::Error, TrErr>> where P: Parameters + Send + 'static, @@ -510,7 +511,7 @@ where { let request = service::GetAddressUtxosArg { addresses: db_data - .get_transparent_receivers(account_id) + .get_transparent_receivers(account_id, true, include_ephemeral) .map_err(Error::Wallet)? .into_keys() .map(|addr| addr.encode(params)) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 22b31c43d2..3df3437849 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -9,6 +9,9 @@ and this library adheres to Rust's notion of ### Changed - Migrated to `nonempty 0.11` +- `zcash_client_sqlite::error::SqliteClientError` variants have changed: + - The `EphemeralAddressReuse` variant has been removed and replaced + by a new generalized `AddressReuse` error variant. ## [0.14.0] - 2024-12-16 diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 407358eb95..6f1e1c6e19 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -3,20 +3,21 @@ use std::error; use std::fmt; +use nonempty::NonEmpty; use shardtree::error::ShardTreeError; use zcash_address::ParseError; use zcash_client_backend::data_api::NoteFilter; -use zcash_client_backend::PoolType; use zcash_keys::keys::AddressGenerationError; -use zcash_primitives::zip32; use zcash_primitives::{consensus::BlockHeight, transaction::components::amount::BalanceError}; +use zcash_protocol::{PoolType, TxId}; +use zip32; use crate::{wallet::commitment_tree, AccountUuid}; #[cfg(feature = "transparent-inputs")] use { + ::transparent::address::TransparentAddress, zcash_client_backend::encoding::TransparentCodecError, - zcash_primitives::{legacy::TransparentAddress, transaction::TxId}, }; /// The primary error type for the SQLite wallet backend. @@ -121,16 +122,16 @@ pub enum SqliteClientError { NoteFilterInvalid(NoteFilter), /// The proposal cannot be constructed until transactions with previously reserved - /// ephemeral address outputs have been mined. The parameters are the account UUID and - /// the index that could not safely be reserved. + /// ephemeral address outputs have been mined. The error contains the index that could not + /// safely be reserved. #[cfg(feature = "transparent-inputs")] - ReachedGapLimit(AccountUuid, u32), + ReachedGapLimit(u32), - /// An ephemeral address would be reused. The parameters are the address in string - /// form, and the txid of the earliest transaction in which it is known to have been - /// used. - #[cfg(feature = "transparent-inputs")] - EphemeralAddressReuse(String, TxId), + /// The wallet attempted to create a transaction that would use of one of the wallet's + /// previously-used addresses, potentially creating a problem with on-chain transaction + /// linkability. The returned value contains the string encoding of the address and the txid(s) + /// of the transactions in which it is known to have been used. + AddressReuse(String, NonEmpty), } impl error::Error for SqliteClientError { @@ -187,12 +188,13 @@ impl fmt::Display for SqliteClientError { SqliteClientError::BalanceError(e) => write!(f, "Balance error: {}", e), SqliteClientError::NoteFilterInvalid(s) => write!(f, "Could not evaluate filter query: {:?}", s), #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit(account_id, bad_index) => write!(f, - "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined. \ - The ephemeral address in account {account_id:?} at index {bad_index} could not be safely reserved.", + SqliteClientError::ReachedGapLimit(bad_index) => write!(f, + "The proposal cannot be constructed until transactions with outputs to previously reserved ephemeral addresses have been mined. \ + The ephemeral address at index {bad_index} could not be safely reserved.", ), - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(address_str, txid) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), + SqliteClientError::AddressReuse(address_str, txids) => { + write!(f, "The address {address_str} previously used in txid(s) {:?} would be reused.", txids) + } } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index b1b4c7cb99..b9424ec7f6 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -34,7 +34,7 @@ use incrementalmerkletree::{Marking, Position, Retention}; use nonempty::NonEmpty; -use rusqlite::{self, Connection}; +use rusqlite::{self, named_params, Connection}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, ShardTree}; use std::{ @@ -63,7 +63,7 @@ use zcash_client_backend::{ wallet::{Note, NoteId, ReceivedNote, WalletTransparentOutput}, ShieldedProtocol, TransferType, }; - +use zcash_keys::keys::ReceiverRequirement; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, @@ -82,7 +82,6 @@ use zcash_protocol::PoolType; use { incrementalmerkletree::frontier::Frontier, shardtree::store::{Checkpoint, ShardStore}, - std::collections::BTreeMap, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT, }; @@ -94,7 +93,7 @@ use { }; #[cfg(any(feature = "orchard", feature = "transparent-inputs"))] -use zcash_keys::keys::ReceiverRequirement; +use std::collections::BTreeMap; #[cfg(feature = "multicore")] use maybe_rayon::{ @@ -131,9 +130,10 @@ pub mod chain; pub mod error; pub mod wallet; use wallet::{ + chain_tip_height, commitment_tree::{self, put_shard_roots}, common::spendable_notes_meta, - SubtreeProgressEstimator, + KeyScope, SubtreeProgressEstimator, }; #[cfg(test)] @@ -179,7 +179,7 @@ pub(crate) const UA_TRANSPARENT: ReceiverRequirement = ReceiverRequirement::Requ /// events". Examples of these include: /// - Restoring a wallet from a backed-up seed. /// - Importing the same viewing key into two different wallet instances. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct AccountUuid(#[cfg_attr(feature = "serde", serde(with = "uuid::serde::compact"))] Uuid); @@ -210,7 +210,7 @@ impl AccountUuid { /// /// This is an ephemeral value for efficiently and generically working with accounts in a /// [`WalletDb`]. To reference accounts in external contexts, use [`AccountUuid`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] pub(crate) struct AccountRef(u32); /// This implementation is retained under `#[cfg(test)]` for pre-AccountUuid testing. @@ -243,10 +243,46 @@ pub struct UtxoId(pub i64); #[derive(Debug, Copy, Clone, PartialEq, Eq)] struct TxRef(pub i64); +/// A newtype wrapper for sqlite primary key values for the addresses table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct AddressRef(pub(crate) i64); + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct GapLimits { + external: u32, + transparent_internal: u32, + ephemeral: u32, +} + +impl GapLimits { + pub(crate) fn external(&self) -> u32 { + self.external + } + + pub(crate) fn transparent_internal(&self) -> u32 { + self.transparent_internal + } + + pub(crate) fn ephemeral(&self) -> u32 { + self.ephemeral + } +} + +impl Default for GapLimits { + fn default() -> Self { + Self { + external: 20, + transparent_internal: 3, + ephemeral: 3, + } + } +} + /// A wrapper for the SQLite connection to the wallet database. pub struct WalletDb { conn: C, params: P, + gap_limits: GapLimits, } /// A wrapper for a SQLite transaction affecting the wallet database. @@ -263,7 +299,11 @@ impl WalletDb { pub fn for_path>(path: F, params: P) -> Result { Connection::open(path).and_then(move |conn| { rusqlite::vtab::array::load_module(&conn)?; - Ok(WalletDb { conn, params }) + Ok(WalletDb { + conn, + params, + gap_limits: GapLimits::default(), + }) }) } @@ -275,6 +315,7 @@ impl WalletDb { let mut wdb = WalletDb { conn: SqlTransaction(&tx), params: self.params.clone(), + gap_limits: self.gap_limits, }; let result = f(&mut wdb)?; tx.commit()?; @@ -622,8 +663,21 @@ impl, P: consensus::Parameters> WalletRead for W fn get_transparent_receivers( &self, account: Self::AccountId, + include_change: bool, + include_ephemeral: bool, ) -> Result>, Self::Error> { - wallet::transparent::get_transparent_receivers(self.conn.borrow(), &self.params, account) + let key_scopes: &[KeyScope] = match (include_change, include_ephemeral) { + (true, true) => &[KeyScope::EXTERNAL, KeyScope::INTERNAL, KeyScope::Ephemeral], + (true, false) => &[KeyScope::EXTERNAL, KeyScope::INTERNAL], + (false, true) => &[KeyScope::EXTERNAL, KeyScope::Ephemeral], + (false, false) => &[KeyScope::EXTERNAL], + }; + wallet::transparent::get_transparent_receivers( + self.conn.borrow(), + &self.params, + account, + key_scopes, + ) } #[cfg(feature = "transparent-inputs")] @@ -665,7 +719,7 @@ impl, P: consensus::Parameters> WalletRead for W self.conn.borrow(), &self.params, account_id, - index_range.map(|i| i.start.index()..i.end.index()), + index_range, ) } @@ -708,7 +762,6 @@ impl, P: consensus::Parameters> WalletTest for W protocol: ShieldedProtocol, ) -> Result, ::Error> { use crate::wallet::pool_code; - use rusqlite::named_params; let mut stmt_sent_notes = self.conn.borrow().prepare( "SELECT output_index @@ -735,29 +788,35 @@ impl, P: consensus::Parameters> WalletTest for W &self, txid: &TxId, ) -> Result, ::Error> { - let mut stmt_sent = self - .conn.borrow() - .prepare( - "SELECT value, to_address, ephemeral_addresses.address, ephemeral_addresses.address_index - FROM sent_notes - JOIN transactions ON transactions.id_tx = sent_notes.tx - LEFT JOIN ephemeral_addresses ON ephemeral_addresses.used_in_tx = sent_notes.tx - WHERE transactions.txid = ? - ORDER BY value", - )?; + let mut stmt_sent = self.conn.borrow().prepare( + "SELECT value, to_address, + a.cached_transparent_receiver_address, a.transparent_child_index + FROM sent_notes + JOIN transactions t ON t.id_tx = sent_notes.tx + LEFT JOIN transparent_received_outputs tro ON tro.transaction_id = t.id_tx + LEFT JOIN addresses a ON a.id = tro.address_id AND a.key_scope = :key_scope + WHERE t.txid = :txid + ORDER BY value", + )?; let sends = stmt_sent - .query_map(rusqlite::params![txid.as_ref()], |row| { - let v = row.get(0)?; - let to_address = row - .get::<_, Option>(1)? - .and_then(|s| Address::decode(&self.params, &s)); - let ephemeral_address = row - .get::<_, Option>(2)? - .and_then(|s| Address::decode(&self.params, &s)); - let address_index: Option = row.get(3)?; - Ok((v, to_address, ephemeral_address.zip(address_index))) - })? + .query_map( + named_params![ + ":txid": txid.as_ref(), + ":key_scope": KeyScope::Ephemeral.encode() + ], + |row| { + let v = row.get(0)?; + let to_address = row + .get::<_, Option>(1)? + .and_then(|s| Address::decode(&self.params, &s)); + let ephemeral_address = row + .get::<_, Option>(2)? + .and_then(|s| Address::decode(&self.params, &s)); + let address_index: Option = row.get(3)?; + Ok((v, to_address, ephemeral_address.zip(address_index))) + }, + )? .map(|res| { let (amount, external_recipient, ephemeral_address) = res?; Ok::<_, ::Error>(OutputOfSentTx::from_parts( @@ -862,6 +921,7 @@ impl WalletWrite for WalletDb let account = wallet::add_account( wdb.conn.0, &wdb.params, + &wdb.gap_limits, account_name, &AccountSource::Derived { derivation: Zip32Derivation::new(seed_fingerprint, zip32_account_index), @@ -899,6 +959,7 @@ impl WalletWrite for WalletDb let account = wallet::add_account( wdb.conn.0, &wdb.params, + &wdb.gap_limits, account_name, &AccountSource::Derived { derivation: Zip32Derivation::new(seed_fingerprint, account_index), @@ -924,6 +985,7 @@ impl WalletWrite for WalletDb wallet::add_account( wdb.conn.0, &wdb.params, + &wdb.gap_limits, account_name, &AccountSource::Imported { purpose, @@ -955,14 +1017,16 @@ impl WalletWrite for WalletDb }; let (addr, diversifier_index) = ufvk.find_address(search_from, request)?; - let account_id = wallet::get_account_ref(wdb.conn.0, account_uuid)?; - wallet::insert_address( + let chain_tip_height = chain_tip_height(wdb.conn.0)? + .ok_or(SqliteClientError::ChainHeightUnknown)?; + wallet::upsert_address( wdb.conn.0, &wdb.params, account_id, diversifier_index, &addr, + Some(chain_tip_height), )?; Ok(Some(addr)) @@ -972,6 +1036,15 @@ impl WalletWrite for WalletDb ) } + fn get_address_for_index( + &mut self, + _account: Self::AccountId, + _diversifier_index: DiversifierIndex, + _request: Option, + ) -> Result, Self::Error> { + todo!() + } + fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error> { let tx = self.conn.transaction()?; wallet::scanning::update_chain_tip(&tx, &self.params, tip_height)?; @@ -1014,11 +1087,14 @@ impl WalletWrite for WalletDb ), }; + #[cfg(feature = "transparent-inputs")] + let mut receiving_accounts = BTreeMap::new(); let mut sapling_commitments = vec![]; #[cfg(feature = "orchard")] let mut orchard_commitments = vec![]; let mut last_scanned_height = None; let mut note_positions = vec![]; + for block in blocks.into_iter() { if last_scanned_height .iter() @@ -1069,7 +1145,19 @@ impl WalletWrite for WalletDb .transpose()? .flatten(); - wallet::sapling::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + let account_id = wallet::sapling::put_received_note( + wdb.conn.0, + &wdb.params, + output, + tx_row, + Some(block.height()), + spent_in, + )?; + + #[cfg(feature = "transparent-inputs")] + if output.recipient_key_scope() == Some(zip32::Scope::External) { + receiving_accounts.insert(account_id, KeyScope::EXTERNAL); + } } #[cfg(feature = "orchard")] for output in tx.orchard_outputs() { @@ -1087,7 +1175,19 @@ impl WalletWrite for WalletDb .transpose()? .flatten(); - wallet::orchard::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + let account_id = wallet::orchard::put_received_note( + wdb.conn.0, + &wdb.params, + output, + tx_row, + Some(block.height()), + spent_in, + )?; + + #[cfg(feature = "transparent-inputs")] + if output.recipient_key_scope() == Some(zip32::Scope::External) { + receiving_accounts.insert(account_id, KeyScope::EXTERNAL); + } } } @@ -1166,6 +1266,18 @@ impl WalletWrite for WalletDb )?; } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in receiving_accounts { + wallet::transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + } + // We will have a start position and a last scanned height in all cases where // `blocks` is non-empty. if let Some(last_scanned_height) = last_scanned_height { @@ -1412,11 +1524,27 @@ impl WalletWrite for WalletDb _output: &WalletTransparentOutput, ) -> Result { #[cfg(feature = "transparent-inputs")] - return wallet::transparent::put_received_transparent_utxo( - &self.conn, - &self.params, - _output, - ); + { + self.transactionally(|wdb| { + let (account_id, key_scope, utxo_id) = + wallet::transparent::put_received_transparent_utxo( + wdb.conn.0, + &wdb.params, + _output, + )?; + + wallet::transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + + Ok(utxo_id) + }) + } #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -1428,7 +1556,9 @@ impl WalletWrite for WalletDb &mut self, d_tx: DecryptedTransaction, ) -> Result<(), Self::Error> { - self.transactionally(|wdb| wallet::store_decrypted_tx(wdb.conn.0, &wdb.params, d_tx)) + self.transactionally(|wdb| { + wallet::store_decrypted_tx(wdb.conn.0, &wdb.params, &wdb.gap_limits, d_tx) + }) } fn store_transactions_to_be_sent( @@ -1455,12 +1585,16 @@ impl WalletWrite for WalletDb ) -> Result, Self::Error> { self.transactionally(|wdb| { let account_id = wallet::get_account_ref(wdb.conn.0, account_id)?; - wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses( + let reserved = wallet::transparent::reserve_next_n_addresses( wdb.conn.0, &wdb.params, account_id, + wallet::KeyScope::Ephemeral, + wdb.gap_limits.ephemeral(), n, - ) + )?; + + Ok(reserved.into_iter().map(|(_, a, m)| (a, m)).collect()) }) } @@ -1987,6 +2121,12 @@ mod tests { .build(); let account = st.test_account().cloned().unwrap(); + // We have to have the chain tip height in order to allocate new addresses, to record the + // exposed-at height. + st.wallet_mut() + .update_chain_tip(account.birthday().height()) + .unwrap(); + let current_addr = st.wallet().get_current_address(account.id()).unwrap(); assert!(current_addr.is_some()); @@ -2239,7 +2379,10 @@ mod tests { let ufvk = account.usk().to_unified_full_viewing_key(); let (taddr, _) = account.usk().default_transparent_address(); - let receivers = st.wallet().get_transparent_receivers(account.id()).unwrap(); + let receivers = st + .wallet() + .get_transparent_receivers(account.id(), false, false) + .unwrap(); // The receiver for the default UA should be in the set. assert!(receivers.contains_key( diff --git a/zcash_client_sqlite/src/testing/db.rs b/zcash_client_sqlite/src/testing/db.rs index c05eb23bc9..9aeb4fc54e 100644 --- a/zcash_client_sqlite/src/testing/db.rs +++ b/zcash_client_sqlite/src/testing/db.rs @@ -31,6 +31,7 @@ use zcash_primitives::{ transaction::{components::amount::NonNegativeAmount, Transaction, TxId}, }; use zcash_protocol::{consensus::BlockHeight, local_consensus::LocalNetwork, memo::Memo}; +use zip32::DiversifierIndex; use crate::{ error::SqliteClientError, diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 1b5132a36f..95edb8be85 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -48,11 +48,11 @@ pub(crate) fn send_multi_step_proposed_transfer() { zcash_client_backend::data_api::testing::pool::send_multi_step_proposed_transfer::( TestDbFactory::default(), BlockCache::new(), - |e, account_id, expected_bad_index| { + |e, _, expected_bad_index| { matches!( e, - crate::error::SqliteClientError::ReachedGapLimit(acct, bad_index) - if acct == &account_id && bad_index == &expected_bad_index) + crate::error::SqliteClientError::ReachedGapLimit(bad_index) + if bad_index == &expected_bad_index) }, ) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index af3d2fcf29..c68ba023dd 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -69,28 +69,25 @@ use incrementalmerkletree::{Marking, Retention}; use rusqlite::{self, named_params, params, Connection, OptionalExtension}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; +use tracing::{debug, warn}; use uuid::Uuid; -use zcash_client_backend::data_api::{ - AccountPurpose, DecryptedTransaction, Progress, TransactionDataRequest, TransactionStatus, - Zip32Derivation, -}; -use zip32::fingerprint::SeedFingerprint; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryFrom; use std::io::{self, Cursor}; use std::marker::PhantomData; use std::num::NonZeroU32; use std::ops::RangeInclusive; -use tracing::{debug, warn}; - +use ::transparent::keys::{NonHardenedChildIndex, TransparentKeyScope}; use zcash_address::ZcashAddress; use zcash_client_backend::{ data_api::{ scanning::{ScanPriority, ScanRange}, - Account as _, AccountBalance, AccountBirthday, AccountSource, BlockMetadata, Ratio, - SentTransaction, SentTransactionOutput, WalletSummary, SAPLING_SHARD_HEIGHT, + Account as _, AccountBalance, AccountBirthday, AccountPurpose, AccountSource, + BlockMetadata, DecryptedTransaction, Progress, Ratio, SentTransaction, + SentTransactionOutput, TransactionDataRequest, TransactionStatus, WalletSummary, + Zip32Derivation, SAPLING_SHARD_HEIGHT, }, encoding::AddressCodec, keys::UnifiedFullViewingKey, @@ -106,15 +103,17 @@ use zcash_keys::{ }; use zcash_primitives::{ block::BlockHash, - consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, - memo::{Memo, MemoBytes}, merkle_tree::read_commitment_tree, transaction::{ components::{amount::NonNegativeAmount, Amount, OutPoint}, Transaction, TransactionData, TxId, }, }; -use zip32::{self, DiversifierIndex, Scope}; +use zcash_protocol::{ + consensus::{self, BlockHeight, BranchId, NetworkConstants as _, NetworkUpgrade, Parameters}, + memo::{Memo, MemoBytes}, +}; +use zip32::{self, fingerprint::SeedFingerprint, DiversifierIndex}; use crate::{ error::SqliteClientError, @@ -122,7 +121,7 @@ use crate::{ AccountRef, SqlTransaction, TransferType, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; -use crate::{AccountUuid, TxRef, VERIFY_LOOKAHEAD}; +use crate::{AccountUuid, AddressRef, GapLimits, TxRef, VERIFY_LOOKAHEAD}; #[cfg(feature = "transparent-inputs")] use zcash_primitives::transaction::components::TxOut; @@ -213,6 +212,7 @@ pub(crate) enum ViewingKey { /// An account stored in a `zcash_client_sqlite` database. #[derive(Debug, Clone)] pub struct Account { + id: AccountRef, uuid: AccountUuid, name: Option, kind: AccountSource, @@ -231,6 +231,10 @@ impl Account { ) -> Result<(UnifiedAddress, DiversifierIndex), AddressGenerationError> { self.uivk().default_address(request) } + + pub(crate) fn internal_id(&self) -> AccountRef { + self.id + } } impl zcash_client_backend::data_api::Account for Account { @@ -342,21 +346,90 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { } } -pub(crate) fn scope_code(scope: Scope) -> i64 { - match scope { - Scope::External => 0i64, - Scope::Internal => 1i64, +/// An enumeration of the scopes of keys that are generated by the `zcash_client_sqlite` +/// implementation of the `WalletWrite` trait. +/// +/// This extends the [`zip32::Scope`] type to include the custom scope used to generate keys for +/// ephemeral transparent addresses. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum KeyScope { + /// A key scope corresponding to a [`zip32::Scope`]. + Zip32(zip32::Scope), + /// An ephemeral transparent address, which is derived from an account's transparent + /// [`AccountPubKey`] with the BIP 44 path `change` level index set to the value `2`. + /// + /// [`AccountPubKey`]: zcash_primitives::legacy::keys::AccountPubKey + Ephemeral, +} + +impl KeyScope { + pub(crate) const EXTERNAL: KeyScope = KeyScope::Zip32(zip32::Scope::External); + pub(crate) const INTERNAL: KeyScope = KeyScope::Zip32(zip32::Scope::Internal); + + pub(crate) fn encode(&self) -> i64 { + match self { + KeyScope::Zip32(zip32::Scope::External) => 0i64, + KeyScope::Zip32(zip32::Scope::Internal) => 1i64, + KeyScope::Ephemeral => 2i64, + } + } + + pub(crate) fn decode(code: i64) -> Result { + match code { + 0i64 => Ok(KeyScope::EXTERNAL), + 1i64 => Ok(KeyScope::INTERNAL), + 2i64 => Ok(KeyScope::Ephemeral), + other => Err(SqliteClientError::CorruptedData(format!( + "Invalid key scope code: {}", + other + ))), + } + } +} + +impl From for KeyScope { + fn from(value: zip32::Scope) -> Self { + KeyScope::Zip32(value) + } +} + +impl From for TransparentKeyScope { + fn from(value: KeyScope) -> Self { + match value { + KeyScope::Zip32(scope) => scope.into(), + KeyScope::Ephemeral => TransparentKeyScope::custom(2).expect("valid scope"), + } } } -pub(crate) fn parse_scope(code: i64) -> Option { - match code { - 0i64 => Some(Scope::External), - 1i64 => Some(Scope::Internal), - _ => None, +impl TryFrom for zip32::Scope { + type Error = (); + + fn try_from(value: KeyScope) -> Result { + match value { + KeyScope::Zip32(scope) => Ok(scope), + KeyScope::Ephemeral => Err(()), + } } } +pub(crate) fn encode_diversifier_index_be(idx: DiversifierIndex) -> [u8; 11] { + let mut di_be = *idx.as_bytes(); + di_be.reverse(); + di_be +} + +pub(crate) fn decode_diversifier_index_be( + di_be: &[u8], +) -> Result { + let mut di_be: [u8; 11] = di_be.try_into().map_err(|_| { + SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) + })?; + di_be.reverse(); + + Ok(DiversifierIndex::from(di_be)) +} + pub(crate) fn memo_repr(memo: Option<&MemoBytes>) -> Option<&[u8]> { memo.map(|m| { if m == &MemoBytes::empty() { @@ -388,6 +461,7 @@ pub(crate) fn max_zip32_account_index( pub(crate) fn add_account( conn: &rusqlite::Transaction, params: &P, + gap_limits: &GapLimits, account_name: &str, kind: &AccountSource, viewing_key: ViewingKey, @@ -510,6 +584,7 @@ pub(crate) fn add_account( })?; let account = Account { + id: account_id, name: Some(account_name.to_owned()), uuid: account_uuid, kind: kind.clone(), @@ -613,11 +688,21 @@ pub(crate) fn add_account( // key has fewer components than the wallet supports (most likely due to this being an // imported viewing key), derive an address containing the common subset of receivers. let (address, d_idx) = account.default_address(None)?; - insert_address(conn, params, account_id, d_idx, &address)?; + upsert_address( + conn, + params, + account_id, + d_idx, + &address, + Some(birthday.height()), + )?; - // Initialize the `ephemeral_addresses` table. + // Pre-generate transparent addresses up to the gap limits for the external, internal, + // and ephemeral key scopes. #[cfg(feature = "transparent-inputs")] - transparent::ephemeral::init_account(conn, params, account_id)?; + for key_scope in [KeyScope::EXTERNAL, KeyScope::INTERNAL, KeyScope::Ephemeral] { + transparent::generate_gap_addresses(conn, params, account_id, key_scope, gap_limits, None)?; + } Ok(account) } @@ -627,26 +712,31 @@ pub(crate) fn get_current_address( params: &P, account_uuid: AccountUuid, ) -> Result, SqliteClientError> { + let ua_prefix = params.network_type().hrp_unified_address(); // This returns the most recently generated address. let addr: Option<(String, Vec)> = conn .query_row( - "SELECT address, diversifier_index_be - FROM addresses - JOIN accounts ON addresses.account_id = accounts.id - WHERE accounts.uuid = :account_uuid - ORDER BY diversifier_index_be DESC - LIMIT 1", - named_params![":account_uuid": account_uuid.0], + &format!( + "SELECT address, diversifier_index_be + FROM addresses + JOIN accounts ON addresses.account_id = accounts.id + WHERE accounts.uuid = :account_uuid + AND key_scope = :key_scope + AND address LIKE '{ua_prefix}%' + AND exposed_at_height IS NOT NULL + ORDER BY diversifier_index_be DESC + LIMIT 1" + ), + named_params![ + ":account_uuid": account_uuid.0, + ":key_scope": KeyScope::EXTERNAL.encode() + ], |row| Ok((row.get(0)?, row.get(1)?)), ) .optional()?; addr.map(|(addr_str, di_vec)| { - let mut di_be: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di_be.reverse(); - + let diversifier_index = decode_diversifier_index_be(&di_vec)?; Address::decode(params, &addr_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) @@ -658,47 +748,66 @@ pub(crate) fn get_current_address( addr_str, ))), }) - .map(|addr| (addr, DiversifierIndex::from(di_be))) + .map(|addr| (addr, diversifier_index)) }) .transpose() } -/// Adds the given address and diversifier index to the addresses table. +/// Adds the given external address and diversifier index to the addresses table. /// -/// Returns the database row for the newly-inserted address. -pub(crate) fn insert_address( +/// Returns the primary key identifier for the newly-inserted address. +pub(crate) fn upsert_address( conn: &rusqlite::Connection, params: &P, account_id: AccountRef, diversifier_index: DiversifierIndex, address: &UnifiedAddress, -) -> Result<(), SqliteClientError> { + exposed_at_height: Option, +) -> Result { let mut stmt = conn.prepare_cached( "INSERT INTO addresses ( account_id, diversifier_index_be, + key_scope, address, - cached_transparent_receiver_address + transparent_child_index, + cached_transparent_receiver_address, + exposed_at_height ) VALUES ( :account_id, :diversifier_index_be, + :key_scope, :address, - :cached_transparent_receiver_address - )", + :transparent_child_index, + :cached_transparent_receiver_address, + :exposed_at_height + ) + ON CONFLICT (account_id, diversifier_index_be, key_scope) DO UPDATE + SET exposed_at_height = :exposed_at_height + RETURNING id", )?; - // the diversifier index is stored in big-endian order to allow sorting - let mut di_be = *diversifier_index.as_bytes(); - di_be.reverse(); - stmt.execute(named_params![ - ":account_id": account_id.0, - ":diversifier_index_be": &di_be[..], - ":address": &address.encode(params), - ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), - ])?; + #[cfg(feature = "transparent-inputs")] + let transparent_child_index = NonHardenedChildIndex::try_from(diversifier_index) + .ok() + .map(|i| i.index()); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_child_index: Option = None; - Ok(()) + stmt.query_row( + named_params![ + ":account_id": account_id.0, + // the diversifier index is stored in big-endian order to allow sorting + ":diversifier_index_be": encode_diversifier_index_be(diversifier_index), + ":key_scope": KeyScope::EXTERNAL.encode(), + ":address": &address.encode(params), + ":transparent_child_index": transparent_child_index, + ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), + ":exposed_at_height": exposed_at_height.map(u32::from) + ], + |row| row.get(0).map(AddressRef) + ).map_err(SqliteClientError::from) } /// Returns the [`UnifiedFullViewingKey`]s for the wallet. @@ -734,6 +843,7 @@ fn parse_account_row( row: &rusqlite::Row<'_>, params: &P, ) -> Result { + let account_id = AccountRef(row.get("id")?); let account_name = row.get("name")?; let account_uuid = AccountUuid(row.get("uuid")?); let kind = parse_account_source( @@ -767,6 +877,7 @@ fn parse_account_row( }; Ok(Account { + id: account_id, name: account_name, uuid: account_uuid, kind, @@ -781,7 +892,7 @@ pub(crate) fn get_account( ) -> Result, SqliteClientError> { let mut stmt = conn.prepare_cached( r#" - SELECT name, uuid, account_kind, + SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -804,7 +915,7 @@ pub(crate) fn get_account_internal( ) -> Result, SqliteClientError> { let mut stmt = conn.prepare_cached( r#" - SELECT name, uuid, account_kind, + SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -840,7 +951,7 @@ pub(crate) fn get_account_for_ufvk( let transparent_item: Option> = None; let mut stmt = conn.prepare( - "SELECT name, uuid, account_kind, + "SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -878,7 +989,7 @@ pub(crate) fn get_derived_account( account_index: zip32::AccountId, ) -> Result, SqliteClientError> { let mut stmt = conn.prepare( - "SELECT name, key_source, uuid, ufvk + "SELECT id, name, key_source, uuid, ufvk FROM accounts WHERE hd_seed_fingerprint = :hd_seed_fingerprint AND hd_account_index = :hd_account_index", @@ -890,6 +1001,7 @@ pub(crate) fn get_derived_account( ":hd_account_index": u32::from(account_index), ], |row| { + let account_id = AccountRef(row.get("id")?); let account_name = row.get("name")?; let key_source = row.get("key_source")?; let account_uuid = AccountUuid(row.get("uuid")?); @@ -906,6 +1018,7 @@ pub(crate) fn get_derived_account( }), }?; Ok(Account { + id: account_id, name: account_name, uuid: account_uuid, kind: AccountSource::Derived { @@ -1963,20 +2076,6 @@ pub(crate) fn get_account_ref( .ok_or(SqliteClientError::AccountUnknown) } -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_account_uuid( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - conn.query_row( - "SELECT uuid FROM accounts WHERE id = :account_id", - named_params! {":account_id": account_id.0}, - |row| row.get("uuid").map(AccountUuid), - ) - .optional()? - .ok_or(SqliteClientError::AccountUnknown) -} - /// Returns the minimum and maximum heights of blocks in the chain which may be scanned. pub(crate) fn chain_tip_height( conn: &rusqlite::Connection, @@ -2282,6 +2381,9 @@ pub(crate) fn store_transaction_to_be_sent( transparent::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, utxo_outpoint)?; } + #[cfg(feature = "transparent-inputs")] + let mut receiving_accounts = BTreeMap::new(); + for output in sent_tx.outputs() { insert_sent_output( wdb.conn.0, @@ -2292,6 +2394,13 @@ pub(crate) fn store_transaction_to_be_sent( )?; match output.recipient() { + Recipient::External { + recipient_address: _addr, + output_pool: _pool, + .. + } => { + // Nothing to do for external recipients. + } Recipient::InternalAccount { receiving_account, note: Note::Sapling(note), @@ -2299,6 +2408,7 @@ pub(crate) fn store_transaction_to_be_sent( } => { sapling::put_received_note( wdb.conn.0, + &wdb.params, &DecryptedOutput::new( output.output_index(), note.clone(), @@ -2309,6 +2419,7 @@ pub(crate) fn store_transaction_to_be_sent( TransferType::WalletInternal, ), tx_ref, + Some(sent_tx.target_height()), None, )?; } @@ -2320,6 +2431,7 @@ pub(crate) fn store_transaction_to_be_sent( } => { orchard::put_received_note( wdb.conn.0, + &wdb.params, &DecryptedOutput::new( output.output_index(), *note, @@ -2330,16 +2442,25 @@ pub(crate) fn store_transaction_to_be_sent( TransferType::WalletInternal, ), tx_ref, + Some(sent_tx.target_height()), None, )?; } #[cfg(feature = "transparent-inputs")] Recipient::EphemeralTransparent { - receiving_account, ephemeral_address, outpoint_metadata, + .. } => { - transparent::put_transparent_output( + // First check to verify that creation of this output does not result in reuse of + // an ephemeral address. + transparent::check_address_reuse( + wdb.conn.0, + &wdb.params, + &Address::Transparent(*ephemeral_address), + )?; + + let (account_id, _, _) = transparent::put_transparent_output( wdb.conn.0, &wdb.params, outpoint_metadata, @@ -2349,20 +2470,26 @@ pub(crate) fn store_transaction_to_be_sent( }, None, ephemeral_address, - *receiving_account, true, )?; - transparent::ephemeral::mark_ephemeral_address_as_used( - wdb.conn.0, - &wdb.params, - ephemeral_address, - tx_ref, - )?; + + receiving_accounts.insert(account_id, KeyScope::Ephemeral); } - _ => {} } } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in receiving_accounts { + transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + } + // Add the transaction to the set to be queried for transaction status. This is only necessary // at present for fully transparent transactions, because any transaction with a shielded // component will be detected via ordinary chain scanning and/or nullifier checking. @@ -2568,6 +2695,7 @@ pub(crate) fn truncate_to_height( let mut wdb = WalletDb { conn: SqlTransaction(conn), params: params.clone(), + gap_limits: GapLimits::default(), }; wdb.with_sapling_tree_mut(|tree| { tree.truncate_to_checkpoint(&truncation_height)?; @@ -2724,6 +2852,7 @@ pub(crate) fn put_block( pub(crate) fn store_decrypted_tx( conn: &rusqlite::Transaction, params: &P, + gap_limits: &GapLimits, d_tx: DecryptedTransaction, ) -> Result<(), SqliteClientError> { let tx_ref = put_tx_data(conn, d_tx.tx(), None, None, None)?; @@ -2749,6 +2878,9 @@ pub(crate) fn store_decrypted_tx( #[cfg(feature = "transparent-inputs")] let mut tx_has_wallet_outputs = false; + #[cfg(feature = "transparent-inputs")] + let mut receiving_accounts = BTreeMap::new(); + for output in d_tx.sapling_outputs() { #[cfg(feature = "transparent-inputs")] { @@ -2781,7 +2913,14 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::WalletInternal => { - sapling::put_received_note(conn, output, tx_ref, None)?; + sapling::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; let recipient = Recipient::InternalAccount { receiving_account: *output.account(), @@ -2801,7 +2940,17 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::Incoming => { - sapling::put_received_note(conn, output, tx_ref, None)?; + let _account_id = sapling::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; + + #[cfg(feature = "transparent-inputs")] + receiving_accounts.insert(_account_id, KeyScope::EXTERNAL); if let Some(account_id) = funding_account { let recipient = Recipient::InternalAccount { @@ -2871,7 +3020,14 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::WalletInternal => { - orchard::put_received_note(conn, output, tx_ref, None)?; + orchard::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; let recipient = Recipient::InternalAccount { receiving_account: *output.account(), @@ -2891,7 +3047,17 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::Incoming => { - orchard::put_received_note(conn, output, tx_ref, None)?; + let _account_id = orchard::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; + + #[cfg(feature = "transparent-inputs")] + receiving_accounts.insert(_account_id, KeyScope::EXTERNAL); if let Some(account_id) = funding_account { // Even if the recipient address is external, record the send as internal. @@ -2957,18 +3123,9 @@ pub(crate) fn store_decrypted_tx( address.encode(params) ); - // The transaction is not necessarily mined yet, but we want to record - // that an output to the address was seen in this tx anyway. This will - // advance the gap regardless of whether it is mined, but an output in - // an unmined transaction won't advance the range of safe indices. - #[cfg(feature = "transparent-inputs")] - transparent::ephemeral::mark_ephemeral_address_as_seen( - conn, params, &address, tx_ref, - )?; - // If the output belongs to the wallet, add it to `transparent_received_outputs`. #[cfg(feature = "transparent-inputs")] - if let Some(account_uuid) = + if let Some((account_uuid, key_scope)) = transparent::find_account_uuid_for_transparent_address(conn, params, &address)? { debug!( @@ -2977,7 +3134,7 @@ pub(crate) fn store_decrypted_tx( output_index, account_uuid ); - transparent::put_transparent_output( + let (account_id, _, _) = transparent::put_transparent_output( conn, params, &OutPoint::new( @@ -2987,10 +3144,11 @@ pub(crate) fn store_decrypted_tx( txout, d_tx.mined_height(), &address, - account_uuid, false, )?; + receiving_accounts.insert(account_id, key_scope); + // Since the wallet created the transparent output, we need to ensure // that any transparent inputs belonging to the wallet will be // discovered. @@ -3063,6 +3221,11 @@ pub(crate) fn store_decrypted_tx( } } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in receiving_accounts { + transparent::generate_gap_addresses(conn, params, account_id, key_scope, gap_limits, None)?; + } + // If the transaction has outputs that belong to the wallet as well as transparent // inputs, we may need to download the transactions corresponding to the transparent // prevout references to determine whether the transaction was created (at least in @@ -3141,10 +3304,14 @@ pub(crate) fn select_receiving_address( "SELECT address FROM addresses JOIN accounts ON accounts.id = addresses.account_id - WHERE accounts.uuid = :account_uuid", + WHERE accounts.uuid = :account_uuid + AND key_scope = :key_scope", )?; - let mut result = stmt.query(named_params! { ":account_uuid": account.0 })?; + let mut result = stmt.query(named_params! { + ":account_uuid": account.0, + ":key_scope": KeyScope::EXTERNAL.encode(), + })?; while let Some(row) = result.next()? { let addr_str = row.get::<_, String>(0)?; let decoded = addr_str.parse::()?; @@ -3395,7 +3562,7 @@ fn flag_previously_received_change( ), named_params! { ":tx": tx_ref.0, - ":internal_scope": scope_code(Scope::Internal) + ":internal_scope": KeyScope::INTERNAL.encode() }, ) }; diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 6520ac1834..b2a0b613b8 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -13,9 +13,7 @@ // from showing up in `cargo doc --document-private-items`. #![allow(dead_code)] -use static_assertions::const_assert_eq; - -use zcash_client_backend::data_api::{scanning::ScanPriority, GAP_LIMIT}; +use zcash_client_backend::data_api::scanning::ScanPriority; use zcash_protocol::consensus::{NetworkUpgrade, Parameters}; use crate::wallet::scanning::priority_code; @@ -63,85 +61,56 @@ pub(super) const INDEX_ACCOUNTS_UIVK: &str = pub(super) const INDEX_HD_ACCOUNT: &str = r#"CREATE UNIQUE INDEX hd_account ON accounts (hd_seed_fingerprint, hd_account_index)"#; -/// Stores diversified Unified Addresses that have been generated from accounts in the -/// wallet. +/// Stores addresses that have been generated from accounts in the wallet. +/// +/// ### Columns /// -/// - The `cached_transparent_receiver_address` column contains the transparent receiver component -/// of the UA. It is cached directly in the table to make account lookups for transparent outputs -/// more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. +/// - `account_id`: the account whose IVK was used to derive this address. +/// - `diversifier_index_be`: the diversifier index at which this address was derived. +/// - `key_scope`: the key scope for which this address was derived. +/// - `address`: The Unified, Sapling, or transparent address. For Unified and Sapling addresses, +/// only external-key scoped addresses should be stored in this table; for purely transparent +/// addresses, this may be an internal-scope (change) address, so that we can provide +/// compatibility with HD-derived change addresses produced by transparent-only wallets. +/// - `transparent_child_index`: the diversifier index, if it is in the range of a non-hardened +/// transparent address index. This is used for gap limit handling and is always populated if the +/// diversifier index is in that range; since the diversifier index is stored as a byte array we +/// cannot use SQL integer operations on it and thus need it as an integer as well. +/// - `cached_transparent_receiver_address`: the transparent receiver component of address (which +/// may be the same as `address` in the case of an internal-scope transparent change address or a +/// ZIP 320 interstitial address). It is cached directly in the table to make account lookups for +/// transparent outputs more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. +/// - `used_in_tx`: a transaction (the first observed by the wallet) in which the address was used. +/// This column is essentially a flag; rows for which this column is set to `null` correspond to +/// addresses that have been reserved by the gap-limit handling process but which have not yet +/// been used in the creation of a transaction or observed in a mined transaction. pub(super) const TABLE_ADDRESSES: &str = r#" CREATE TABLE "addresses" ( - account_id INTEGER NOT NULL, + id INTEGER NOT NULL PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES accounts(id), diversifier_index_be BLOB NOT NULL, + key_scope INTEGER NOT NULL DEFAULT 0, address TEXT NOT NULL, + transparent_child_index INTEGER, cached_transparent_receiver_address TEXT, - FOREIGN KEY (account_id) REFERENCES accounts(id), - CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be) + exposed_at_height INTEGER, + CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be, key_scope), + CONSTRAINT transparent_index_consistency CHECK ( + (transparent_child_index IS NOT NULL) == (cached_transparent_receiver_address IS NOT NULL) + ) )"#; pub(super) const INDEX_ADDRESSES_ACCOUNTS: &str = r#" -CREATE INDEX "addresses_accounts" ON "addresses" ( - "account_id" ASC +CREATE INDEX idx_addresses_accounts ON addresses ( + account_id ASC +)"#; +pub(super) const INDEX_ADDRESSES_INDICES: &str = r#" +CREATE INDEX idx_addresses_indices ON addresses ( + diversifier_index_be ASC +)"#; +pub(super) const INDEX_ADDRESSES_T_INDICES: &str = r#" +CREATE INDEX idx_addresses_t_indices ON addresses ( + transparent_child_index ASC )"#; - -/// Stores ephemeral transparent addresses used for ZIP 320. -/// -/// For each account, these addresses are allocated sequentially by address index under scope 2 -/// (`TransparentKeyScope::EPHEMERAL`) at the "change" level of the BIP 32 address hierarchy. -/// The ephemeral addresses stored in the table are exactly the "reserved" ephemeral addresses -/// (that is addresses that have been allocated for use in a ZIP 320 transaction proposal), plus -/// the addresses at the next [`GAP_LIMIT`] indices. -/// -/// Addresses are never removed. New ones should only be reserved via the -/// `WalletWrite::reserve_next_n_ephemeral_addresses` API. All of the addresses in the table -/// should be scanned for incoming funds. -/// -/// ### Columns -/// - `address` contains the string (Base58Check) encoding of a transparent P2PKH address. -/// - `used_in_tx` indicates that the address has been used by this wallet in a transaction (which -/// has not necessarily been mined yet). This should only be set once, when the txid is known. -/// - `seen_in_tx` is non-null iff an output to the address has been seed in a transaction observed -/// on the network and passed to `store_decrypted_tx`. The transaction may have been sent by this -// wallet or another one using the same seed, or by a TEX address recipient sending back the -/// funds. This is used to advance the "gap", as well as to heuristically reduce the chance of -/// address reuse collisions with another wallet using the same seed. -/// -/// It is an external invariant that within each account: -/// - the address indices are contiguous and start from 0; -/// - the last [`GAP_LIMIT`] addresses have `used_in_tx` and `seen_in_tx` both NULL. -/// -/// All but the last [`GAP_LIMIT`] addresses are defined to be "reserved" addresses. Since the next -/// index to reserve is determined by dead reckoning from the last stored address, we use dummy -/// entries having `NULL` for the value of the `address` column after the maximum valid index in -/// order to allow the last [`GAP_LIMIT`] addresses at the end of the index range to be used. -/// -/// Note that the fact that `used_in_tx` references a specific transaction is just a debugging aid. -/// The same is mostly true of `seen_in_tx`, but we also take into account whether the referenced -/// transaction is unmined in order to determine the last index that is safe to reserve. -pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" -CREATE TABLE ephemeral_addresses ( - account_id INTEGER NOT NULL, - address_index INTEGER NOT NULL, - -- nullability of this column is controlled by the index_range_and_address_nullity check - address TEXT, - used_in_tx INTEGER, - seen_in_tx INTEGER, - FOREIGN KEY (account_id) REFERENCES accounts(id), - FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), - FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), - PRIMARY KEY (account_id, address_index), - CONSTRAINT ephemeral_addr_uniq UNIQUE (address), - CONSTRAINT used_implies_seen CHECK ( - used_in_tx IS NULL OR seen_in_tx IS NOT NULL - ), - CONSTRAINT index_range_and_address_nullity CHECK ( - (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR - (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND seen_in_tx IS NULL) - ) -) WITHOUT ROWID"#; -// Hexadecimal integer literals were added in SQLite version 3.8.6 (2014-08-15). -// libsqlite3-sys requires at least version 3.14.0. -// "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. -const_assert_eq!(GAP_LIMIT, 20); /// Stores information about every block that the wallet has scanned. /// @@ -216,6 +185,7 @@ CREATE TABLE "sapling_received_notes" ( memo BLOB, commitment_tree_position INTEGER, recipient_key_scope INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT tx_output UNIQUE (tx, output_index) @@ -269,6 +239,7 @@ CREATE TABLE orchard_received_notes ( memo BLOB, commitment_tree_position INTEGER, recipient_key_scope INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT tx_output UNIQUE (tx, action_index) @@ -339,6 +310,7 @@ CREATE TABLE transparent_received_outputs ( script BLOB NOT NULL, value_zat INTEGER NOT NULL, max_observed_unspent_height INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) @@ -690,7 +662,8 @@ CREATE VIEW v_received_outputs AS sapling_received_notes.value, is_change, sapling_received_notes.memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + sapling_received_notes.address_id FROM sapling_received_notes LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -705,7 +678,8 @@ UNION orchard_received_notes.value, is_change, orchard_received_notes.memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + orchard_received_notes.address_id FROM orchard_received_notes LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -720,7 +694,8 @@ UNION u.value_zat AS value, 0 AS is_change, NULL AS memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + u.address_id FROM transparent_received_outputs u LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -1072,3 +1047,37 @@ GROUP BY subtree_start_height, subtree_end_height, contains_marked"; + +pub(super) const VIEW_ADDRESS_USES: &str = " +CREATE VIEW v_address_uses AS + SELECT orn.address_id, orn.account_id, orn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM orchard_received_notes orn + JOIN addresses a ON a.id = orn.address_id + JOIN transactions t ON t.id_tx = orn.tx +UNION + SELECT srn.address_id, srn.account_id, srn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM sapling_received_notes srn + JOIN addresses a ON a.id = srn.address_id + JOIN transactions t ON t.id_tx = srn.tx +UNION + SELECT tro.address_id, tro.account_id, tro.transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM transparent_received_outputs tro + JOIN addresses a ON a.id = tro.address_id + JOIN transactions t ON t.id_tx = tro.transaction_id"; + +pub(super) const VIEW_ADDRESS_FIRST_USE: &str = " + CREATE VIEW v_address_first_use AS + SELECT + address_id, + account_id, + key_scope, + diversifier_index_be, + transparent_child_index, + MIN(mined_height) AS first_use_height + FROM v_address_uses + GROUP BY + address_id, account_id, key_scope, + diversifier_index_be, transparent_child_index"; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 0d28aab632..7c7cbd8ac7 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -222,12 +222,11 @@ fn sqlite_client_error_to_wallet_migration_error(e: SqliteClientError) -> Wallet unreachable!("we don't call methods that require a known chain height") } #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit(_, _) => { + SqliteClientError::ReachedGapLimit(_) => { unreachable!("we don't do ephemeral address tracking") } - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(_, _) => { - unreachable!("we don't do ephemeral address tracking") + SqliteClientError::AddressReuse(_, _) => { + unreachable!("we don't create transactions in migrations") } SqliteClientError::NoteFilterInvalid(_) => { unreachable!("we don't do note selection in migrations") @@ -492,7 +491,6 @@ mod tests { db::TABLE_ACCOUNTS, db::TABLE_ADDRESSES, db::TABLE_BLOCKS, - db::TABLE_EPHEMERAL_ADDRESSES, db::TABLE_NULLIFIER_MAP, db::TABLE_ORCHARD_RECEIVED_NOTE_SPENDS, db::TABLE_ORCHARD_RECEIVED_NOTES, @@ -534,6 +532,8 @@ mod tests { db::INDEX_ACCOUNTS_UUID, db::INDEX_HD_ACCOUNT, db::INDEX_ADDRESSES_ACCOUNTS, + db::INDEX_ADDRESSES_INDICES, + db::INDEX_ADDRESSES_T_INDICES, db::INDEX_NF_MAP_LOCATOR_IDX, db::INDEX_ORCHARD_RECEIVED_NOTES_ACCOUNT, db::INDEX_ORCHARD_RECEIVED_NOTES_TX, @@ -562,6 +562,8 @@ mod tests { } let expected_views = vec![ + db::VIEW_ADDRESS_FIRST_USE.to_owned(), + db::VIEW_ADDRESS_USES.to_owned(), db::view_orchard_shard_scan_ranges(st.network()), db::view_orchard_shard_unscanned_ranges(), db::VIEW_ORCHARD_SHARDS_SCAN_STATE.to_owned(), @@ -1084,6 +1086,11 @@ mod tests { let (account_id, _usk) = db_data .create_account("", &Secret::new(seed.to_vec()), &birthday, None) .unwrap(); + + // We have to have the chain tip height in order to allocate new addresses, to record the + // exposed-at height. + db_data.update_chain_tip(birthday.height()).unwrap(); + assert_matches!( db_data.get_account(account_id), Ok(Some(account)) if matches!( diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1e076736ed..8aa056ced2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -19,6 +19,7 @@ mod sent_notes_to_internal; mod shardtree_support; mod spend_key_available; mod support_legacy_sqlite; +mod transparent_gap_limit_handling; mod tx_retrieval_queue; mod ufvk_support; mod utxos_table; @@ -84,8 +85,8 @@ pub(super) fn all_migrations( // support_legacy_sqlite // / \ // fix_broken_commitment_trees add_account_uuids - // | - // fix_bad_change_flagging + // | | + // fix_bad_change_flagging transparent_gap_limit_handling vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -149,6 +150,9 @@ pub(super) fn all_migrations( }), Box::new(fix_bad_change_flagging::Migration), Box::new(add_account_uuids::Migration), + Box::new(transparent_gap_limit_handling::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs index d3c62b7d3f..faad0d03b8 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs @@ -21,7 +21,7 @@ use { keys::{IncomingViewingKey, NonHardenedChildIndex}, TransparentAddress, }, - zip32::{AccountId, DiversifierIndex, Scope}, + zip32::{AccountId, Scope}, }; /// This migration adds an account identifier column to the UTXOs table. @@ -134,6 +134,8 @@ fn get_transparent_receivers( params: &P, account: AccountId, ) -> Result>, SqliteClientError> { + use crate::wallet::decode_diversifier_index_be; + let mut ret: HashMap> = HashMap::new(); // Get all UAs derived @@ -143,11 +145,7 @@ fn get_transparent_receivers( while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; - let di_vec: Vec = row.get(1)?; - let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion + let di = decode_diversifier_index_be(&row.get::<_, Vec>(1)?)?; let ua = Address::decode(params, &ua_str) .ok_or_else(|| { @@ -162,13 +160,11 @@ fn get_transparent_receivers( })?; if let Some(taddr) = ua.transparent() { - let index = NonHardenedChildIndex::from_index( - DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_owned(), - ) - })?, - ) + let index = NonHardenedChildIndex::from_index(u32::try_from(di).map_err(|_| { + SqliteClientError::CorruptedData( + "Unable to get diversifier for transparent address.".to_owned(), + ) + })?) .ok_or_else(|| { SqliteClientError::CorruptedData( "Unexpected hardened index for transparent address.".to_owned(), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 109f307fff..d5b8bdea38 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -1,19 +1,25 @@ //! The migration that records ephemeral addresses for each account. use std::collections::HashSet; +use rusqlite::named_params; use schemerz_rusqlite::RusqliteMigration; +use transparent::keys::NonHardenedChildIndex; use uuid::Uuid; +use zcash_keys::{ + encoding::AddressCodec, + keys::{AddressGenerationError, UnifiedFullViewingKey}, +}; use zcash_protocol::consensus; +use zip32::DiversifierIndex; -use crate::wallet::init::WalletMigrationError; - -#[cfg(feature = "transparent-inputs")] -use crate::{wallet::transparent::ephemeral, AccountRef}; +use crate::{error::SqliteClientError, wallet::init::WalletMigrationError, AccountRef}; use super::utxos_to_txos; pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0e1d4274_1f8e_44e2_909d_689a4bc2967b); +const EPHEMERAL_GAP_LIMIT: u32 = 5; + const DEPENDENCIES: &[Uuid] = &[utxos_to_txos::MIGRATION_ID]; #[allow(dead_code)] @@ -35,6 +41,60 @@ impl

schemerz::Migration for Migration

{ } } +fn init_accounts( + transaction: &rusqlite::Transaction, + params: &P, +) -> Result<(), SqliteClientError> { + let mut stmt = transaction.prepare("SELECT id, ufvk FROM accounts")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let account_id = AccountRef(row.get(0)?); + let ufvk_str: Option = row.get(1)?; + if let Some(ufvk_str) = ufvk_str { + #[cfg(feature = "transparent-inputs")] + if let Some(tfvk) = UnifiedFullViewingKey::decode(params, &ufvk_str) + .map_err(SqliteClientError::CorruptedData)? + .transparent() + { + let ephemeral_ivk = tfvk.derive_ephemeral_ivk().map_err(|_| { + SqliteClientError::CorruptedData( + "Unexpected failure to derive ephemeral transparent IVK".to_owned(), + ) + })?; + + let mut ea_insert = transaction.prepare( + "INSERT INTO ephemeral_addresses (account_id, address_index, address) + VALUES (:account_id, :address_index, :address)", + )?; + + // NB: we have reduced the initial space of generated ephemeral addresses + // from 20 addresses to 5, as ephemeral addresses should always be used in + // a transaction immediatly after being reserved, and as a consequence + // there is no significant benefit in having a larger gap limit. + for i in 0..EPHEMERAL_GAP_LIMIT { + let address = ephemeral_ivk + .derive_ephemeral_address( + NonHardenedChildIndex::from_index(i).expect("index is valid"), + ) + .map_err(|_| { + AddressGenerationError::InvalidTransparentChildIndex( + DiversifierIndex::from(i), + ) + })?; + + ea_insert.execute(named_params! { + ":account_id": account_id.0, + ":address_index": i, + ":address": address.encode(params) + })?; + } + } + } + } + + Ok(()) +} + impl RusqliteMigration for Migration

{ type Error = WalletMigrationError; @@ -62,17 +122,13 @@ impl RusqliteMigration for Migration

{ ) WITHOUT ROWID;" )?; - // Make sure that at least `GAP_LIMIT` ephemeral transparent addresses are + // Make sure that at least `EPHEMERAL_GAP_LIMIT` ephemeral transparent addresses are // stored in each account. #[cfg(feature = "transparent-inputs")] { - let mut stmt = transaction.prepare("SELECT id FROM accounts")?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let account_id = AccountRef(row.get(0)?); - ephemeral::init_account(transaction, &self.params, account_id)?; - } + init_accounts(transaction, &self.params)?; } + Ok(()) } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs b/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs index 0923b16307..ad5268c0f9 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs @@ -5,12 +5,11 @@ use std::collections::HashSet; use rusqlite::named_params; use schemerz_rusqlite::RusqliteMigration; use uuid::Uuid; -use zip32::Scope; use crate::{ wallet::{ init::{migrations::fix_broken_commitment_trees, WalletMigrationError}, - scope_code, + KeyScope, }, SAPLING_TABLES_PREFIX, }; @@ -52,7 +51,7 @@ impl RusqliteMigration for Migration { AND sn.from_account_id = {table_prefix}_received_notes.account_id AND {table_prefix}_received_notes.recipient_key_scope = :internal_scope" ), - named_params! {":internal_scope": scope_code(Scope::Internal)}, + named_params! {":internal_scope": KeyScope::INTERNAL.encode()}, ) }; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index 7822a076ca..064635214e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -30,7 +30,7 @@ use crate::{ chain_tip_height, commitment_tree::SqliteShardStore, init::{migrations::shardtree_support, WalletMigrationError}, - scope_code, + KeyScope, }, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; @@ -110,7 +110,7 @@ impl RusqliteMigration for Migration

{ transaction.execute_batch( &format!( "ALTER TABLE sapling_received_notes ADD COLUMN recipient_key_scope INTEGER NOT NULL DEFAULT {};", - scope_code(Scope::External) + KeyScope::EXTERNAL.encode() ) )?; @@ -205,7 +205,7 @@ impl RusqliteMigration for Migration

{ transaction.execute( "UPDATE sapling_received_notes SET recipient_key_scope = :scope WHERE id_note = :note_id", - named_params! {":scope": scope_code(Scope::Internal), ":note_id": note_id}, + named_params! {":scope": KeyScope::INTERNAL.encode(), ":note_id": note_id}, )?; } } else { @@ -263,7 +263,7 @@ impl RusqliteMigration for Migration

{ transaction.execute( "UPDATE sapling_received_notes SET recipient_key_scope = :scope WHERE id_note = :note_id", - named_params! {":scope": scope_code(Scope::Internal), ":note_id": note_id}, + named_params! {":scope": KeyScope::INTERNAL.encode(), ":note_id": note_id}, )?; } } @@ -324,8 +324,9 @@ mod tests { init_wallet_db_internal, migrations::{add_account_birthdays, shardtree_support, wallet_summaries}, }, - memo_repr, parse_scope, + memo_repr, sapling::ReceivedSaplingOutput, + KeyScope, }, AccountRef, TxRef, WalletDb, }; @@ -602,10 +603,10 @@ mod tests { while let Some(row) = rows.next().unwrap() { row_count += 1; let value: u64 = row.get(0).unwrap(); - let scope = parse_scope(row.get(1).unwrap()); + let scope = KeyScope::decode(row.get(1).unwrap()).unwrap(); match value { - EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), - INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), + EXTERNAL_VALUE => assert_eq!(scope, KeyScope::EXTERNAL), + INTERNAL_VALUE => assert_eq!(scope, KeyScope::INTERNAL), _ => { panic!( "(Value, Scope) pair {:?} is not expected to exist in the wallet.", @@ -780,10 +781,10 @@ mod tests { while let Some(row) = rows.next().unwrap() { row_count += 1; let value: u64 = row.get(0).unwrap(); - let scope = parse_scope(row.get(1).unwrap()); + let scope = KeyScope::decode(row.get(1).unwrap()).unwrap(); match value { - EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), - INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), + EXTERNAL_VALUE => assert_eq!(scope, KeyScope::EXTERNAL), + INTERNAL_VALUE => assert_eq!(scope, KeyScope::INTERNAL), _ => { panic!( "(Value, Scope) pair {:?} is not expected to exist in the wallet.", diff --git a/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs b/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs new file mode 100644 index 0000000000..ae3d545d75 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs @@ -0,0 +1,492 @@ +//! Add support for general transparent gap limit handling, and unify the `addresses` and +//! `ephemeral_addresses` tables. + +use std::collections::HashSet; +use uuid::Uuid; + +use rusqlite::{named_params, Transaction}; +use schemerz_rusqlite::RusqliteMigration; + +use ::transparent::keys::NonHardenedChildIndex; +use zcash_keys::{encoding::AddressCodec as _, keys::UnifiedIncomingViewingKey}; +use zcash_protocol::consensus::{self, BlockHeight}; +use zip32::DiversifierIndex; + +use crate::{ + wallet::{ + self, decode_diversifier_index_be, encode_diversifier_index_be, init::WalletMigrationError, + KeyScope, + }, + AccountRef, +}; + +#[cfg(feature = "transparent-inputs")] +use transparent::keys::IncomingViewingKey as _; + +use super::add_account_uuids; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xc41dfc0e_e870_4859_be47_d2f572f5ca73); + +const DEPENDENCIES: &[Uuid] = &[add_account_uuids::MIGRATION_ID]; + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemerz::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + DEPENDENCIES.iter().copied().collect() + } + + fn description(&self) -> &'static str { + "Add support for general transparent gap limit handling, unifying the `addresses` and `ephemeral_addresses` tables." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, conn: &Transaction) -> Result<(), WalletMigrationError> { + let decode_uivk = |uivk_str: String| { + UnifiedIncomingViewingKey::decode(&self.params, &uivk_str).map_err(|e| { + WalletMigrationError::CorruptedData(format!( + "Invalid UIVK encoding {}: {}", + uivk_str, e + )) + }) + }; + + let external_scope_code = KeyScope::EXTERNAL.encode(); + + conn.execute_batch(&format!( + r#" + ALTER TABLE addresses ADD COLUMN key_scope INTEGER NOT NULL DEFAULT {external_scope_code}; + ALTER TABLE addresses ADD COLUMN transparent_child_index INTEGER; + "# + ))?; + + #[cfg(feature = "transparent-inputs")] + { + // If the diversifier index is in the valid range of non-hardened child indices, set + // `transparent_child_index` so that we can use it for gap limit handling. + let mut di_query = conn.prepare( + r#" + SELECT account_id, accounts.uivk AS uivk, diversifier_index_be + FROM addresses + JOIN accounts ON accounts.id = account_id + GROUP BY account_id, uivk, diversifier_index_be + "#, + )?; + let mut rows = di_query.query([])?; + while let Some(row) = rows.next()? { + let account_id: i64 = row.get("account_id")?; + let uivk = decode_uivk(row.get("uivk")?)?; + let di_be: Vec = row.get("diversifier_index_be")?; + let diversifier_index = decode_diversifier_index_be(&di_be)?; + + let transparent_external = NonHardenedChildIndex::try_from(diversifier_index) + .ok() + .and_then(|idx| { + uivk.transparent() + .as_ref() + .and_then(|external_ivk| external_ivk.derive_address(idx).ok()) + .map(|t_addr| (idx, t_addr)) + }); + + // Add transparent address index metadata and the transparent address corresponding + // to the index to the addresses table. We unconditionally set the cached + // transparent receiver address in order to simplify gap limit handling; even if a + // unified address is generated without a transparent receiver, we still assume + // that a transparent-only wallet for which we have imported the seed may have + // generated an address at that index. + if let Some((idx, t_addr)) = transparent_external { + conn.execute( + r#" + UPDATE addresses + SET transparent_child_index = :transparent_child_index, + cached_transparent_receiver_address = :t_addr + WHERE account_id = :account_id + AND diversifier_index_be = :diversifier_index_be + AND key_scope = :external_scope_code + "#, + named_params! { + ":account_id": account_id, + ":diversifier_index_be": &di_be[..], + ":external_scope_code": external_scope_code, + ":transparent_child_index": idx.index(), + ":t_addr": t_addr.encode(&self.params), + }, + )?; + } + } + } + + // We now have to re-create the `addresses` table in order to fix the constraints. + // Note that we do not include `used_in_tx` or `seen_in_tx` columns as these are + // duplicative of information that can be discovered via joins with the various + // `*_received_{notes|outputs}` tables, which we will create a view to perform below. + conn.execute_batch(&format!( + r#" + CREATE TABLE addresses_new ( + id INTEGER NOT NULL PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES accounts(id), + diversifier_index_be BLOB NOT NULL, + key_scope INTEGER NOT NULL DEFAULT {external_scope_code}, + address TEXT NOT NULL, + transparent_child_index INTEGER, + cached_transparent_receiver_address TEXT, + exposed_at_height INTEGER, + CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be, key_scope), + CONSTRAINT transparent_index_consistency CHECK ( + (transparent_child_index IS NOT NULL) == (cached_transparent_receiver_address IS NOT NULL) + ) + ); + + INSERT INTO addresses_new ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) + SELECT + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + FROM addresses; + "# + ))?; + + // Now, we add the ephemeral addresses to the newly unified `addresses` table. + #[cfg(feature = "transparent-inputs")] + { + let mut ea_insert = conn.prepare( + r#" + INSERT INTO addresses_new ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) VALUES ( + :account_id, :diversifier_index_be, :key_scope, :address, + :transparent_child_index, :cached_transparent_receiver_address + ) + "#, + )?; + + let mut ea_query = conn.prepare( + r#" + SELECT account_id, address_index, address + FROM ephemeral_addresses + "#, + )?; + let mut rows = ea_query.query([])?; + while let Some(row) = rows.next()? { + let account_id: i64 = row.get("account_id")?; + let transparent_child_index = row.get::<_, i64>("address_index")?; + let diversifier_index = DiversifierIndex::from( + u32::try_from(transparent_child_index).map_err(|_| { + WalletMigrationError::CorruptedData( + "ephermeral address indices must be in the range of `u32`".to_owned(), + ) + })?, + ); + let address: String = row.get("address")?; + + // We set both the `address` column and the `cached_transparent_receiver_address` + // column to the same value here; there is no Unified address that corresponds to + // this transparent address. + ea_insert.execute(named_params! { + ":account_id": account_id, + ":diversifier_index_be": encode_diversifier_index_be(diversifier_index), + ":key_scope": KeyScope::Ephemeral.encode(), + ":address": address, + ":transparent_child_index": transparent_child_index, + ":cached_transparent_receiver_address": address + })?; + } + } + + conn.execute_batch( + r#" + PRAGMA legacy_alter_table = ON; + + DROP TABLE addresses; + ALTER TABLE addresses_new RENAME TO addresses; + CREATE INDEX idx_addresses_accounts ON addresses ( + account_id ASC + ); + CREATE INDEX idx_addresses_indices ON addresses ( + diversifier_index_be ASC + ); + CREATE INDEX idx_addresses_t_indices ON addresses ( + transparent_child_index ASC + ); + + DROP TABLE ephemeral_addresses; + + PRAGMA legacy_alter_table = OFF; + "#, + )?; + + // Add foreign key references from the *_received_{notes|outputs} tables to the addresses + // table to make it possible to identify which address was involved. These foreign key + // columns must be nullable as for shielded account-internal. Ideally the foreign key + // relationship between `transparent_received_outputs` and `addresses` would not be + // nullable, but we allow it to be so here in order to avoid having to re-create that + // table. + // + // While it would be possible to only add the address reference to + // `transparent_received_outputs`, that would mean that a note received at a shielded + // component of a diversified Unified Address would not update the position of the + // transparent "address gap". Since we will include shielded address indices in the gap + // computation, transparent-only wallets may not be able to discover all transparent funds, + // but users of shielded wallets will be guaranteed to be able to recover all of their + // funds. + conn.execute_batch( + r#" + ALTER TABLE orchard_received_notes + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + ALTER TABLE sapling_received_notes + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + ALTER TABLE transparent_received_outputs + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + "#, + )?; + + // Ensure that an address exists for each received Orchard note, and populate the + // `address_id` column. + #[cfg(feature = "orchard")] + { + let mut stmt_rn_diversifiers = conn.prepare( + r#" + SELECT orn.id, orn.account_id, accounts.uivk, + orn.recipient_key_scope, orn.diversifier, t.mined_height + FROM orchard_received_notes orn + JOIN accounts ON accounts.id = account_id + JOIN transactions t on t.id_tx = orn.tx + "#, + )?; + + let mut rows = stmt_rn_diversifiers.query([])?; + while let Some(row) = rows.next()? { + let scope = KeyScope::decode(row.get("recipient_key_scope")?)?; + // for Orchard and Sapling, we only store addresses for externally-scoped keys. + if scope == KeyScope::EXTERNAL { + let row_id: i64 = row.get("id")?; + let account_id = AccountRef(row.get("account_id")?); + let mined_height = row + .get::<_, Option>("mined_height")? + .map(BlockHeight::from); + + let uivk = decode_uivk(row.get("uivk")?)?; + let diversifier = + orchard::keys::Diversifier::from_bytes(row.get("diversifier")?); + + // TODO: It's annoying that `IncomingViewingKey` doesn't expose the ability to + // decrypt the diversifier to find the index directly, and doesn't provide an + // accessor for `dk`. We already know we have the right IVK. + let ivk = uivk + .orchard() + .as_ref() + .expect("previously received an Orchard output"); + let di = ivk + .diversifier_index(&ivk.address(diversifier)) + .expect("roundtrip"); + let ua = uivk.address(di, None)?; + let address_id = wallet::upsert_address( + conn, + &self.params, + account_id, + di, + &ua, + mined_height, + )?; + + conn.execute( + "UPDATE orchard_received_notes + SET address_id = :address_id + WHERE id = :row_id", + named_params! { + ":address_id": address_id.0, + ":row_id": row_id + }, + )?; + } + } + } + + // Ensure that an address exists for each received Sapling note, and populate the + // `address_id` column. + { + let mut stmt_rn_diversifiers = conn.prepare( + r#" + SELECT srn.id, srn.account_id, accounts.uivk, + srn.recipient_key_scope, srn.diversifier, t.mined_height + FROM sapling_received_notes srn + JOIN accounts ON accounts.id = account_id + JOIN transactions t ON t.id_tx = srn.tx + "#, + )?; + + let mut rows = stmt_rn_diversifiers.query([])?; + while let Some(row) = rows.next()? { + let scope = KeyScope::decode(row.get("recipient_key_scope")?)?; + // for Orchard and Sapling, we only store addresses for externally-scoped keys. + if scope == KeyScope::EXTERNAL { + let row_id: i64 = row.get("id")?; + let account_id = AccountRef(row.get("account_id")?); + let mined_height = row + .get::<_, Option>("mined_height")? + .map(BlockHeight::from); + + let uivk = decode_uivk(row.get("uivk")?)?; + let diversifier = sapling::Diversifier(row.get("diversifier")?); + + // TODO: It's annoying that `IncomingViewingKey` doesn't expose the ability to + // decrypt the diversifier to find the index directly, and doesn't provide an + // accessor for `dk`. We already know we have the right IVK. + let ivk = uivk + .sapling() + .as_ref() + .expect("previously received a Sapling output"); + let di = ivk + .decrypt_diversifier( + &ivk.address(diversifier) + .expect("previously generated an address"), + ) + .expect("roundtrip"); + let ua = uivk.address(di, None)?; + let address_id = wallet::upsert_address( + conn, + &self.params, + account_id, + di, + &ua, + mined_height, + )?; + + conn.execute( + "UPDATE sapling_received_notes + SET address_id = :address_id + WHERE id = :row_id", + named_params! { + ":address_id": address_id.0, + ":row_id": row_id + }, + )?; + } + } + } + + // At this point, every address on which we've received a transparent output should have a + // corresponding row in the `addresses` table with a valid + // `cached_transparent_receiver_address` entry, because we will only have queried the light + // wallet server for outputs from exactly these addresses. So for transparent outputs, we + // join to the addresses table using the address itself in order to obtain the address index. + #[cfg(feature = "transparent-inputs")] + { + conn.execute( + r#" + UPDATE transparent_received_outputs + SET address_id = addresses.id + FROM addresses + WHERE addresses.cached_transparent_receiver_address = transparent_received_outputs.address + "#, + [] + )?; + } + + // Construct a view that identifies the minimum block height at which each address was + // first used + conn.execute_batch( + r#" + CREATE VIEW v_address_uses AS + SELECT orn.address_id, orn.account_id, orn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM orchard_received_notes orn + JOIN addresses a ON a.id = orn.address_id + JOIN transactions t ON t.id_tx = orn.tx + UNION + SELECT srn.address_id, srn.account_id, srn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM sapling_received_notes srn + JOIN addresses a ON a.id = srn.address_id + JOIN transactions t ON t.id_tx = srn.tx + UNION + SELECT tro.address_id, tro.account_id, tro.transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM transparent_received_outputs tro + JOIN addresses a ON a.id = tro.address_id + JOIN transactions t ON t.id_tx = tro.transaction_id; + + CREATE VIEW v_address_first_use AS + SELECT + address_id, + account_id, + key_scope, + diversifier_index_be, + transparent_child_index, + MIN(mined_height) AS first_use_height + FROM v_address_uses + GROUP BY + address_id, account_id, key_scope, + diversifier_index_be, transparent_child_index; + + DROP VIEW v_received_outputs; + CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id, + sapling_received_notes.address_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id, + orchard_received_notes.address_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) + UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id, + u.address_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index); + "#, + )?; + + Ok(()) + } + + fn down(&self, _: &Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs index e2c6015f36..e39ee26d78 100644 --- a/zcash_client_sqlite/src/wallet/orchard.rs +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -5,10 +5,10 @@ use orchard::{ keys::Diversifier, note::{Note, Nullifier, RandomSeed, Rho}, }; -use rusqlite::{named_params, types::Value, Connection, Row, Transaction}; +use rusqlite::{named_params, types::Value, Connection, Row}; use zcash_client_backend::{ - data_api::NullifierQuery, + data_api::{Account as _, NullifierQuery}, wallet::{ReceivedNote, WalletOrchardOutput}, DecryptedOutput, ShieldedProtocol, TransferType, }; @@ -21,9 +21,9 @@ use zcash_protocol::{ }; use zip32::Scope; -use crate::{error::SqliteClientError, AccountUuid, ReceivedNoteId, TxRef}; +use crate::{error::SqliteClientError, AccountRef, AccountUuid, AddressRef, ReceivedNoteId, TxRef}; -use super::{get_account_ref, memo_repr, parse_scope, scope_code}; +use super::{get_account, get_account_ref, memo_repr, upsert_address, KeyScope}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedOrchardOutput { @@ -159,9 +159,14 @@ fn to_spendable_note( let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; + let spending_key_scope = zip32::Scope::try_from(KeyScope::decode(scope_code)?) + .map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Invalid key scope code {}", + scope_code + )) + })?; + let recipient = ufvk .orchard() .map(|fvk| fvk.to_ivk(spending_key_scope).address(diversifier)) @@ -225,34 +230,80 @@ pub(crate) fn select_spendable_orchard_notes( ) } +pub(crate) fn ensure_address< + T: ReceivedOrchardOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, + output: &T, + exposure_height: Option, +) -> Result, SqliteClientError> { + if output.recipient_key_scope() != Some(Scope::Internal) { + let account = get_account(conn, params, output.account_id())? + .ok_or(SqliteClientError::AccountUnknown)?; + + let uivk = account.uivk(); + let ivk = uivk + .orchard() + .as_ref() + .expect("uivk decrypted this output."); + let to = output.note().recipient(); + let diversifier_index = ivk + .diversifier_index(&to) + .expect("address corresponds to account"); + + let ua = account.uivk().address(diversifier_index, None)?; + upsert_address( + conn, + params, + account.internal_id(), + diversifier_index, + &ua, + exposure_height, + ) + .map(Some) + } else { + Ok(None) + } +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note>( - conn: &Transaction, +/// +/// Returns the internal account identifier of the account that received the output. +pub(crate) fn put_received_note< + T: ReceivedOrchardOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, output: &T, tx_ref: TxRef, + target_or_mined_height: Option, spent_in: Option, -) -> Result<(), SqliteClientError> { +) -> Result { let account_id = get_account_ref(conn, output.account_id())?; + let address_id = ensure_address(conn, params, output, target_or_mined_height)?; let mut stmt_upsert_received_note = conn.prepare_cached( - "INSERT INTO orchard_received_notes - ( - tx, action_index, account_id, + "INSERT INTO orchard_received_notes ( + tx, action_index, account_id, address_id, diversifier, value, rho, rseed, memo, nf, is_change, commitment_tree_position, recipient_key_scope ) VALUES ( - :tx, :action_index, :account_id, + :tx, :action_index, :account_id, :address_id, :diversifier, :value, :rho, :rseed, :memo, :nf, :is_change, :commitment_tree_position, :recipient_key_scope ) ON CONFLICT (tx, action_index) DO UPDATE SET account_id = :account_id, + address_id = :address_id, diversifier = :diversifier, value = :value, rho = :rho, @@ -273,6 +324,7 @@ pub(crate) fn put_received_note( let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; + let spending_key_scope = zip32::Scope::try_from(KeyScope::decode(scope_code)?) + .map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Invalid key scope code {}", + scope_code + )) + })?; let recipient = match spending_key_scope { Scope::Internal => ufvk @@ -328,27 +332,76 @@ pub(crate) fn mark_sapling_note_spent( } } +pub(crate) fn ensure_address< + T: ReceivedSaplingOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, + output: &T, + exposure_height: Option, +) -> Result, SqliteClientError> { + if output.recipient_key_scope() != Some(Scope::Internal) { + let account = get_account(conn, params, output.account_id())? + .ok_or(SqliteClientError::AccountUnknown)?; + + let uivk = account.uivk(); + let ivk = uivk + .sapling() + .as_ref() + .expect("uivk decrypted this output."); + let to = output.note().recipient(); + let diversifier_index = ivk + .decrypt_diversifier(&to) + .expect("address corresponds to account"); + + let ua = account.uivk().address(diversifier_index, None)?; + upsert_address( + conn, + params, + account.internal_id(), + diversifier_index, + &ua, + exposure_height, + ) + .map(Some) + } else { + Ok(None) + } +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note>( - conn: &Transaction, +/// +/// Returns the internal account identifier of the account that received the output. +pub(crate) fn put_received_note< + T: ReceivedSaplingOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, output: &T, tx_ref: TxRef, + target_or_mined_height: Option, spent_in: Option, -) -> Result<(), SqliteClientError> { +) -> Result { let account_id = get_account_ref(conn, output.account_id())?; + let address_id = ensure_address(conn, params, output, target_or_mined_height)?; let mut stmt_upsert_received_note = conn.prepare_cached( - "INSERT INTO sapling_received_notes - (tx, output_index, account_id, diversifier, value, rcm, memo, nf, - is_change, commitment_tree_position, - recipient_key_scope) + "INSERT INTO sapling_received_notes ( + tx, output_index, account_id, address_id, + diversifier, value, rcm, memo, nf, + is_change, commitment_tree_position, + recipient_key_scope + ) VALUES ( :tx, :output_index, :account_id, + :address_id, :diversifier, :value, :rcm, @@ -360,6 +413,7 @@ pub(crate) fn put_received_note( fn address_index_from_diversifier_index_be( diversifier_index_be: &[u8], ) -> Result { - let mut di: [u8; 11] = diversifier_index_be.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion + let di = decode_diversifier_index_be(diversifier_index_be)?; - NonHardenedChildIndex::from_index(DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_string(), - ) - })?) - .ok_or_else(|| { + NonHardenedChildIndex::try_from(di).map_err(|_| { SqliteClientError::CorruptedData( "Unexpected hardened index for transparent address.".to_string(), ) @@ -83,45 +87,48 @@ pub(crate) fn get_transparent_receivers( conn: &rusqlite::Connection, params: &P, account_uuid: AccountUuid, + scopes: &[KeyScope], ) -> Result>, SqliteClientError> { let mut ret: HashMap> = HashMap::new(); - // Get all UAs derived - let mut ua_query = conn.prepare( - "SELECT address, diversifier_index_be - FROM addresses + // Get all addresses with the provided scopes. + let mut addr_query = conn.prepare( + "SELECT address, diversifier_index_be, key_scope + FROM addresses JOIN accounts ON accounts.id = addresses.account_id - WHERE accounts.uuid = :account_uuid", + WHERE accounts.uuid = :account_uuid + AND key_scope IN rarray(:scopes_ptr)", )?; - let mut rows = ua_query.query(named_params![":account_uuid": account_uuid.0])?; + + let scope_values: Vec = scopes.iter().map(|s| Value::Integer(s.encode())).collect(); + let scopes_ptr = Rc::new(scope_values); + let mut rows = addr_query.query(named_params![ + ":account_uuid": account_uuid.0, + ":scopes_ptr": &scopes_ptr + ])?; while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; let di_vec: Vec = row.get(1)?; + let scope = KeyScope::decode(row.get(2)?)?; - let ua = Address::decode(params, &ua_str) + let taddr = Address::decode(params, &ua_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) - }) - .and_then(|addr| match addr { - Address::Unified(ua) => Ok(ua), - _ => Err(SqliteClientError::CorruptedData(format!( - "Addresses table contains {} which is not a unified address", - ua_str, - ))), - })?; - - if let Some(taddr) = ua.transparent() { + })? + .to_transparent_address(); + + if let Some(taddr) = taddr { let address_index = address_index_from_diversifier_index_be(&di_vec)?; - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); - ret.insert(*taddr, Some(metadata)); + let metadata = TransparentAddressMetadata::new(scope.into(), address_index); + ret.insert(taddr, Some(metadata)); } } if let Some((taddr, address_index)) = get_legacy_transparent_address(params, conn, account_uuid)? { - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + let metadata = TransparentAddressMetadata::new(KeyScope::EXTERNAL.into(), address_index); ret.insert(taddr, Some(metadata)); } @@ -183,6 +190,335 @@ pub(crate) fn get_legacy_transparent_address( Ok(None) } +/// Returns the transparent address index at the start of the first gap of at least `gap_limit` +/// indices in the given account, considering only addresses derived for the specified key scope. +/// +/// Returns `Ok(None)` if the gap would start at an index greater than the maximum valid +/// non-hardened transparent child index. +pub(crate) fn find_gap_start( + conn: &rusqlite::Connection, + account_id: AccountRef, + key_scope: KeyScope, + gap_limit: u32, +) -> Result, SqliteClientError> { + match conn + .query_row( + r#" + WITH offsets AS ( + SELECT + a.transparent_child_index, + LEAD(a.transparent_child_index) + OVER (ORDER BY a.transparent_child_index) + AS next_child_index + FROM v_address_first_use a + WHERE a.account_id = :account_id + AND a.key_scope = :key_scope + AND a.transparent_child_index IS NOT NULL + AND a.first_use_height IS NOT NULL + ) + SELECT + transparent_child_index + 1, + next_child_index - transparent_child_index - 1 AS gap_len + FROM offsets + WHERE gap_len >= :gap_limit OR next_child_index IS NULL + ORDER BY transparent_child_index + LIMIT 1 + "#, + named_params![ + ":account_id": account_id.0, + ":key_scope": key_scope.encode(), + ":gap_limit": gap_limit + ], + |row| row.get::<_, u32>(0), + ) + .optional()? + { + Some(i) => Ok(NonHardenedChildIndex::from_index(i)), + None => Ok(Some(NonHardenedChildIndex::ZERO)), + } +} + +pub(crate) fn decode_transparent_child_index( + value: i64, +) -> Result { + u32::try_from(value) + .ok() + .and_then(NonHardenedChildIndex::from_index) + .ok_or_else(|| { + SqliteClientError::CorruptedData(format!("Illegal transparent child index {value}")) + }) +} + +/// Returns a vector with the next `n` previously unreserved transparent addresses for +/// the given account. These addresses must have been previously generated using +/// `generate_gap_addresses`. +/// +/// # Errors +/// +/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. +/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses +/// within the gap limit after the last address in this account that is known to have an +/// output in a mined transaction. +/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, +/// if the limit on transparent address indices has been reached. +pub(crate) fn reserve_next_n_addresses( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountRef, + key_scope: KeyScope, + gap_limit: u32, + n: usize, +) -> Result, SqliteClientError> { + if n == 0 { + return Ok(vec![]); + } + + let gap_start = find_gap_start(conn, account_id, key_scope, gap_limit)?.ok_or( + SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted), + )?; + + let mut stmt_addrs_to_reserve = conn.prepare( + "SELECT id, transparent_child_index, cached_transparent_receiver_address + FROM addresses + WHERE account_id = :account_id + AND key_scope = :key_scope + AND transparent_child_index >= :gap_start + AND transparent_child_index < :gap_end + AND exposed_at_height IS NULL + ORDER BY transparent_child_index + LIMIT :n", + )?; + + let addresses_to_reserve = stmt_addrs_to_reserve + .query_and_then( + named_params! { + ":account_id": account_id.0, + ":key_scope": key_scope.encode(), + ":gap_start": gap_start.index(), + ":gap_end": gap_start.saturating_add(gap_limit).index(), + ":n": n + }, + |row| { + let address_id = row.get("id").map(AddressRef)?; + let transparent_child_index = row + .get::<_, Option>("transparent_child_index")? + .map(decode_transparent_child_index) + .transpose()?; + let address = row + .get::<_, Option>("cached_transparent_receiver_address")? + .map(|addr_str| TransparentAddress::decode(params, &addr_str)) + .transpose()?; + + Ok::<_, SqliteClientError>(transparent_child_index.zip(address).map(|(i, a)| { + ( + address_id, + a, + TransparentAddressMetadata::new(key_scope.into(), i), + ) + })) + }, + )? + .filter_map(|r| r.transpose()) + .collect::, _>>()?; + + if addresses_to_reserve.len() < n { + return Err(SqliteClientError::ReachedGapLimit( + gap_start.index() + gap_limit, + )); + } + + let current_chain_tip = chain_tip_height(conn)?.ok_or(SqliteClientError::ChainHeightUnknown)?; + + let reserve_id_values: Vec = addresses_to_reserve + .iter() + .map(|(id, _, _)| Value::Integer(id.0)) + .collect(); + let reserved_ptr = Rc::new(reserve_id_values); + conn.execute( + "UPDATE addresses + SET exposed_at_height = :chain_tip_height + WHERE id IN rarray(:reserved_ptr)", + named_params! { + ":chain_tip_height": u32::from(current_chain_tip), + ":reserved_ptr": &reserved_ptr + }, + )?; + + Ok(addresses_to_reserve) +} + +/// Extend the range of preallocated addresses in an account to ensure that a full `gap_limit` of +/// transparent addresses is available from the first gap in existing indices of addresses at which +/// a received transaction has been observed on the chain, for each key scope. +/// +/// The provided [`UnifiedAddressRequest`] is used to pregenerate unified addresses that correspond +/// to the transparent address index in question; such unified addresses need not internally +/// contain a transparent receiver, and may be overwritten when these addresses are exposed via the +/// [`WalletWrite::get_next_available_address`] or [`WalletWrite::get_address_for_index`] methods. +/// If no request is provided, each address so generated will contain a receiver for each possible +/// pool: i.e., a recevier for each data item in the account's UFVK or UIVK where the transparent +/// child index is valid. +/// +/// [`WalletWrite::get_next_available_address`]: zcash_client_backend::data_api::WalletWrite::get_next_available_address +/// [`WalletWrite::get_address_for_index`]: zcash_client_backend::data_api::WalletWrite::get_address_for_index +pub(crate) fn generate_gap_addresses( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountRef, + key_scope: KeyScope, + gap_limits: &GapLimits, + request: Option, +) -> Result<(), SqliteClientError> { + let account = get_account_internal(conn, params, account_id)? + .ok_or_else(|| SqliteClientError::AccountUnknown)?; + + let request = request.unwrap_or_else(|| { + use ReceiverRequirement::*; + UnifiedAddressRequest::unsafe_new(Allow, Allow, Require) + }); + + let gen_addrs = |key_scope: KeyScope, index: NonHardenedChildIndex| { + Ok::<_, SqliteClientError>(match key_scope { + KeyScope::Zip32(zip32::Scope::External) => { + let ua = account.uivk().address(index.into(), Some(request)); + let transparent_address = account + .uivk() + .transparent() + .as_ref() + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_address(index)?; + ( + ua.map_or_else( + |e| { + if matches!(e, AddressGenerationError::ShieldedReceiverRequired) { + // fall back to the transparent-only address + Ok(Address::from(transparent_address).to_zcash_address(params)) + } else { + // other address generation errors are allowed to propagate + Err(e) + } + }, + |addr| Ok(Address::from(addr).to_zcash_address(params)), + )?, + transparent_address, + ) + } + KeyScope::Zip32(zip32::Scope::Internal) => { + let internal_address = account + .ufvk() + .and_then(|k| k.transparent()) + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_internal_ivk()? + .derive_address(index)?; + ( + Address::from(internal_address).to_zcash_address(params), + internal_address, + ) + } + KeyScope::Ephemeral => { + let ephemeral_address = account + .ufvk() + .and_then(|k| k.transparent()) + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_ephemeral_ivk()? + .derive_ephemeral_address(index)?; + ( + Address::from(ephemeral_address).to_zcash_address(params), + ephemeral_address, + ) + } + }) + }; + + let gap_limit = match key_scope { + KeyScope::Zip32(zip32::Scope::External) => gap_limits.external(), + KeyScope::Zip32(zip32::Scope::Internal) => gap_limits.transparent_internal(), + KeyScope::Ephemeral => gap_limits.ephemeral(), + }; + + if let Some(gap_start) = find_gap_start(conn, account_id, key_scope, gap_limit)? { + let range_to_store = gap_start.index()..gap_start.saturating_add(gap_limit).index(); + if range_to_store.is_empty() { + return Ok(()); + } + // exposed_at_height is initially NULL + let mut stmt_insert_address = conn.prepare_cached( + "INSERT INTO addresses ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) + VALUES ( + :account_id, :diversifier_index_be, :key_scope, :address, + :transparent_child_index, :transparent_address + ) + ON CONFLICT (account_id, diversifier_index_be, key_scope) DO NOTHING", + )?; + + for raw_index in range_to_store { + let transparent_child_index = NonHardenedChildIndex::from_index(raw_index) + .expect("restricted to valid range above"); + let (zcash_address, transparent_address) = + gen_addrs(key_scope, transparent_child_index)?; + + stmt_insert_address.execute(named_params![ + ":account_id": account_id.0, + ":diversifier_index_be": encode_diversifier_index_be(transparent_child_index.into()), + ":key_scope": key_scope.encode(), + ":address": zcash_address.encode(), + ":transparent_child_index": raw_index, + ":transparent_address": transparent_address.encode(params) + ])?; + } + } + + Ok(()) +} + +/// If `address` is one of our addresses, check whether it was used as the recipient address for +/// any of our received outputs. +/// +/// If the address was already used in an output we received, this method will return +/// [`SqliteClientError::AddressReuse`]. +pub(crate) fn check_address_reuse( + conn: &rusqlite::Transaction, + params: &P, + address: &Address, +) -> Result<(), SqliteClientError> { + // TODO: ideally we would do something better than string matching here - the best would be to + // have the diversifier index for the address passed to us instead of the address itself, but + // not all call sites currently have a good way to obtain the diversifier index. We could + // trial-decrypt with each of the wallet's IVKs if we wanted to do it here, but a better + // approach is to restructure the call sites so that we don't discard diversifier index + // information in the process of passing it through to here. + let addr_str = address.encode(params); + let taddr_str = address.to_transparent_address().map(|a| a.encode(params)); + + let mut stmt = conn.prepare_cached( + "SELECT t.txid + FROM transactions t + JOIN v_received_outputs vro ON vro.transaction_id = t.id_tx + JOIN addresses a ON a.id = vro.address_id + WHERE a.address = :address + OR a.cached_transparent_receiver_address = :transparent_address", + )?; + + let txids = stmt + .query_and_then( + named_params![ + ":address": addr_str, + ":transparent_address": taddr_str, + ], + |row| Ok(TxId::from_bytes(row.get::<_, [u8; 32]>(0)?)), + )? + .collect::, SqliteClientError>>()?; + + if let Some(txids) = NonEmpty::from_vec(txids) { + return Err(SqliteClientError::AddressReuse(addr_str, txids)); + } + + Ok(()) +} + fn to_unspent_transparent_output(row: &Row) -> Result { let txid: Vec = row.get("txid")?; let mut txid_bytes = [0u8; 32]; @@ -191,7 +527,7 @@ fn to_unspent_transparent_output(row: &Row) -> Result = row.get("received_height")?; @@ -353,7 +689,7 @@ pub(crate) fn get_transparent_balances( params: &P, account_uuid: AccountUuid, summary_height: BlockHeight, -) -> Result, SqliteClientError> { +) -> Result, SqliteClientError> { let chain_tip_height = chain_tip_height(conn)?.ok_or(SqliteClientError::ChainHeightUnknown)?; let mut stmt_address_balances = conn.prepare( @@ -395,7 +731,7 @@ pub(crate) fn get_transparent_balances( while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; let taddr = TransparentAddress::decode(params, &taddr_str)?; - let value = NonNegativeAmount::from_nonnegative_i64(row.get(1)?)?; + let value = Zatoshis::from_nonnegative_i64(row.get(1)?)?; res.insert(taddr, value); } @@ -441,7 +777,7 @@ pub(crate) fn add_transparent_account_balances( while let Some(row) = rows.next()? { let account = AccountUuid(row.get(0)?); let raw_value = row.get(1)?; - let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { + let value = Zatoshis::from_nonnegative_i64(raw_value).map_err(|_| { SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) })?; @@ -484,7 +820,7 @@ pub(crate) fn add_transparent_account_balances( while let Some(row) = rows.next()? { let account = AccountUuid(row.get(0)?); let raw_value = row.get(1)?; - let value = NonNegativeAmount::from_nonnegative_i64(raw_value).map_err(|_| { + let value = Zatoshis::from_nonnegative_i64(raw_value).map_err(|_| { SqliteClientError::CorruptedData(format!("Negative UTXO value {:?}", raw_value)) })?; @@ -559,28 +895,19 @@ pub(crate) fn mark_transparent_utxo_spent( /// Adds the given received UTXO to the datastore. pub(crate) fn put_received_transparent_utxo( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction, params: &P, output: &WalletTransparentOutput, -) -> Result { - let address = output.recipient_address(); - if let Some(receiving_account) = - find_account_uuid_for_transparent_address(conn, params, address)? - { - put_transparent_output( - conn, - params, - output.outpoint(), - output.txout(), - output.mined_height(), - address, - receiving_account, - true, - ) - } else { - // The UTXO was not for any of our transparent addresses. - Err(SqliteClientError::AddressNotRecognized(*address)) - } +) -> Result<(AccountRef, KeyScope, UtxoId), SqliteClientError> { + put_transparent_output( + conn, + params, + output.outpoint(), + output.txout(), + output.mined_height(), + output.recipient_address(), + true, + ) } /// Returns the vector of [`TransactionDataRequest`]s that represents the information needed by the @@ -634,21 +961,29 @@ pub(crate) fn get_transparent_address_metadata( address: &TransparentAddress, ) -> Result, SqliteClientError> { let address_str = address.encode(params); - - if let Some(di_vec) = conn + let addr_meta = conn .query_row( - "SELECT diversifier_index_be FROM addresses + "SELECT diversifier_index_be, key_scope + FROM addresses JOIN accounts ON addresses.account_id = accounts.id - WHERE accounts.uuid = :account_uuid + WHERE accounts.uuid = :account_uuid AND cached_transparent_receiver_address = :address", named_params![":account_uuid": account_uuid.0, ":address": &address_str], - |row| row.get::<_, Vec>(0), + |row| { + let di_be: Vec = row.get(0)?; + let scope_code = row.get(1)?; + Ok(KeyScope::decode(scope_code).and_then(|key_scope| { + address_index_from_diversifier_index_be(&di_be).map(|address_index| { + TransparentAddressMetadata::new(key_scope.into(), address_index) + }) + })) + }, ) .optional()? - { - let address_index = address_index_from_diversifier_index_be(&di_vec)?; - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); - return Ok(Some(metadata)); + .transpose()?; + + if addr_meta.is_some() { + return Ok(addr_meta); } if let Some((legacy_taddr, address_index)) = @@ -660,13 +995,6 @@ pub(crate) fn get_transparent_address_metadata( } } - // Search known ephemeral addresses. - if let Some(address_index) = - ephemeral::find_index_for_ephemeral_address_str(conn, account_uuid, &address_str)? - { - return Ok(Some(ephemeral::metadata(address_index))); - } - Ok(None) } @@ -684,27 +1012,27 @@ pub(crate) fn find_account_uuid_for_transparent_address Result, SqliteClientError> { +) -> Result, SqliteClientError> { let address_str = address.encode(params); - if let Some(account_id) = conn + if let Some((account_id, key_scope_code)) = conn .query_row( - "SELECT accounts.uuid - FROM addresses + "SELECT accounts.uuid, addresses.key_scope + FROM addresses JOIN accounts ON accounts.id = addresses.account_id WHERE cached_transparent_receiver_address = :address", named_params![":address": &address_str], - |row| Ok(AccountUuid(row.get(0)?)), + |row| Ok((AccountUuid(row.get(0)?), row.get(1)?)), ) .optional()? { - return Ok(Some(account_id)); + return Ok(Some((account_id, KeyScope::decode(key_scope_code)?))); } // Search known ephemeral addresses. if let Some(account_id) = ephemeral::find_account_for_ephemeral_address_str(conn, &address_str)? { - return Ok(Some(account_id)); + return Ok(Some((account_id, KeyScope::Ephemeral))); } let account_ids = get_account_ids(conn)?; @@ -718,7 +1046,7 @@ pub(crate) fn find_account_uuid_for_transparent_address( txout: &TxOut, output_height: Option, address: &TransparentAddress, - receiving_account_uuid: AccountUuid, known_unspent: bool, -) -> Result { +) -> Result<(AccountRef, KeyScope, UtxoId), SqliteClientError> { + let addr_str = address.encode(params); + + // Unlike the shielded pools, we only can receive transparent outputs on addresses for which we + // have an `addresses` table entry, so we can just query for that here. + let (address_id, account_id, key_scope_code) = conn.query_row( + "SELECT id, account_id, key_scope + FROM addresses + WHERE cached_transparent_receiver_address = :transparent_address", + named_params! {":transparent_address": addr_str}, + |row| { + Ok(( + row.get("id").map(AddressRef)?, + row.get("account_id").map(AccountRef)?, + row.get("key_scope")?, + )) + }, + )?; + + let key_scope = KeyScope::decode(key_scope_code)?; + let output_height = output_height.map(u32::from); - let receiving_account_id = super::get_account_ref(conn, receiving_account_uuid)?; // Check whether we have an entry in the blocks table for the output height; // if not, the transaction will be updated with its mined height when the @@ -810,16 +1156,17 @@ pub(crate) fn put_transparent_output( let mut stmt_upsert_transparent_output = conn.prepare_cached( "INSERT INTO transparent_received_outputs ( transaction_id, output_index, - account_id, address, script, + account_id, address_id, address, script, value_zat, max_observed_unspent_height ) VALUES ( :transaction_id, :output_index, - :account_id, :address, :script, + :account_id, :address_id, :address, :script, :value_zat, :max_observed_unspent_height ) ON CONFLICT (transaction_id, output_index) DO UPDATE SET account_id = :account_id, + address_id = :address_id, address = :address, script = :script, value_zat = :value_zat, @@ -830,10 +1177,11 @@ pub(crate) fn put_transparent_output( let sql_args = named_params![ ":transaction_id": id_tx, ":output_index": &outpoint.n(), - ":account_id": receiving_account_id.0, + ":account_id": account_id.0, + ":address_id": address_id.0, ":address": &address.encode(params), ":script": &txout.script_pubkey.0, - ":value_zat": &i64::from(Amount::from(txout.value)), + ":value_zat": &i64::from(ZatBalance::from(txout.value)), ":max_observed_unspent_height": max_observed_unspent.map(u32::from), ]; @@ -862,7 +1210,7 @@ pub(crate) fn put_transparent_output( mark_transparent_utxo_spent(conn, spending_transaction_id, outpoint)?; } - Ok(utxo_id) + Ok((account_id, key_scope, utxo_id)) } /// Adds a request to retrieve transactions involving the specified address to the transparent @@ -903,14 +1251,19 @@ mod tests { use secrecy::Secret; use transparent::keys::NonHardenedChildIndex; use zcash_client_backend::{ - data_api::{testing::TestBuilder, Account as _, WalletWrite, GAP_LIMIT}, + data_api::{testing::TestBuilder, Account as _, WalletWrite}, wallet::TransparentAddressMetadata, }; use zcash_primitives::block::BlockHash; use crate::{ + error::SqliteClientError, testing::{db::TestDbFactory, BlockCache}, - wallet::{get_account_ref, transparent::ephemeral}, + wallet::{ + get_account_ref, + transparent::{ephemeral, find_gap_start, reserve_next_n_addresses}, + KeyScope, + }, WalletDb, }; @@ -949,21 +1302,59 @@ mod tests { let account0_uuid = st.test_account().unwrap().account().id(); let account0_id = get_account_ref(&st.wallet().db().conn, account0_uuid).unwrap(); + // The chain height must be known in order to reserve addresses, as we store the height at + // which the address was considered to be exposed. + st.wallet_mut() + .db_mut() + .update_chain_tip(birthday.height()) + .unwrap(); + let check = |db: &WalletDb<_, _>, account_id| { eprintln!("checking {account_id:?}"); - assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); - assert_matches!(ephemeral::first_unreserved_index(&db.conn, account_id), Ok(addr_index) if addr_index == 0); + assert_matches!( + find_gap_start(&db.conn, account_id, KeyScope::Ephemeral, db.gap_limits.ephemeral()), Ok(addr_index) + if addr_index == Some(NonHardenedChildIndex::ZERO) + ); + //assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); let known_addrs = ephemeral::get_known_ephemeral_addresses(&db.conn, &db.params, account_id, None) .unwrap(); - let expected_metadata: Vec = (0..GAP_LIMIT) + let expected_metadata: Vec = (0..db.gap_limits.ephemeral()) .map(|i| ephemeral::metadata(NonHardenedChildIndex::from_index(i).unwrap())) .collect(); let actual_metadata: Vec = known_addrs.into_iter().map(|(_, meta)| meta).collect(); assert_eq!(actual_metadata, expected_metadata); + + let transaction = &db.conn.unchecked_transaction().unwrap(); + // reserve half the addresses (rounding down) + let reserved = reserve_next_n_addresses( + transaction, + &db.params, + account_id, + KeyScope::Ephemeral, + db.gap_limits.ephemeral(), + (db.gap_limits.ephemeral() / 2) as usize, + ) + .unwrap(); + assert_eq!(reserved.len(), (db.gap_limits.ephemeral() / 2) as usize); + + // we have not yet used any of the addresses, so the maximum available address index + // should not have increased, and therefore attempting to reserve a full gap limit + // worth of addresses should fail. + assert_matches!( + reserve_next_n_addresses( + transaction, + &db.params, + account_id, + KeyScope::Ephemeral, + db.gap_limits.ephemeral(), + db.gap_limits.ephemeral() as usize + ), + Err(SqliteClientError::ReachedGapLimit(_)) + ); }; check(st.wallet().db(), account0_id); diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 21b2f9d371..3a7745b53b 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -1,24 +1,17 @@ //! Functions for wallet support of ephemeral transparent addresses. -use std::cmp::{max, min}; use std::ops::Range; use rusqlite::{named_params, OptionalExtension}; -use zcash_client_backend::{data_api::GAP_LIMIT, wallet::TransparentAddressMetadata}; -use zcash_keys::keys::UnifiedFullViewingKey; -use zcash_keys::{encoding::AddressCodec, keys::AddressGenerationError}; -use zcash_primitives::{ - legacy::{ - keys::{EphemeralIvk, NonHardenedChildIndex, TransparentKeyScope}, - TransparentAddress, - }, - transaction::TxId, +use ::transparent::{ + address::TransparentAddress, + keys::{NonHardenedChildIndex, TransparentKeyScope}, }; +use zcash_client_backend::wallet::TransparentAddressMetadata; +use zcash_keys::encoding::AddressCodec; use zcash_protocol::consensus; -use crate::wallet::{self, get_account_ref}; -use crate::AccountUuid; -use crate::{error::SqliteClientError, AccountRef, TxRef}; +use crate::{error::SqliteClientError, wallet::KeyScope, AccountRef, AccountUuid}; // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. @@ -26,161 +19,50 @@ pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddre TransparentAddressMetadata::new(TransparentKeyScope::EPHEMERAL, address_index) } -/// Returns the first unstored ephemeral address index in the given account. -pub(crate) fn first_unstored_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - match conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id - ORDER BY address_index DESC - LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - { - Some(i) if i >= (1 << 31) + GAP_LIMIT => { - unreachable!("violates constraint index_range_and_address_nullity") - } - Some(i) => Ok(i.checked_add(1).unwrap()), - None => Ok(0), - } -} - -/// Returns the first unreserved ephemeral address index in the given account. -pub(crate) fn first_unreserved_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - first_unstored_index(conn, account_id)? - .checked_sub(GAP_LIMIT) - .ok_or(SqliteClientError::CorruptedData( - "ephemeral_addresses table has not been initialized".to_owned(), - )) -} - -/// Returns the first ephemeral address index in the given account that -/// would violate the gap invariant if used. -pub(crate) fn first_unsafe_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - // The inner join with `transactions` excludes addresses for which - // `seen_in_tx` is NULL. The query also excludes addresses observed - // to have been mined in a transaction that we currently see as unmined. - // This is conservative in terms of avoiding violation of the gap - // invariant: it can only cause us to get to the end of the gap sooner. - // - // TODO: do we want to only consider transactions with a minimum number - // of confirmations here? - let first_unmined_index: u32 = match conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - JOIN transactions t ON t.id_tx = seen_in_tx - WHERE account_id = :account_id AND t.mined_height IS NOT NULL - ORDER BY address_index DESC - LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - { - Some(i) if i >= 1 << 31 => { - unreachable!("violates constraint index_range_and_address_nullity") - } - Some(i) => i.checked_add(1).unwrap(), - None => 0, - }; - Ok(min( - 1 << 31, - first_unmined_index.checked_add(GAP_LIMIT).unwrap(), - )) -} - -/// Utility function to return an `Range` that starts at `i` -/// and is of length up to `n`. The range is truncated if necessary -/// so that it contains no elements beyond the maximum valid address -/// index, `(1 << 31) - 1`. -pub(crate) fn range_from(i: u32, n: u32) -> Range { - let first = min(1 << 31, i); - let last = min(1 << 31, i.saturating_add(n)); - first..last -} - -/// Returns the ephemeral transparent IVK for a given account ID. -pub(crate) fn get_ephemeral_ivk( - conn: &rusqlite::Connection, - params: &P, - account_id: AccountRef, -) -> Result, SqliteClientError> { - let ufvk = conn - .query_row( - "SELECT ufvk FROM accounts WHERE id = :account_id", - named_params![":account_id": account_id.0], - |row| { - let ufvk_str: Option = row.get("ufvk")?; - Ok(ufvk_str.map(|s| { - UnifiedFullViewingKey::decode(params, &s[..]) - .map_err(SqliteClientError::BadAccountData) - })) - }, - ) - .optional()? - .ok_or(SqliteClientError::AccountUnknown)? - .transpose()?; - - let eivk = ufvk - .as_ref() - .and_then(|ufvk| ufvk.transparent()) - .map(|t| t.derive_ephemeral_ivk()) - .transpose()?; - - Ok(eivk) -} - -/// Returns a vector of ephemeral transparent addresses associated with the given -/// account controlled by this wallet, along with their metadata. The result includes -/// reserved addresses, and addresses for `GAP_LIMIT` additional indices (capped to -/// the maximum index). +/// Returns a vector of ephemeral transparent addresses associated with the given account +/// controlled by this wallet, along with their metadata. The result includes reserved addresses, +/// and addresses for the wallet's configured ephemeral address gap limit of additional indices +/// (capped to the maximum index). /// -/// If `index_range` is some `Range`, it limits the result to addresses with indices -/// in that range. +/// If `index_range` is some `Range`, it limits the result to addresses with indices in that range. pub(crate) fn get_known_ephemeral_addresses( conn: &rusqlite::Connection, params: &P, account_id: AccountRef, - index_range: Option>, + index_range: Option>, ) -> Result, SqliteClientError> { - let index_range = index_range.unwrap_or(0..(1 << 31)); - let mut stmt = conn.prepare( - "SELECT address, address_index - FROM ephemeral_addresses ea - WHERE ea.account_id = :account_id - AND address_index >= :start - AND address_index < :end - ORDER BY address_index", + "SELECT cached_transparent_receiver_address, transparent_child_index + FROM addresses + WHERE account_id = :account_id + AND transparent_child_index >= :start + AND transparent_child_index < :end + AND key_scope = :key_scope + ORDER BY transparent_child_index", )?; - let mut rows = stmt.query(named_params![ - ":account_id": account_id.0, - ":start": index_range.start, - ":end": min(1 << 31, index_range.end), - ])?; - let mut result = vec![]; + let results = stmt + .query_and_then( + named_params![ + ":account_id": account_id.0, + ":start": index_range.as_ref().map_or(NonHardenedChildIndex::ZERO, |i| i.start).index(), + ":end": index_range.as_ref().map_or(NonHardenedChildIndex::MAX, |i| i.end).index(), + ":key_scope": KeyScope::Ephemeral.encode() + ], + |row| { + let addr_str: String = row.get(0)?; + let raw_index: u32 = row.get(1)?; + let address_index = NonHardenedChildIndex::from_index(raw_index) + .expect("where clause ensures this is in range"); + Ok::<_, SqliteClientError>(( + TransparentAddress::decode(params, &addr_str)?, + metadata(address_index) + )) + }, + )? + .collect::, _>>()?; - while let Some(row) = rows.next()? { - let addr_str: String = row.get(0)?; - let raw_index: u32 = row.get(1)?; - let address_index = NonHardenedChildIndex::from_index(raw_index) - .expect("where clause ensures this is in range"); - let address = TransparentAddress::decode(params, &addr_str)?; - result.push((address, metadata(address_index))); - } - Ok(result) + Ok(results) } /// If this is a known ephemeral address in any account, return its account id. @@ -191,277 +73,15 @@ pub(crate) fn find_account_for_ephemeral_address_str( Ok(conn .query_row( "SELECT accounts.uuid - FROM ephemeral_addresses ea - JOIN accounts ON accounts.id = ea.account_id - WHERE address = :address", - named_params![":address": &address_str], + FROM addresses + JOIN accounts ON accounts.id = account_id + WHERE cached_transparent_receiver_address = :address + AND key_scope = :key_scope", + named_params![ + ":address": &address_str, + ":key_scope": KeyScope::Ephemeral.encode() + ], |row| Ok(AccountUuid(row.get(0)?)), ) .optional()?) } - -/// If this is a known ephemeral address in the given account, return its index. -pub(crate) fn find_index_for_ephemeral_address_str( - conn: &rusqlite::Connection, - account_uuid: AccountUuid, - address_str: &str, -) -> Result, SqliteClientError> { - let account_id = get_account_ref(conn, account_uuid)?; - Ok(conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id AND address = :address", - named_params![":account_id": account_id.0, ":address": &address_str], - |row| row.get::<_, u32>(0), - ) - .optional()? - .map(|index| { - NonHardenedChildIndex::from_index(index) - .expect("valid by constraint index_range_and_address_nullity") - })) -} - -/// Returns a vector with the next `n` previously unreserved ephemeral addresses for -/// the given account. -/// -/// # Errors -/// -/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. -/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and -/// it is not possible to derive new addresses for it. -/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses -/// within the gap limit after the last address in this account that is known to have an -/// output in a mined transaction. -/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, -/// if the limit on transparent address indices has been reached. -pub(crate) fn reserve_next_n_ephemeral_addresses( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, - n: usize, -) -> Result, SqliteClientError> { - if n == 0 { - return Ok(vec![]); - } - - let first_unreserved = first_unreserved_index(conn, account_id)?; - let first_unsafe = first_unsafe_index(conn, account_id)?; - let allocation = range_from( - first_unreserved, - u32::try_from(n).map_err(|_| AddressGenerationError::DiversifierSpaceExhausted)?, - ); - - if allocation.len() < n { - return Err(AddressGenerationError::DiversifierSpaceExhausted.into()); - } - if allocation.end > first_unsafe { - let account_uuid = wallet::get_account_uuid(conn, account_id)?; - return Err(SqliteClientError::ReachedGapLimit( - account_uuid, - max(first_unreserved, first_unsafe), - )); - } - reserve_until(conn, params, account_id, allocation.end)?; - get_known_ephemeral_addresses(conn, params, account_id, Some(allocation)) -} - -/// Initialize the `ephemeral_addresses` table. This must be called when -/// creating or migrating an account. -pub(crate) fn init_account( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, -) -> Result<(), SqliteClientError> { - reserve_until(conn, params, account_id, 0) -} - -/// Extend the range of stored addresses in an account if necessary so that the index of the next -/// address to reserve will be *at least* `next_to_reserve`. If no transparent key exists for the -/// given account or it would already have been at least `next_to_reserve`, then do nothing. -/// -/// Note that this is called from database migration code. -/// -/// # Panics -/// -/// Panics if the precondition `next_to_reserve <= (1 << 31)` does not hold. -fn reserve_until( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, - next_to_reserve: u32, -) -> Result<(), SqliteClientError> { - assert!(next_to_reserve <= 1 << 31); - - if let Some(ephemeral_ivk) = get_ephemeral_ivk(conn, params, account_id)? { - let first_unstored = first_unstored_index(conn, account_id)?; - let range_to_store = first_unstored..(next_to_reserve.checked_add(GAP_LIMIT).unwrap()); - if range_to_store.is_empty() { - return Ok(()); - } - - // used_in_tx and seen_in_tx are initially NULL - let mut stmt_insert_ephemeral_address = conn.prepare_cached( - "INSERT INTO ephemeral_addresses (account_id, address_index, address) - VALUES (:account_id, :address_index, :address)", - )?; - - for raw_index in range_to_store { - // The range to store may contain indicies that are out of the valid range of non hardened - // child indices; we still store explicit rows in the ephemeral_addresses table for these - // so that it's possible to find the first unused address using dead reckoning with the gap - // limit. - let address_str_opt = NonHardenedChildIndex::from_index(raw_index) - .map(|address_index| { - ephemeral_ivk - .derive_ephemeral_address(address_index) - .map(|addr| addr.encode(params)) - }) - .transpose()?; - - stmt_insert_ephemeral_address.execute(named_params![ - ":account_id": account_id.0, - ":address_index": raw_index, - ":address": address_str_opt, - ])?; - } - } - - Ok(()) -} - -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -fn ephemeral_address_reuse_check( - conn: &rusqlite::Transaction, - address_str: &str, -) -> Result<(), SqliteClientError> { - // It is intentional that we don't require `t.mined_height` to be non-null. - // That is, we conservatively treat an ephemeral address as potentially - // reused even if we think that the transaction where we had evidence of - // its use is at present unmined. This should never occur in supported - // situations where only a single correctly operating wallet instance is - // using a given seed, because such a wallet will not reuse an address that - // it ever reserved. - // - // `COALESCE(used_in_tx, seen_in_tx)` can only differ from `used_in_tx` - // if the address was reserved, an error occurred in transaction creation - // before calling `mark_ephemeral_address_as_used`, and then we saw the - // address in another transaction (presumably created by another wallet - // instance, or as a result of a bug) anyway. - let res = conn - .query_row( - "SELECT t.txid FROM ephemeral_addresses - LEFT OUTER JOIN transactions t - ON t.id_tx = COALESCE(used_in_tx, seen_in_tx) - WHERE address = :address", - named_params![":address": address_str], - |row| row.get::<_, Option>>(0), - ) - .optional()? - .flatten(); - - if let Some(txid_bytes) = res { - let txid = TxId::from_bytes( - txid_bytes - .try_into() - .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, - ); - Err(SqliteClientError::EphemeralAddressReuse( - address_str.to_owned(), - txid, - )) - } else { - Ok(()) - } -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in a transaction that we have just created. This has no effect if `address` is -/// not one of our ephemeral addresses. -/// -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -pub(crate) fn mark_ephemeral_address_as_used( - conn: &rusqlite::Transaction, - params: &P, - ephemeral_address: &TransparentAddress, - tx_ref: TxRef, -) -> Result<(), SqliteClientError> { - let address_str = ephemeral_address.encode(params); - ephemeral_address_reuse_check(conn, &address_str)?; - - // We update both `used_in_tx` and `seen_in_tx` here, because a used address has - // necessarily been seen in a transaction. We will not treat this as extending the - // range of addresses that are safe to reserve unless and until the transaction is - // observed as mined. - let update_result = conn - .query_row( - "UPDATE ephemeral_addresses - SET used_in_tx = :tx_ref, seen_in_tx = :tx_ref - WHERE address = :address - RETURNING account_id, address_index", - named_params![":tx_ref": tx_ref.0, ":address": address_str], - |row| Ok((AccountRef(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), - ) - .optional()?; - - // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. - if let Some((account_id, address_index)) = update_result { - let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); - reserve_until(conn, params, account_id, next_to_reserve)?; - } - Ok(()) -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in the given mined transaction (which may or may not be a transaction we sent). -/// -/// `tx_ref` must be a valid transaction reference. This call has no effect if -/// `address` is not one of our ephemeral addresses. -pub(crate) fn mark_ephemeral_address_as_seen( - conn: &rusqlite::Transaction, - params: &P, - address: &TransparentAddress, - tx_ref: TxRef, -) -> Result<(), SqliteClientError> { - let address_str = address.encode(params); - - // Figure out which transaction was mined earlier: `tx_ref`, or any existing - // tx referenced by `seen_in_tx` for the given address. Prefer the existing - // reference in case of a tie or if both transactions are unmined. - // This slightly reduces the chance of unnecessarily reaching the gap limit - // too early in some corner cases (because the earlier transaction is less - // likely to be unmined). - // - // The query should always return a value if `tx_ref` is valid. - let earlier_ref = conn.query_row( - "SELECT id_tx FROM transactions - LEFT OUTER JOIN ephemeral_addresses e - ON id_tx = e.seen_in_tx - WHERE id_tx = :tx_ref OR e.address = :address - ORDER BY mined_height ASC NULLS LAST, - tx_index ASC NULLS LAST, - e.seen_in_tx ASC NULLS LAST - LIMIT 1", - named_params![":tx_ref": tx_ref.0, ":address": address_str], - |row| row.get::<_, i64>(0), - )?; - - let update_result = conn - .query_row( - "UPDATE ephemeral_addresses - SET seen_in_tx = :seen_in_tx - WHERE address = :address - RETURNING account_id, address_index", - named_params![":seen_in_tx": &earlier_ref, ":address": address_str], - |row| Ok((AccountRef(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), - ) - .optional()?; - - // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. - if let Some((account_id, address_index)) = update_result { - let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); - reserve_until(conn, params, account_id, next_to_reserve)?; - } - Ok(()) -} diff --git a/zcash_keys/src/address.rs b/zcash_keys/src/address.rs index 15b7fb8fd3..3a196ecf99 100644 --- a/zcash_keys/src/address.rs +++ b/zcash_keys/src/address.rs @@ -424,6 +424,18 @@ impl Address { }, } } + + /// Returns the transparent address corresponding to this address, if it is a transparent + /// address, a Unified address with a transparent receiver, or ZIP 320 (TEX) address. + pub fn to_transparent_address(&self) -> Option { + match self { + #[cfg(feature = "sapling")] + Address::Sapling(_) => None, + Address::Transparent(addr) => Some(*addr), + Address::Unified(ua) => ua.transparent().copied(), + Address::Tex(addr_bytes) => Some(TransparentAddress::PublicKeyHash(*addr_bytes)), + } + } } #[cfg(all( diff --git a/zcash_transparent/CHANGELOG.md b/zcash_transparent/CHANGELOG.md index 46dc2a245c..5c7fe55363 100644 --- a/zcash_transparent/CHANGELOG.md +++ b/zcash_transparent/CHANGELOG.md @@ -13,6 +13,9 @@ and this library adheres to Rust's notion of ### Added - `zcash_transparent::pczt::Bip32Derivation::extract_bip_44_fields` +- `zcash_transparent::keys::NonHardenedChildIndex::saturating_sub` +- `zcash_transparent::keys::NonHardenedChildIndex::saturating_add` +- `impl From for zip32::DiversifierIndex` ## [0.1.0] - 2024-12-16 diff --git a/zcash_transparent/src/keys.rs b/zcash_transparent/src/keys.rs index 74c6dccb39..fc068766d9 100644 --- a/zcash_transparent/src/keys.rs +++ b/zcash_transparent/src/keys.rs @@ -2,6 +2,7 @@ use bip32::ChildNumber; use subtle::{Choice, ConstantTimeEq}; +use zip32::DiversifierIndex; #[cfg(feature = "transparent-inputs")] use { @@ -67,7 +68,7 @@ impl From for ChildNumber { /// A child index for a derived transparent address. /// /// Only NON-hardened derivation is supported. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct NonHardenedChildIndex(u32); impl ConstantTimeEq for NonHardenedChildIndex { @@ -77,13 +78,17 @@ impl ConstantTimeEq for NonHardenedChildIndex { } impl NonHardenedChildIndex { + /// The minimum valid non-hardened child index. pub const ZERO: NonHardenedChildIndex = NonHardenedChildIndex(0); + /// The maximum valid non-hardened child index. + pub const MAX: NonHardenedChildIndex = NonHardenedChildIndex((1 << 31) - 1); + /// Parses the given ZIP 32 child index. /// /// Returns `None` if the hardened bit is set. - pub fn from_index(i: u32) -> Option { - if i < (1 << 31) { + pub const fn from_index(i: u32) -> Option { + if i <= Self::MAX.0 { Some(NonHardenedChildIndex(i)) } else { None @@ -91,26 +96,39 @@ impl NonHardenedChildIndex { } /// Returns the index as a 32-bit integer. - pub fn index(&self) -> u32 { + pub const fn index(&self) -> u32 { self.0 } - pub fn next(&self) -> Option { + /// Returns the successor to this index. + pub const fn next(&self) -> Option { // overflow cannot happen because self.0 is 31 bits, and the next index is at most 32 bits // which in that case would lead from_index to return None. Self::from_index(self.0 + 1) } + + /// Subtracts the given delta from this index. + pub const fn saturating_sub(&self, delta: u32) -> Self { + NonHardenedChildIndex(self.0.saturating_sub(delta)) + } + + /// Adds the given delta to this index, returning a maximum possible value of + /// [`NonHardenedChildIndex::MAX`]. + pub const fn saturating_add(&self, delta: u32) -> Self { + let idx = self.0 + delta; + if idx > Self::MAX.0 { + Self::MAX + } else { + NonHardenedChildIndex(idx) + } + } } impl TryFrom for NonHardenedChildIndex { type Error = (); fn try_from(value: ChildNumber) -> Result { - if value.is_hardened() { - Err(()) - } else { - NonHardenedChildIndex::from_index(value.index()).ok_or(()) - } + NonHardenedChildIndex::from_index(value.index()).ok_or(()) } } @@ -120,6 +138,21 @@ impl From for ChildNumber { } } +impl TryFrom for NonHardenedChildIndex { + type Error = (); + + fn try_from(value: DiversifierIndex) -> Result { + let idx = u32::try_from(value).map_err(|_| ())?; + NonHardenedChildIndex::from_index(idx).ok_or(()) + } +} + +impl From for DiversifierIndex { + fn from(value: NonHardenedChildIndex) -> Self { + DiversifierIndex::from(value.0) + } +} + /// A [BIP44] private key at the account path level `m/44'/'/'`. /// /// [BIP44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki