From d316591212d169e3d3f3fd2b47262547c4aea0ee Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 2 Dec 2024 08:36:24 -0700 Subject: [PATCH] zcash_client_sqlite: Add handling for the transparent address gap limit. This is a large change that unifies the handling of ephemeral transparent addresses for ZIP 320 support with generalized "gap limit" handling for transparent wallet recovery. The best way to understand this commit is to start from the `transparent_gap_limit_handling` database migration that drives the change in behavior. --- zcash_client_backend/CHANGELOG.md | 16 +- zcash_client_backend/src/data_api.rs | 40 +- zcash_client_backend/src/data_api/testing.rs | 11 + .../src/data_api/testing/pool.rs | 164 ++--- zcash_client_backend/src/sync.rs | 5 +- zcash_client_sqlite/CHANGELOG.md | 8 + zcash_client_sqlite/src/error.rs | 36 +- zcash_client_sqlite/src/lib.rs | 282 +++++++-- zcash_client_sqlite/src/testing/db.rs | 11 +- zcash_client_sqlite/src/testing/pool.rs | 6 +- zcash_client_sqlite/src/wallet.rs | 435 +++++++++---- zcash_client_sqlite/src/wallet/db.rs | 157 ++--- zcash_client_sqlite/src/wallet/init.rs | 17 +- .../src/wallet/init/migrations.rs | 8 +- .../init/migrations/add_utxo_account.rs | 22 +- .../init/migrations/ephemeral_addresses.rs | 82 ++- .../migrations/fix_bad_change_flagging.rs | 5 +- .../init/migrations/receiving_key_scopes.rs | 23 +- .../transparent_gap_limit_handling.rs | 491 +++++++++++++++ zcash_client_sqlite/src/wallet/orchard.rs | 88 ++- zcash_client_sqlite/src/wallet/sapling.rs | 95 ++- zcash_client_sqlite/src/wallet/transparent.rs | 577 +++++++++++++++--- .../src/wallet/transparent/ephemeral.rs | 472 ++------------ zcash_transparent/CHANGELOG.md | 5 + zcash_transparent/src/keys.rs | 47 +- 25 files changed, 2109 insertions(+), 994 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 1d3860dcd..917e20421 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -18,9 +18,19 @@ and this library adheres to Rust's notion of `map_internal_account_note` and `map_ephemeral_transparent_outpoint` and `internal_account_note_transpose_option` methods have consequently been removed. -- `zcash_client_backend::data_api::WalletRead::get_known_ephemeral_addresses` - now takes a `Range` as its - argument instead of a `Range` +- `zcash_client_backend::data_api::WalletRead`: + - `get_transparent_receivers` now takes additional `include_change` and + `include_ephemeral` arguments. + - `get_known_ephemeral_addresses` now takes a + `Range` as its argument + instead of a `Range` +- `zcash_client_backend::data_api::WalletWrite` has an added method + `get_address_for_index` + +### Removed +- `zcash_client_backend::data_api::GAP_LIMIT` gap limits are now configured + based upon the key scope that they're associated with; there is no longer a + globally applicable gap limit. ### Deprecated - `zcash_client_backend::address` (use `zcash_keys::address` instead) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 15f9dafbc..0c6989c2e 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -81,7 +81,7 @@ use zcash_protocol::{ value::{BalanceError, Zatoshis}, ShieldedProtocol, TxId, }; -use zip32::fingerprint::SeedFingerprint; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use self::{ chain::{ChainState, CommitmentTreeRoot}, @@ -127,10 +127,6 @@ pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; #[cfg(feature = "orchard")] pub const ORCHARD_SHARD_HEIGHT: u8 = { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 } / 2; -/// The number of ephemeral addresses that can be safely reserved without observing any -/// of them to be mined. This is the same as the gap limit in Bitcoin. -pub const GAP_LIMIT: u32 = 20; - /// An enumeration of constraints that can be applied when querying for nullifiers for notes /// belonging to the wallet. pub enum NullifierQuery { @@ -1369,6 +1365,8 @@ pub trait WalletRead { fn get_transparent_receivers( &self, _account: Self::AccountId, + _include_change: bool, + _include_ephemeral: bool, ) -> Result>, Self::Error> { Ok(HashMap::new()) } @@ -1393,7 +1391,7 @@ pub trait WalletRead { /// This is equivalent to (but may be implemented more efficiently than): /// ```compile_fail /// Ok( - /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + /// if let Some(result) = self.get_transparent_receivers(account, true, true)?.get(address) { /// result.clone() /// } else { /// self.get_known_ephemeral_addresses(account, None)? @@ -1414,7 +1412,10 @@ pub trait WalletRead { ) -> Result, Self::Error> { // This should be overridden. Ok( - if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + if let Some(result) = self + .get_transparent_receivers(account, true, true)? + .get(address) + { result.clone() } else { self.get_known_ephemeral_addresses(account, None)? @@ -2360,9 +2361,10 @@ pub trait WalletWrite: WalletRead { key_source: Option<&str>, ) -> Result; - /// Generates and persists the next available diversified address for the specified account, - /// given the current addresses known to the wallet. If the `request` parameter is `None`, - /// an address should be generated using all of the available receivers for the account's UFVK. + /// Generates, persists, and marks as exposed the next available diversified address for the + /// specified account, given the current addresses known to the wallet. If the `request` + /// parameter is `None`, an address should be generated using all of the available receivers + /// for the account's UFVK. /// /// Returns `Ok(None)` if the account identifier does not correspond to a known /// account. @@ -2372,6 +2374,24 @@ pub trait WalletWrite: WalletRead { request: Option, ) -> Result, Self::Error>; + /// Generates, persists, and marks as exposed a diversified address for the specified account + /// at the provided diversifier index. If the `request` parameter is `None`, an address should + /// be generated using all of the available receivers for the account's UFVK. + /// + /// In the case that the diversifier index is outside of the range of valid transparent address + /// indexes, no transparent receiver should be generated in the resulting unified address. If a + /// transparent receiver is specifically requested for such a diversifier index, + /// implementations of this method should return an error. + /// + /// Address generation should fail if a transparent receiver would be generated that violates + /// the backend's internally configured gap limit for HD-seed-based recovery. + fn get_address_for_index( + &mut self, + account: Self::AccountId, + diversifier_index: DiversifierIndex, + request: Option, + ) -> Result, Self::Error>; + /// Updates the wallet's view of the blockchain. /// /// This method is used to provide the wallet with information about the state of the diff --git a/zcash_client_backend/src/data_api/testing.rs b/zcash_client_backend/src/data_api/testing.rs index 4b88bb032..188ef9c83 100644 --- a/zcash_client_backend/src/data_api/testing.rs +++ b/zcash_client_backend/src/data_api/testing.rs @@ -2599,6 +2599,8 @@ impl WalletRead for MockWalletDb { fn get_transparent_receivers( &self, _account: Self::AccountId, + _include_change: bool, + _include_ephemeral: bool, ) -> Result>, Self::Error> { Ok(HashMap::new()) } @@ -2689,6 +2691,15 @@ impl WalletWrite for MockWalletDb { Ok(None) } + fn get_address_for_index( + &mut self, + _account: Self::AccountId, + _diversifier_index: DiversifierIndex, + _request: Option, + ) -> Result, Self::Error> { + Ok(None) + } + #[allow(clippy::type_complexity)] fn put_blocks( &mut self, diff --git a/zcash_client_backend/src/data_api/testing/pool.rs b/zcash_client_backend/src/data_api/testing/pool.rs index a58e73b78..951f92a6c 100644 --- a/zcash_client_backend/src/data_api/testing/pool.rs +++ b/zcash_client_backend/src/data_api/testing/pool.rs @@ -60,7 +60,7 @@ use super::{DataStoreFactory, Reset, TestCache, TestFvk, TestState}; #[cfg(feature = "transparent-inputs")] use { crate::{ - data_api::{TransactionDataRequest, TransactionStatus}, + data_api::TransactionDataRequest, fees::ChangeValue, proposal::{Proposal, ProposalError, StepOutput, StepOutputIndex}, wallet::{TransparentAddressMetadata, WalletTransparentOutput}, @@ -86,6 +86,11 @@ use zcash_protocol::PoolType; #[cfg(feature = "pczt")] use pczt::roles::{prover::Prover, signer::Signer}; +/// The number of ephemeral addresses that can be safely reserved without observing any +/// of them to be mined. +#[cfg(feature = "transparent-inputs")] +pub const EPHEMERAL_ADDR_GAP_LIMIT: usize = 5; + /// Trait that exposes the pool-specific types and operations necessary to run the /// single-shielded-pool tests on a given pool. /// @@ -510,7 +515,7 @@ pub fn send_multi_step_proposed_transfer( { use ::transparent::builder::TransparentSigningSet; - use crate::data_api::{OutputOfSentTx, GAP_LIMIT}; + use crate::data_api::OutputOfSentTx; let mut st = TestBuilder::new() .with_data_store_factory(ds_factory) @@ -535,24 +540,30 @@ pub fn send_multi_step_proposed_transfer( .block_height(), h ); - assert_eq!(st.get_spendable_balance(account_id, 1), value); h }; let value = Zatoshis::const_from_u64(100000); let transfer_amount = Zatoshis::const_from_u64(50000); - let run_test = |st: &mut TestState<_, DSF::DataStore, _>, expected_index| { + let run_test = |st: &mut TestState<_, DSF::DataStore, _>, expected_index, prior_balance| { // Add funds to the wallet. add_funds(st, value); + let initial_balance: Option = prior_balance + value; + assert_eq!( + st.get_spendable_balance(account_id, 1), + initial_balance.unwrap() + ); let expected_step0_fee = (zip317::MARGINAL_FEE * 3u64).unwrap(); let expected_step1_fee = zip317::MINIMUM_FEE; let expected_ephemeral = (transfer_amount + expected_step1_fee).unwrap(); let expected_step0_change = - (value - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); + (initial_balance - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); assert!(expected_step0_change.is_positive()); + let total_sent = (expected_step0_fee + expected_step1_fee + transfer_amount).unwrap(); + // Generate a ZIP 320 proposal, sending to the wallet's default transparent address // expressed as a TEX address. let tex_addr = match default_addr { @@ -598,6 +609,12 @@ pub fn send_multi_step_proposed_transfer( assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); let txids = create_proposed_result.unwrap(); + // Mine the created transactions. + for txid in txids.iter() { + let (h, _) = st.generate_next_block_including(*txid); + st.scan_cached_blocks(h, 1); + } + // Check that there are sent outputs with the correct values. let confirmed_sent: Vec> = txids .iter() @@ -661,12 +678,15 @@ pub fn send_multi_step_proposed_transfer( -ZatBalance::from(expected_ephemeral), ); - (ephemeral_address.unwrap().0, txids) + let ending_balance = st.get_spendable_balance(account_id, 1); + assert_eq!(initial_balance - total_sent, ending_balance.into()); + + (ephemeral_address.unwrap().0, txids, ending_balance) }; // Each transfer should use a different ephemeral address. - let (ephemeral0, txids0) = run_test(&mut st, 0); - let (ephemeral1, txids1) = run_test(&mut st, 1); + let (ephemeral0, _, bal_0) = run_test(&mut st, 0, Zatoshis::ZERO); + let (ephemeral1, _, _) = run_test(&mut st, 1, bal_0); assert_ne!(ephemeral0, ephemeral1); let height = add_funds(&mut st, value); @@ -707,7 +727,7 @@ pub fn send_multi_step_proposed_transfer( .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(known_addrs.len(), (GAP_LIMIT as usize) + 2); + assert_eq!(known_addrs.len(), EPHEMERAL_ADDR_GAP_LIMIT + 2); // Check that the addresses are all distinct. let known_set: HashSet<_> = known_addrs.iter().map(|(addr, _)| addr).collect(); @@ -732,7 +752,7 @@ pub fn send_multi_step_proposed_transfer( }, ); let mut transparent_signing_set = TransparentSigningSet::new(); - let (colliding_addr, _) = &known_addrs[10]; + let (colliding_addr, _) = &known_addrs[EPHEMERAL_ADDR_GAP_LIMIT - 1]; let utxo_value = (value - zip317::MINIMUM_FEE).unwrap(); assert_matches!( builder.add_transparent_output(colliding_addr, utxo_value), @@ -773,9 +793,15 @@ pub fn send_multi_step_proposed_transfer( ) .unwrap(); let txid = build_result.transaction().txid(); + + // Now, store the transaction, pretending it has been mined (we will actually mine the block + // next). This will cause the the gap start to move & a new `EPHEMERAL_ADDR_GAP_LIMIT` of + + // addresses to be created. + let target_height = st.latest_cached_block().unwrap().height() + 1; st.wallet_mut() .store_decrypted_tx(DecryptedTransaction::new( - None, + Some(target_height), build_result.transaction(), vec![], #[cfg(feature = "orchard")] @@ -783,25 +809,20 @@ pub fn send_multi_step_proposed_transfer( )) .unwrap(); - // Verify that storing the fully transparent transaction causes a transaction - // status request to be generated. - let tx_data_requests = st.wallet().transaction_data_requests().unwrap(); - assert!(tx_data_requests.contains(&TransactionDataRequest::GetStatus(txid))); - - // We call get_transparent_output with `allow_unspendable = true` to verify - // storage because the decrypted transaction has not yet been mined. - let utxo = st - .wallet() - .get_transparent_output(&OutPoint::new(txid.into(), 0), true) - .unwrap(); - assert_matches!(utxo, Some(v) if v.value() == utxo_value); + // Mine the transaction & scan it so that it is will be detected as mined. Note that + // `generate_next_block_including` does not actually do anything with fully-transparent + // transactions; we're doing this just to get the mined block that we added via + // `store_decrypted_tx` into the database. + let (h, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(h, 1); - // That should have advanced the start of the gap to index 11. + // At this point the start of the gap should be at index `EPHEMERAL_ADDR_GAP_LIMIT` and the new + // size of the known address set should be `EPHEMERAL_ADDR_GAP_LIMIT * 2`. let new_known_addrs = st .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(new_known_addrs.len(), (GAP_LIMIT as usize) + 11); + assert_eq!(new_known_addrs.len(), EPHEMERAL_ADDR_GAP_LIMIT * 2); assert!(new_known_addrs.starts_with(&known_addrs)); let reservation_should_succeed = |st: &mut TestState<_, DSF::DataStore, _>, n| { @@ -821,85 +842,20 @@ pub fn send_multi_step_proposed_transfer( }; let next_reserved = reservation_should_succeed(&mut st, 1); - assert_eq!(next_reserved[0], known_addrs[11]); - - // Calling `reserve_next_n_ephemeral_addresses(account_id, 1)` will have advanced - // the start of the gap to index 12. This also tests the `index_range` parameter. - let newer_known_addrs = st - .wallet() - .get_known_ephemeral_addresses( - account_id, - Some( - NonHardenedChildIndex::from_index(5).unwrap() - ..NonHardenedChildIndex::from_index(100).unwrap(), - ), - ) - .unwrap(); - assert_eq!(newer_known_addrs.len(), (GAP_LIMIT as usize) + 12 - 5); - assert!(newer_known_addrs.starts_with(&new_known_addrs[5..])); - - // None of the five transactions created above (two from each proposal and the - // one built manually) have been mined yet. So, the range of address indices - // that are safe to reserve is still 0..20, and we have already reserved 12 - // addresses, so trying to reserve another 9 should fail. - reservation_should_fail(&mut st, 9, 20); - reservation_should_succeed(&mut st, 8); - reservation_should_fail(&mut st, 1, 20); - - // Now mine the transaction with the ephemeral output at index 1. - // We already reserved 20 addresses, so this should allow 2 more (..22). - // It does not matter that the transaction with ephemeral output at index 0 - // remains unmined. - let (h, _) = st.generate_next_block_including(txids1.head); - st.scan_cached_blocks(h, 1); - reservation_should_succeed(&mut st, 2); - reservation_should_fail(&mut st, 1, 22); - - // Mining the transaction with the ephemeral output at index 0 at this point - // should make no difference. - let (h, _) = st.generate_next_block_including(txids0.head); - st.scan_cached_blocks(h, 1); - reservation_should_fail(&mut st, 1, 22); - - // Now mine the transaction with the ephemeral output at index 10. - let tx = build_result.transaction(); - let tx_index = 1; - let (h, _) = st.generate_next_block_from_tx(tx_index, tx); - st.scan_cached_blocks(h, 1); - - // The above `scan_cached_blocks` does not detect `tx` as interesting to the - // wallet. If a transaction is in the database with a null `mined_height`, - // as in this case, its `mined_height` will remain null unless either - // `put_tx_meta` or `set_transaction_status` is called on it. The former - // is normally called internally via `put_blocks` as a result of scanning, - // but not for the case of a fully transparent transaction. The latter is - // called by the wallet implementation in response to processing the - // `transaction_data_requests` queue. - - // The reservation should fail because `tx` is not yet seen as mined. - reservation_should_fail(&mut st, 1, 22); - - // Simulate the wallet processing the `transaction_data_requests` queue. - let tx_data_requests = st.wallet().transaction_data_requests().unwrap(); - assert!(tx_data_requests.contains(&TransactionDataRequest::GetStatus(tx.txid()))); - - // Respond to the GetStatus request. - st.wallet_mut() - .set_transaction_status(tx.txid(), TransactionStatus::Mined(h)) - .unwrap(); - - // We already reserved 22 addresses, so mining the transaction with the - // ephemeral output at index 10 should allow 9 more (..31). - reservation_should_succeed(&mut st, 9); - reservation_should_fail(&mut st, 1, 31); - - let newest_known_addrs = st - .wallet() - .get_known_ephemeral_addresses(account_id, None) - .unwrap(); - assert_eq!(newest_known_addrs.len(), (GAP_LIMIT as usize) + 31); - assert!(newest_known_addrs.starts_with(&known_addrs)); - assert!(newest_known_addrs[5..].starts_with(&newer_known_addrs)); + assert_eq!(next_reserved[0], known_addrs[EPHEMERAL_ADDR_GAP_LIMIT]); + + // The range of address indices that are safe to reserve now is + // 0..(EPHEMERAL_ADDR_GAP_LIMIT * 2 - 1)`, and we have already reserved or used + // `EPHEMERAL_ADDR_GAP_LIMIT + 1`, addresses, so trying to reserve another + // `EPHEMERAL_ADDR_GAP_LIMIT` should fail. + reservation_should_fail( + &mut st, + EPHEMERAL_ADDR_GAP_LIMIT, + (EPHEMERAL_ADDR_GAP_LIMIT * 2) as u32, + ); + reservation_should_succeed(&mut st, EPHEMERAL_ADDR_GAP_LIMIT - 1); + // Now we've reserved everything we can, we can't reserve one more + reservation_should_fail(&mut st, 1, (EPHEMERAL_ADDR_GAP_LIMIT * 2) as u32); } #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_backend/src/sync.rs b/zcash_client_backend/src/sync.rs index e2a868154..8519d19fc 100644 --- a/zcash_client_backend/src/sync.rs +++ b/zcash_client_backend/src/sync.rs @@ -135,7 +135,7 @@ where "Refreshing UTXOs for {:?} from height {}", account_id, start_height, ); - refresh_utxos(params, client, db_data, account_id, start_height).await?; + refresh_utxos(params, client, db_data, account_id, start_height, false).await?; } // 5) Get the suggested scan ranges from the wallet database @@ -498,6 +498,7 @@ async fn refresh_utxos( db_data: &mut DbT, account_id: DbT::AccountId, start_height: BlockHeight, + include_ephemeral: bool, ) -> Result<(), Error::Error, TrErr>> where P: Parameters + Send + 'static, @@ -510,7 +511,7 @@ where { let request = service::GetAddressUtxosArg { addresses: db_data - .get_transparent_receivers(account_id) + .get_transparent_receivers(account_id, true, include_ephemeral) .map_err(Error::Wallet)? .into_keys() .map(|addr| addr.encode(params)) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 22b31c43d..e8cd3542b 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -9,6 +9,14 @@ and this library adheres to Rust's notion of ### Changed - Migrated to `nonempty 0.11` +- `zcash_client_sqlite::error::SqliteClientError` variants have changed: + - The `EphemeralAddressReuse` variant has been removed and replaced + by a new generalized `AddressReuse` error variant. + - The `ReachedGapLimit` variant no longer includes the account UUID + for the account that reached the limit in its payload. + - Each row returned from the `v_received_outputs` view now exposes an + internal identifier for the address that received that output. This should + be ignored by external consumers of this view. ## [0.14.0] - 2024-12-16 diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 7c9e48cc3..05958e435 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -3,19 +3,18 @@ use std::error; use std::fmt; +use nonempty::NonEmpty; use shardtree::error::ShardTreeError; + use zcash_address::ParseError; use zcash_client_backend::data_api::NoteFilter; use zcash_keys::keys::AddressGenerationError; -use zcash_protocol::{consensus::BlockHeight, value::BalanceError, PoolType}; +use zcash_protocol::{consensus::BlockHeight, value::BalanceError, PoolType, TxId}; use crate::{wallet::commitment_tree, AccountUuid}; #[cfg(feature = "transparent-inputs")] -use { - ::transparent::address::TransparentAddress, zcash_keys::encoding::TransparentCodecError, - zcash_primitives::transaction::TxId, -}; +use {::transparent::address::TransparentAddress, zcash_keys::encoding::TransparentCodecError}; /// The primary error type for the SQLite wallet backend. #[derive(Debug)] @@ -119,16 +118,16 @@ pub enum SqliteClientError { NoteFilterInvalid(NoteFilter), /// The proposal cannot be constructed until transactions with previously reserved - /// ephemeral address outputs have been mined. The parameters are the account UUID and - /// the index that could not safely be reserved. + /// ephemeral address outputs have been mined. The error contains the index that could not + /// safely be reserved. #[cfg(feature = "transparent-inputs")] - ReachedGapLimit(AccountUuid, u32), + ReachedGapLimit(u32), - /// An ephemeral address would be reused. The parameters are the address in string - /// form, and the txid of the earliest transaction in which it is known to have been - /// used. - #[cfg(feature = "transparent-inputs")] - EphemeralAddressReuse(String, TxId), + /// The wallet attempted to create a transaction that would use of one of the wallet's + /// previously-used addresses, potentially creating a problem with on-chain transaction + /// linkability. The returned value contains the string encoding of the address and the txid(s) + /// of the transactions in which it is known to have been used. + AddressReuse(String, NonEmpty), } impl error::Error for SqliteClientError { @@ -185,12 +184,13 @@ impl fmt::Display for SqliteClientError { SqliteClientError::BalanceError(e) => write!(f, "Balance error: {}", e), SqliteClientError::NoteFilterInvalid(s) => write!(f, "Could not evaluate filter query: {:?}", s), #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit(account_id, bad_index) => write!(f, - "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined. \ - The ephemeral address in account {account_id:?} at index {bad_index} could not be safely reserved.", + SqliteClientError::ReachedGapLimit(bad_index) => write!(f, + "The proposal cannot be constructed until transactions with outputs to previously reserved ephemeral addresses have been mined. \ + The ephemeral address at index {bad_index} could not be safely reserved.", ), - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(address_str, txid) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), + SqliteClientError::AddressReuse(address_str, txids) => { + write!(f, "The address {address_str} previously used in txid(s) {:?} would be reused.", txids) + } } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index bcf277197..6a464e761 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -79,9 +79,12 @@ use zcash_protocol::{ use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use crate::{error::SqliteClientError, wallet::commitment_tree::SqliteShardStore}; - -#[cfg(any(test, feature = "test-dependencies", not(feature = "orchard")))] -use zcash_protocol::PoolType; +use wallet::{ + chain_tip_height, + commitment_tree::{self, put_shard_roots}, + common::spendable_notes_meta, + SubtreeProgressEstimator, +}; #[cfg(feature = "orchard")] use { @@ -94,6 +97,7 @@ use { #[cfg(feature = "transparent-inputs")] use { ::transparent::{address::TransparentAddress, bundle::OutPoint, keys::NonHardenedChildIndex}, + std::collections::BTreeSet, zcash_client_backend::wallet::TransparentAddressMetadata, zcash_keys::encoding::AddressCodec, }; @@ -106,10 +110,17 @@ use maybe_rayon::{ #[cfg(any(test, feature = "test-dependencies"))] use { + rusqlite::named_params, zcash_client_backend::data_api::{testing::TransactionSummary, OutputOfSentTx, WalletTest}, zcash_keys::address::Address, }; +#[cfg(any(test, feature = "test-dependencies", feature = "transparent-inputs"))] +use wallet::KeyScope; + +#[cfg(any(test, feature = "test-dependencies", not(feature = "orchard")))] +use zcash_protocol::PoolType; + /// `maybe-rayon` doesn't provide this as a fallback, so we have to. #[cfg(not(feature = "multicore"))] trait ParallelSliceMut { @@ -131,15 +142,9 @@ use { pub mod chain; pub mod error; -pub mod wallet; -use wallet::{ - commitment_tree::{self, put_shard_roots}, - common::spendable_notes_meta, - SubtreeProgressEstimator, -}; - #[cfg(test)] mod testing; +pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than @@ -181,7 +186,7 @@ pub(crate) const UA_TRANSPARENT: ReceiverRequirement = ReceiverRequirement::Requ /// events". Examples of these include: /// - Restoring a wallet from a backed-up seed. /// - Importing the same viewing key into two different wallet instances. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct AccountUuid(#[cfg_attr(feature = "serde", serde(with = "uuid::serde::compact"))] Uuid); @@ -212,7 +217,7 @@ impl AccountUuid { /// /// This is an ephemeral value for efficiently and generically working with accounts in a /// [`WalletDb`]. To reference accounts in external contexts, use [`AccountUuid`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] pub(crate) struct AccountRef(u32); /// This implementation is retained under `#[cfg(test)]` for pre-AccountUuid testing. @@ -242,13 +247,63 @@ impl fmt::Display for ReceivedNoteId { pub struct UtxoId(pub i64); /// A newtype wrapper for sqlite primary key values for the transactions table. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] struct TxRef(pub i64); +/// A newtype wrapper for sqlite primary key values for the addresses table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct AddressRef(pub(crate) i64); + +/// A data structure that can be used to configure custom gap limits for use in transparent address +/// rotation. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[cfg(feature = "transparent-inputs")] +pub struct GapLimits { + external: u32, + transparent_internal: u32, + ephemeral: u32, +} + +#[cfg(feature = "transparent-inputs")] +impl GapLimits { + pub fn new(external: u32, transparent_internal: u32, ephemeral: u32) -> Self { + Self { + external, + transparent_internal, + ephemeral, + } + } + + pub(crate) fn external(&self) -> u32 { + self.external + } + + pub(crate) fn transparent_internal(&self) -> u32 { + self.transparent_internal + } + + pub(crate) fn ephemeral(&self) -> u32 { + self.ephemeral + } +} + +#[cfg(feature = "transparent-inputs")] +impl Default for GapLimits { + fn default() -> Self { + Self { + external: 20, + transparent_internal: 5, + ephemeral: 5, + } + } +} + /// A wrapper for the SQLite connection to the wallet database. pub struct WalletDb { conn: C, params: P, + #[cfg(feature = "transparent-inputs")] + gap_limits: GapLimits, } /// A wrapper for a SQLite transaction affecting the wallet database. @@ -265,10 +320,21 @@ impl WalletDb { pub fn for_path>(path: F, params: P) -> Result { Connection::open(path).and_then(move |conn| { rusqlite::vtab::array::load_module(&conn)?; - Ok(WalletDb { conn, params }) + Ok(WalletDb { + conn, + params, + #[cfg(feature = "transparent-inputs")] + gap_limits: GapLimits::default(), + }) }) } + #[cfg(feature = "transparent-inputs")] + pub fn with_gap_limits(mut self, gap_limits: GapLimits) -> Self { + self.gap_limits = gap_limits; + self + } + pub fn transactionally>(&mut self, f: F) -> Result where F: FnOnce(&mut WalletDb, P>) -> Result, @@ -277,6 +343,8 @@ impl WalletDb { let mut wdb = WalletDb { conn: SqlTransaction(&tx), params: self.params.clone(), + #[cfg(feature = "transparent-inputs")] + gap_limits: self.gap_limits, }; let result = f(&mut wdb)?; tx.commit()?; @@ -624,8 +692,21 @@ impl, P: consensus::Parameters> WalletRead for W fn get_transparent_receivers( &self, account: Self::AccountId, + include_change: bool, + include_ephemeral: bool, ) -> Result>, Self::Error> { - wallet::transparent::get_transparent_receivers(self.conn.borrow(), &self.params, account) + let key_scopes: &[KeyScope] = match (include_change, include_ephemeral) { + (true, true) => &[KeyScope::EXTERNAL, KeyScope::INTERNAL, KeyScope::Ephemeral], + (true, false) => &[KeyScope::EXTERNAL, KeyScope::INTERNAL], + (false, true) => &[KeyScope::EXTERNAL, KeyScope::Ephemeral], + (false, false) => &[KeyScope::EXTERNAL], + }; + wallet::transparent::get_transparent_receivers( + self.conn.borrow(), + &self.params, + account, + key_scopes, + ) } #[cfg(feature = "transparent-inputs")] @@ -667,7 +748,7 @@ impl, P: consensus::Parameters> WalletRead for W self.conn.borrow(), &self.params, account_id, - index_range.map(|i| i.start.index()..i.end.index()), + index_range, ) } @@ -710,7 +791,6 @@ impl, P: consensus::Parameters> WalletTest for W protocol: ShieldedProtocol, ) -> Result, ::Error> { use crate::wallet::pool_code; - use rusqlite::named_params; let mut stmt_sent_notes = self.conn.borrow().prepare( "SELECT output_index @@ -737,29 +817,35 @@ impl, P: consensus::Parameters> WalletTest for W &self, txid: &TxId, ) -> Result, ::Error> { - let mut stmt_sent = self - .conn.borrow() - .prepare( - "SELECT value, to_address, ephemeral_addresses.address, ephemeral_addresses.address_index - FROM sent_notes - JOIN transactions ON transactions.id_tx = sent_notes.tx - LEFT JOIN ephemeral_addresses ON ephemeral_addresses.used_in_tx = sent_notes.tx - WHERE transactions.txid = ? - ORDER BY value", - )?; + let mut stmt_sent = self.conn.borrow().prepare( + "SELECT value, to_address, + a.cached_transparent_receiver_address, a.transparent_child_index + FROM sent_notes + JOIN transactions t ON t.id_tx = sent_notes.tx + LEFT JOIN transparent_received_outputs tro ON tro.transaction_id = t.id_tx + LEFT JOIN addresses a ON a.id = tro.address_id AND a.key_scope = :key_scope + WHERE t.txid = :txid + ORDER BY value", + )?; let sends = stmt_sent - .query_map(rusqlite::params![txid.as_ref()], |row| { - let v = row.get(0)?; - let to_address = row - .get::<_, Option>(1)? - .and_then(|s| Address::decode(&self.params, &s)); - let ephemeral_address = row - .get::<_, Option>(2)? - .and_then(|s| Address::decode(&self.params, &s)); - let address_index: Option = row.get(3)?; - Ok((v, to_address, ephemeral_address.zip(address_index))) - })? + .query_map( + named_params![ + ":txid": txid.as_ref(), + ":key_scope": KeyScope::Ephemeral.encode() + ], + |row| { + let v = row.get(0)?; + let to_address = row + .get::<_, Option>(1)? + .and_then(|s| Address::decode(&self.params, &s)); + let ephemeral_address = row + .get::<_, Option>(2)? + .and_then(|s| Address::decode(&self.params, &s)); + let address_index: Option = row.get(3)?; + Ok((v, to_address, ephemeral_address.zip(address_index))) + }, + )? .map(|res| { let (amount, external_recipient, ephemeral_address) = res?; Ok::<_, ::Error>(OutputOfSentTx::from_parts( @@ -871,6 +957,8 @@ impl WalletWrite for WalletDb }, wallet::ViewingKey::Full(Box::new(ufvk)), birthday, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, )?; Ok((account.id(), usk)) @@ -908,6 +996,8 @@ impl WalletWrite for WalletDb }, wallet::ViewingKey::Full(Box::new(ufvk)), birthday, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, )?; Ok((account, usk)) @@ -933,6 +1023,8 @@ impl WalletWrite for WalletDb }, wallet::ViewingKey::Full(Box::new(ufvk.to_owned())), birthday, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, ) }) } @@ -957,14 +1049,16 @@ impl WalletWrite for WalletDb }; let (addr, diversifier_index) = ufvk.find_address(search_from, request)?; - let account_id = wallet::get_account_ref(wdb.conn.0, account_uuid)?; - wallet::insert_address( + let chain_tip_height = chain_tip_height(wdb.conn.0)? + .ok_or(SqliteClientError::ChainHeightUnknown)?; + wallet::upsert_address( wdb.conn.0, &wdb.params, account_id, diversifier_index, &addr, + Some(chain_tip_height), )?; Ok(Some(addr)) @@ -974,6 +1068,15 @@ impl WalletWrite for WalletDb ) } + fn get_address_for_index( + &mut self, + _account: Self::AccountId, + _diversifier_index: DiversifierIndex, + _request: Option, + ) -> Result, Self::Error> { + todo!() + } + fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error> { let tx = self.conn.transaction()?; wallet::scanning::update_chain_tip(&tx, &self.params, tip_height)?; @@ -1021,6 +1124,10 @@ impl WalletWrite for WalletDb let mut orchard_commitments = vec![]; let mut last_scanned_height = None; let mut note_positions = vec![]; + + #[cfg(feature = "transparent-inputs")] + let mut tx_refs = BTreeSet::new(); + for block in blocks.into_iter() { if last_scanned_height .iter() @@ -1044,16 +1151,20 @@ impl WalletWrite for WalletDb )?; for tx in block.transactions() { - let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + let tx_ref = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + + #[cfg(feature = "transparent-inputs")] + tx_refs.insert(tx_ref); + wallet::queue_tx_retrieval(wdb.conn.0, std::iter::once(tx.txid()), None)?; // Mark notes as spent and remove them from the scanning cache for spend in tx.sapling_spends() { - wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; + wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_ref, spend.nf())?; } #[cfg(feature = "orchard")] for spend in tx.orchard_spends() { - wallet::orchard::mark_orchard_note_spent(wdb.conn.0, tx_row, spend.nf())?; + wallet::orchard::mark_orchard_note_spent(wdb.conn.0, tx_ref, spend.nf())?; } for output in tx.sapling_outputs() { @@ -1071,7 +1182,14 @@ impl WalletWrite for WalletDb .transpose()? .flatten(); - wallet::sapling::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + wallet::sapling::put_received_note( + wdb.conn.0, + &wdb.params, + output, + tx_ref, + Some(block.height()), + spent_in, + )?; } #[cfg(feature = "orchard")] for output in tx.orchard_outputs() { @@ -1089,7 +1207,14 @@ impl WalletWrite for WalletDb .transpose()? .flatten(); - wallet::orchard::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + wallet::orchard::put_received_note( + wdb.conn.0, + &wdb.params, + output, + tx_ref, + Some(block.height()), + spent_in, + )?; } } @@ -1160,6 +1285,18 @@ impl WalletWrite for WalletDb orchard_commitments.extend(block_commitments.orchard.into_iter().map(Some)); } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in wallet::involved_accounts(wdb.conn.0, tx_refs)? { + wallet::transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + } + // Prune the nullifier map of entries we no longer need. if let Some(meta) = wdb.block_fully_scanned()? { wallet::prune_nullifier_map( @@ -1414,11 +1551,27 @@ impl WalletWrite for WalletDb _output: &WalletTransparentOutput, ) -> Result { #[cfg(feature = "transparent-inputs")] - return wallet::transparent::put_received_transparent_utxo( - &self.conn, - &self.params, - _output, - ); + { + self.transactionally(|wdb| { + let (account_id, key_scope, utxo_id) = + wallet::transparent::put_received_transparent_utxo( + wdb.conn.0, + &wdb.params, + _output, + )?; + + wallet::transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + + Ok(utxo_id) + }) + } #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -1430,7 +1583,15 @@ impl WalletWrite for WalletDb &mut self, d_tx: DecryptedTransaction, ) -> Result<(), Self::Error> { - self.transactionally(|wdb| wallet::store_decrypted_tx(wdb.conn.0, &wdb.params, d_tx)) + self.transactionally(|wdb| { + wallet::store_decrypted_tx( + wdb.conn.0, + &wdb.params, + d_tx, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, + ) + }) } fn store_transactions_to_be_sent( @@ -1457,12 +1618,16 @@ impl WalletWrite for WalletDb ) -> Result, Self::Error> { self.transactionally(|wdb| { let account_id = wallet::get_account_ref(wdb.conn.0, account_id)?; - wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses( + let reserved = wallet::transparent::reserve_next_n_addresses( wdb.conn.0, &wdb.params, account_id, + wallet::KeyScope::Ephemeral, + wdb.gap_limits.ephemeral(), n, - ) + )?; + + Ok(reserved.into_iter().map(|(_, a, m)| (a, m)).collect()) }) } @@ -1986,6 +2151,12 @@ mod tests { .build(); let account = st.test_account().cloned().unwrap(); + // We have to have the chain tip height in order to allocate new addresses, to record the + // exposed-at height. + st.wallet_mut() + .update_chain_tip(account.birthday().height()) + .unwrap(); + let current_addr = st.wallet().get_current_address(account.id()).unwrap(); assert!(current_addr.is_some()); @@ -2238,7 +2409,10 @@ mod tests { let ufvk = account.usk().to_unified_full_viewing_key(); let (taddr, _) = account.usk().default_transparent_address(); - let receivers = st.wallet().get_transparent_receivers(account.id()).unwrap(); + let receivers = st + .wallet() + .get_transparent_receivers(account.id(), false, false) + .unwrap(); // The receiver for the default UA should be in the set. assert!(receivers.contains_key( diff --git a/zcash_client_sqlite/src/testing/db.rs b/zcash_client_sqlite/src/testing/db.rs index e7fa1d300..1ec0a82f6 100644 --- a/zcash_client_sqlite/src/testing/db.rs +++ b/zcash_client_sqlite/src/testing/db.rs @@ -9,7 +9,6 @@ use tempfile::NamedTempFile; use rusqlite::{self}; use secrecy::SecretVec; use shardtree::{error::ShardTreeError, ShardTree}; -use zip32::fingerprint::SeedFingerprint; use zcash_client_backend::{ data_api::{ @@ -32,6 +31,7 @@ use zcash_protocol::{ consensus::BlockHeight, local_consensus::LocalNetwork, memo::Memo, value::Zatoshis, ShieldedProtocol, }; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use crate::{ error::SqliteClientError, @@ -153,15 +153,6 @@ pub(crate) struct TestDbFactory { target_migrations: Option>, } -impl TestDbFactory { - #[allow(dead_code)] - pub(crate) fn new(target_migrations: Vec) -> Self { - Self { - target_migrations: Some(target_migrations), - } - } -} - impl DataStoreFactory for TestDbFactory { type Error = (); type AccountId = AccountUuid; diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 1b5132a36..95edb8be8 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -48,11 +48,11 @@ pub(crate) fn send_multi_step_proposed_transfer() { zcash_client_backend::data_api::testing::pool::send_multi_step_proposed_transfer::( TestDbFactory::default(), BlockCache::new(), - |e, account_id, expected_bad_index| { + |e, _, expected_bad_index| { matches!( e, - crate::error::SqliteClientError::ReachedGapLimit(acct, bad_index) - if acct == &account_id && bad_index == &expected_bad_index) + crate::error::SqliteClientError::ReachedGapLimit(bad_index) + if bad_index == &expected_bad_index) }, ) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 99db16e47..fc29567d2 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,32 +64,29 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use incrementalmerkletree::{Marking, Retention}; +use std::{ + collections::{HashMap, HashSet}, + convert::TryFrom, + io::{self, Cursor}, + num::NonZeroU32, + ops::RangeInclusive, +}; +use incrementalmerkletree::{Marking, Retention}; use rusqlite::{self, named_params, params, Connection, OptionalExtension}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; -use uuid::Uuid; -use zcash_client_backend::data_api::{ - AccountPurpose, DecryptedTransaction, Progress, TransactionDataRequest, TransactionStatus, - Zip32Derivation, -}; -use zip32::fingerprint::SeedFingerprint; - -use std::collections::{HashMap, HashSet}; -use std::convert::TryFrom; -use std::io::{self, Cursor}; -use std::num::NonZeroU32; -use std::ops::RangeInclusive; - use tracing::{debug, warn}; +use uuid::Uuid; use zcash_address::ZcashAddress; use zcash_client_backend::{ data_api::{ scanning::{ScanPriority, ScanRange}, - Account as _, AccountBalance, AccountBirthday, AccountSource, BlockMetadata, Ratio, - SentTransaction, SentTransactionOutput, WalletSummary, SAPLING_SHARD_HEIGHT, + Account as _, AccountBalance, AccountBirthday, AccountPurpose, AccountSource, + BlockMetadata, DecryptedTransaction, Progress, Ratio, SentTransaction, + SentTransactionOutput, TransactionDataRequest, TransactionStatus, WalletSummary, + Zip32Derivation, SAPLING_SHARD_HEIGHT, }, wallet::{Note, NoteId, Recipient, WalletTx}, DecryptedOutput, @@ -105,27 +102,33 @@ use zcash_keys::{ use zcash_primitives::{ block::BlockHash, merkle_tree::read_commitment_tree, - transaction::{Transaction, TransactionData, TxId}, + transaction::{Transaction, TransactionData}, }; use zcash_protocol::{ - consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, + consensus::{self, BlockHeight, BranchId, NetworkConstants as _, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, value::{ZatBalance, Zatoshis}, - PoolType, ShieldedProtocol, + PoolType, ShieldedProtocol, TxId, }; -use zip32::{DiversifierIndex, Scope}; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use self::scanning::{parse_priority_code, priority_code, replace_queue_entries}; use crate::{ error::SqliteClientError, wallet::commitment_tree::{get_max_checkpointed_height, SqliteShardStore}, - AccountRef, SqlTransaction, TransferType, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, - SAPLING_TABLES_PREFIX, + AccountRef, AccountUuid, AddressRef, SqlTransaction, TransferType, TxRef, + WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, VERIFY_LOOKAHEAD, }; -use crate::{AccountUuid, TxRef, VERIFY_LOOKAHEAD}; #[cfg(feature = "transparent-inputs")] -use ::transparent::bundle::{OutPoint, TxOut}; +use { + crate::GapLimits, + ::transparent::{ + bundle::{OutPoint, TxOut}, + keys::{NonHardenedChildIndex, TransparentKeyScope}, + }, + std::collections::BTreeMap, +}; #[cfg(feature = "orchard")] use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; @@ -211,6 +214,7 @@ pub(crate) enum ViewingKey { /// An account stored in a `zcash_client_sqlite` database. #[derive(Debug, Clone)] pub struct Account { + id: AccountRef, uuid: AccountUuid, name: Option, kind: AccountSource, @@ -229,6 +233,10 @@ impl Account { ) -> Result<(UnifiedAddress, DiversifierIndex), AddressGenerationError> { self.uivk().default_address(request) } + + pub(crate) fn internal_id(&self) -> AccountRef { + self.id + } } impl zcash_client_backend::data_api::Account for Account { @@ -340,21 +348,91 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { } } -pub(crate) fn scope_code(scope: Scope) -> i64 { - match scope { - Scope::External => 0i64, - Scope::Internal => 1i64, +/// An enumeration of the scopes of keys that are generated by the `zcash_client_sqlite` +/// implementation of the `WalletWrite` trait. +/// +/// This extends the [`zip32::Scope`] type to include the custom scope used to generate keys for +/// ephemeral transparent addresses. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum KeyScope { + /// A key scope corresponding to a [`zip32::Scope`]. + Zip32(zip32::Scope), + /// An ephemeral transparent address, which is derived from an account's transparent + /// [`AccountPubKey`] with the BIP 44 path `change` level index set to the value `2`. + /// + /// [`AccountPubKey`]: zcash_primitives::legacy::keys::AccountPubKey + Ephemeral, +} + +impl KeyScope { + pub(crate) const EXTERNAL: KeyScope = KeyScope::Zip32(zip32::Scope::External); + pub(crate) const INTERNAL: KeyScope = KeyScope::Zip32(zip32::Scope::Internal); + + pub(crate) fn encode(&self) -> i64 { + match self { + KeyScope::Zip32(zip32::Scope::External) => 0i64, + KeyScope::Zip32(zip32::Scope::Internal) => 1i64, + KeyScope::Ephemeral => 2i64, + } + } + + pub(crate) fn decode(code: i64) -> Result { + match code { + 0i64 => Ok(KeyScope::EXTERNAL), + 1i64 => Ok(KeyScope::INTERNAL), + 2i64 => Ok(KeyScope::Ephemeral), + other => Err(SqliteClientError::CorruptedData(format!( + "Invalid key scope code: {}", + other + ))), + } + } +} + +impl From for KeyScope { + fn from(value: zip32::Scope) -> Self { + KeyScope::Zip32(value) + } +} + +#[cfg(feature = "transparent-inputs")] +impl From for TransparentKeyScope { + fn from(value: KeyScope) -> Self { + match value { + KeyScope::Zip32(scope) => scope.into(), + KeyScope::Ephemeral => TransparentKeyScope::custom(2).expect("valid scope"), + } } } -pub(crate) fn parse_scope(code: i64) -> Option { - match code { - 0i64 => Some(Scope::External), - 1i64 => Some(Scope::Internal), - _ => None, +impl TryFrom for zip32::Scope { + type Error = (); + + fn try_from(value: KeyScope) -> Result { + match value { + KeyScope::Zip32(scope) => Ok(scope), + KeyScope::Ephemeral => Err(()), + } } } +pub(crate) fn encode_diversifier_index_be(idx: DiversifierIndex) -> [u8; 11] { + let mut di_be = *idx.as_bytes(); + di_be.reverse(); + di_be +} + +pub(crate) fn decode_diversifier_index_be( + di_be: &[u8], +) -> Result { + let mut di_be: [u8; 11] = di_be.try_into().map_err(|_| { + SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) + })?; + di_be.reverse(); + + Ok(DiversifierIndex::from(di_be)) +} + pub(crate) fn memo_repr(memo: Option<&MemoBytes>) -> Option<&[u8]> { memo.map(|m| { if m == &MemoBytes::empty() { @@ -390,6 +468,7 @@ pub(crate) fn add_account( kind: &AccountSource, viewing_key: ViewingKey, birthday: &AccountBirthday, + #[cfg(feature = "transparent-inputs")] gap_limits: &GapLimits, ) -> Result { if let Some(ufvk) = viewing_key.ufvk() { // Check whether any component of this UFVK collides with an existing imported or derived FVK. @@ -508,6 +587,7 @@ pub(crate) fn add_account( })?; let account = Account { + id: account_id, name: Some(account_name.to_owned()), uuid: account_uuid, kind: kind.clone(), @@ -611,11 +691,21 @@ pub(crate) fn add_account( // key has fewer components than the wallet supports (most likely due to this being an // imported viewing key), derive an address containing the common subset of receivers. let (address, d_idx) = account.default_address(None)?; - insert_address(conn, params, account_id, d_idx, &address)?; + upsert_address( + conn, + params, + account_id, + d_idx, + &address, + Some(birthday.height()), + )?; - // Initialize the `ephemeral_addresses` table. + // Pre-generate transparent addresses up to the gap limits for the external, internal, + // and ephemeral key scopes. #[cfg(feature = "transparent-inputs")] - transparent::ephemeral::init_account(conn, params, account_id)?; + for key_scope in [KeyScope::EXTERNAL, KeyScope::INTERNAL, KeyScope::Ephemeral] { + transparent::generate_gap_addresses(conn, params, account_id, key_scope, gap_limits, None)?; + } Ok(account) } @@ -625,26 +715,31 @@ pub(crate) fn get_current_address( params: &P, account_uuid: AccountUuid, ) -> Result, SqliteClientError> { + let ua_prefix = params.network_type().hrp_unified_address(); // This returns the most recently generated address. let addr: Option<(String, Vec)> = conn .query_row( - "SELECT address, diversifier_index_be - FROM addresses - JOIN accounts ON addresses.account_id = accounts.id - WHERE accounts.uuid = :account_uuid - ORDER BY diversifier_index_be DESC - LIMIT 1", - named_params![":account_uuid": account_uuid.0], + &format!( + "SELECT address, diversifier_index_be + FROM addresses + JOIN accounts ON addresses.account_id = accounts.id + WHERE accounts.uuid = :account_uuid + AND key_scope = :key_scope + AND address LIKE '{ua_prefix}%' + AND exposed_at_height IS NOT NULL + ORDER BY diversifier_index_be DESC + LIMIT 1" + ), + named_params![ + ":account_uuid": account_uuid.0, + ":key_scope": KeyScope::EXTERNAL.encode() + ], |row| Ok((row.get(0)?, row.get(1)?)), ) .optional()?; addr.map(|(addr_str, di_vec)| { - let mut di_be: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di_be.reverse(); - + let diversifier_index = decode_diversifier_index_be(&di_vec)?; Address::decode(params, &addr_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) @@ -656,47 +751,99 @@ pub(crate) fn get_current_address( addr_str, ))), }) - .map(|addr| (addr, DiversifierIndex::from(di_be))) + .map(|addr| (addr, diversifier_index)) }) .transpose() } -/// Adds the given address and diversifier index to the addresses table. +/// Adds the given external address and diversifier index to the addresses table. /// -/// Returns the database row for the newly-inserted address. -pub(crate) fn insert_address( +/// Returns the primary key identifier for the newly-inserted address. +pub(crate) fn upsert_address( conn: &rusqlite::Connection, params: &P, account_id: AccountRef, diversifier_index: DiversifierIndex, address: &UnifiedAddress, -) -> Result<(), SqliteClientError> { + exposed_at_height: Option, +) -> Result { let mut stmt = conn.prepare_cached( "INSERT INTO addresses ( account_id, diversifier_index_be, + key_scope, address, - cached_transparent_receiver_address + transparent_child_index, + cached_transparent_receiver_address, + exposed_at_height ) VALUES ( :account_id, :diversifier_index_be, + :key_scope, :address, - :cached_transparent_receiver_address - )", + :transparent_child_index, + :cached_transparent_receiver_address, + :exposed_at_height + ) + ON CONFLICT (account_id, diversifier_index_be, key_scope) DO UPDATE + SET exposed_at_height = :exposed_at_height + RETURNING id", )?; - // the diversifier index is stored in big-endian order to allow sorting - let mut di_be = *diversifier_index.as_bytes(); - di_be.reverse(); - stmt.execute(named_params![ - ":account_id": account_id.0, - ":diversifier_index_be": &di_be[..], - ":address": &address.encode(params), - ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), - ])?; + #[cfg(feature = "transparent-inputs")] + let transparent_child_index = NonHardenedChildIndex::try_from(diversifier_index) + .ok() + .map(|i| i.index()); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_child_index: Option = None; - Ok(()) + stmt.query_row( + named_params![ + ":account_id": account_id.0, + // the diversifier index is stored in big-endian order to allow sorting + ":diversifier_index_be": encode_diversifier_index_be(diversifier_index), + ":key_scope": KeyScope::EXTERNAL.encode(), + ":address": &address.encode(params), + ":transparent_child_index": transparent_child_index, + ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), + ":exposed_at_height": exposed_at_height.map(u32::from) + ], + |row| row.get(0).map(AddressRef) + ).map_err(SqliteClientError::from) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn involved_accounts( + conn: &rusqlite::Connection, + tx_refs: impl IntoIterator, +) -> Result, SqliteClientError> { + use rusqlite::types::Value; + use std::rc::Rc; + + let mut stmt = conn.prepare_cached( + "SELECT account_id, key_scope + FROM v_address_uses + WHERE transaction_id IN rarray(:tx_refs_ptr)", + )?; + + let tx_refs_values: Vec = tx_refs.into_iter().map(|r| Value::Integer(r.0)).collect(); + let tx_refs_ptr = Rc::new(tx_refs_values); + let result = stmt + .query_and_then( + named_params! { + ":tx_refs_ptr": &tx_refs_ptr + }, + |row| { + Ok::<_, SqliteClientError>(( + row.get(0).map(AccountRef)?, + KeyScope::decode(row.get(1)?)?, + )) + }, + )? + .collect::, _>>()?; + + Ok(result) } /// Returns the [`UnifiedFullViewingKey`]s for the wallet. @@ -732,6 +879,7 @@ fn parse_account_row( row: &rusqlite::Row<'_>, params: &P, ) -> Result { + let account_id = AccountRef(row.get("id")?); let account_name = row.get("name")?; let account_uuid = AccountUuid(row.get("uuid")?); let kind = parse_account_source( @@ -765,6 +913,7 @@ fn parse_account_row( }; Ok(Account { + id: account_id, name: account_name, uuid: account_uuid, kind, @@ -779,7 +928,7 @@ pub(crate) fn get_account( ) -> Result, SqliteClientError> { let mut stmt = conn.prepare_cached( r#" - SELECT name, uuid, account_kind, + SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -795,6 +944,30 @@ pub(crate) fn get_account( rows.next().transpose() } +#[cfg(feature = "transparent-inputs")] +pub(crate) fn get_account_internal( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountRef, +) -> Result, SqliteClientError> { + let mut stmt = conn.prepare_cached( + r#" + SELECT id, name, uuid, account_kind, + hd_seed_fingerprint, hd_account_index, key_source, + ufvk, uivk, has_spend_key + FROM accounts + WHERE id = :account_id + "#, + )?; + + let mut rows = stmt.query_and_then::<_, SqliteClientError, _, _>( + named_params![":account_id": account_id.0], + |row| parse_account_row(row, params), + )?; + + rows.next().transpose() +} + /// Returns the account id corresponding to a given [`UnifiedFullViewingKey`], /// if any. pub(crate) fn get_account_for_ufvk( @@ -815,7 +988,7 @@ pub(crate) fn get_account_for_ufvk( let transparent_item: Option> = None; let mut stmt = conn.prepare( - "SELECT name, uuid, account_kind, + "SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -853,7 +1026,7 @@ pub(crate) fn get_derived_account( account_index: zip32::AccountId, ) -> Result, SqliteClientError> { let mut stmt = conn.prepare( - "SELECT name, key_source, uuid, ufvk + "SELECT id, name, key_source, uuid, ufvk FROM accounts WHERE hd_seed_fingerprint = :hd_seed_fingerprint AND hd_account_index = :hd_account_index", @@ -865,6 +1038,7 @@ pub(crate) fn get_derived_account( ":hd_account_index": u32::from(account_index), ], |row| { + let account_id = AccountRef(row.get("id")?); let account_name = row.get("name")?; let key_source = row.get("key_source")?; let account_uuid = AccountUuid(row.get("uuid")?); @@ -881,6 +1055,7 @@ pub(crate) fn get_derived_account( }), }?; Ok(Account { + id: account_id, name: account_name, uuid: account_uuid, kind: AccountSource::Derived { @@ -1933,20 +2108,6 @@ pub(crate) fn get_account_ref( .ok_or(SqliteClientError::AccountUnknown) } -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_account_uuid( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - conn.query_row( - "SELECT uuid FROM accounts WHERE id = :account_id", - named_params! {":account_id": account_id.0}, - |row| row.get("uuid").map(AccountUuid), - ) - .optional()? - .ok_or(SqliteClientError::AccountUnknown) -} - /// Returns the minimum and maximum heights of blocks in the chain which may be scanned. pub(crate) fn chain_tip_height( conn: &rusqlite::Connection, @@ -2262,6 +2423,13 @@ pub(crate) fn store_transaction_to_be_sent( )?; match output.recipient() { + Recipient::External { + recipient_address: _addr, + output_pool: _pool, + .. + } => { + // Nothing to do for external recipients. + } Recipient::InternalAccount { receiving_account, note, @@ -2270,6 +2438,7 @@ pub(crate) fn store_transaction_to_be_sent( Note::Sapling(note) => { sapling::put_received_note( wdb.conn.0, + &wdb.params, &DecryptedOutput::new( output.output_index(), note.clone(), @@ -2280,6 +2449,7 @@ pub(crate) fn store_transaction_to_be_sent( TransferType::WalletInternal, ), tx_ref, + Some(sent_tx.target_height()), None, )?; } @@ -2287,6 +2457,7 @@ pub(crate) fn store_transaction_to_be_sent( Note::Orchard(note) => { orchard::put_received_note( wdb.conn.0, + &wdb.params, &DecryptedOutput::new( output.output_index(), *note, @@ -2297,16 +2468,25 @@ pub(crate) fn store_transaction_to_be_sent( TransferType::WalletInternal, ), tx_ref, + Some(sent_tx.target_height()), None, )?; } }, #[cfg(feature = "transparent-inputs")] Recipient::EphemeralTransparent { - receiving_account, ephemeral_address, outpoint, + .. } => { + // First check to verify that creation of this output does not result in reuse of + // an ephemeral address. + transparent::check_address_reuse( + wdb.conn.0, + &wdb.params, + &Address::Transparent(*ephemeral_address), + )?; + transparent::put_transparent_output( wdb.conn.0, &wdb.params, @@ -2317,17 +2497,9 @@ pub(crate) fn store_transaction_to_be_sent( }, None, ephemeral_address, - *receiving_account, true, )?; - transparent::ephemeral::mark_ephemeral_address_as_used( - wdb.conn.0, - &wdb.params, - ephemeral_address, - tx_ref, - )?; } - _ => {} } } @@ -2536,6 +2708,8 @@ pub(crate) fn truncate_to_height( let mut wdb = WalletDb { conn: SqlTransaction(conn), params: params.clone(), + #[cfg(feature = "transparent-inputs")] + gap_limits: GapLimits::default(), }; wdb.with_sapling_tree_mut(|tree| { tree.truncate_to_checkpoint(&truncation_height)?; @@ -2693,6 +2867,7 @@ pub(crate) fn store_decrypted_tx( conn: &rusqlite::Transaction, params: &P, d_tx: DecryptedTransaction, + #[cfg(feature = "transparent-inputs")] gap_limits: &GapLimits, ) -> Result<(), SqliteClientError> { let tx_ref = put_tx_data(conn, d_tx.tx(), None, None, None)?; if let Some(height) = d_tx.mined_height() { @@ -2717,6 +2892,9 @@ pub(crate) fn store_decrypted_tx( #[cfg(feature = "transparent-inputs")] let mut tx_has_wallet_outputs = false; + #[cfg(feature = "transparent-inputs")] + let mut receiving_accounts = BTreeMap::new(); + for output in d_tx.sapling_outputs() { #[cfg(feature = "transparent-inputs")] { @@ -2748,7 +2926,14 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::WalletInternal => { - sapling::put_received_note(conn, output, tx_ref, None)?; + sapling::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; let recipient = Recipient::InternalAccount { receiving_account: *output.account(), @@ -2768,7 +2953,17 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::Incoming => { - sapling::put_received_note(conn, output, tx_ref, None)?; + let _account_id = sapling::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; + + #[cfg(feature = "transparent-inputs")] + receiving_accounts.insert(_account_id, KeyScope::EXTERNAL); if let Some(account_id) = funding_account { let recipient = Recipient::InternalAccount { @@ -2837,7 +3032,14 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::WalletInternal => { - orchard::put_received_note(conn, output, tx_ref, None)?; + orchard::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; let recipient = Recipient::InternalAccount { receiving_account: *output.account(), @@ -2857,7 +3059,17 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::Incoming => { - orchard::put_received_note(conn, output, tx_ref, None)?; + let _account_id = orchard::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; + + #[cfg(feature = "transparent-inputs")] + receiving_accounts.insert(_account_id, KeyScope::EXTERNAL); if let Some(account_id) = funding_account { // Even if the recipient address is external, record the send as internal. @@ -2923,18 +3135,9 @@ pub(crate) fn store_decrypted_tx( address.encode(params) ); - // The transaction is not necessarily mined yet, but we want to record - // that an output to the address was seen in this tx anyway. This will - // advance the gap regardless of whether it is mined, but an output in - // an unmined transaction won't advance the range of safe indices. - #[cfg(feature = "transparent-inputs")] - transparent::ephemeral::mark_ephemeral_address_as_seen( - conn, params, &address, tx_ref, - )?; - // If the output belongs to the wallet, add it to `transparent_received_outputs`. #[cfg(feature = "transparent-inputs")] - if let Some(account_uuid) = + if let Some((account_uuid, key_scope)) = transparent::find_account_uuid_for_transparent_address(conn, params, &address)? { debug!( @@ -2943,7 +3146,7 @@ pub(crate) fn store_decrypted_tx( output_index, account_uuid ); - transparent::put_transparent_output( + let (account_id, _, _) = transparent::put_transparent_output( conn, params, &OutPoint::new( @@ -2953,10 +3156,11 @@ pub(crate) fn store_decrypted_tx( txout, d_tx.mined_height(), &address, - account_uuid, false, )?; + receiving_accounts.insert(account_id, key_scope); + // Since the wallet created the transparent output, we need to ensure // that any transparent inputs belonging to the wallet will be // discovered. @@ -3028,6 +3232,11 @@ pub(crate) fn store_decrypted_tx( } } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in receiving_accounts { + transparent::generate_gap_addresses(conn, params, account_id, key_scope, gap_limits, None)?; + } + // If the transaction has outputs that belong to the wallet as well as transparent // inputs, we may need to download the transactions corresponding to the transparent // prevout references to determine whether the transaction was created (at least in @@ -3106,10 +3315,14 @@ pub(crate) fn select_receiving_address( "SELECT address FROM addresses JOIN accounts ON accounts.id = addresses.account_id - WHERE accounts.uuid = :account_uuid", + WHERE accounts.uuid = :account_uuid + AND key_scope = :key_scope", )?; - let mut result = stmt.query(named_params! { ":account_uuid": account.0 })?; + let mut result = stmt.query(named_params! { + ":account_uuid": account.0, + ":key_scope": KeyScope::EXTERNAL.encode(), + })?; while let Some(row) = result.next()? { let addr_str = row.get::<_, String>(0)?; let decoded = addr_str.parse::()?; @@ -3360,7 +3573,7 @@ fn flag_previously_received_change( ), named_params! { ":tx": tx_ref.0, - ":internal_scope": scope_code(Scope::Internal) + ":internal_scope": KeyScope::INTERNAL.encode() }, ) }; diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 6520ac183..88a80fd89 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -13,9 +13,7 @@ // from showing up in `cargo doc --document-private-items`. #![allow(dead_code)] -use static_assertions::const_assert_eq; - -use zcash_client_backend::data_api::{scanning::ScanPriority, GAP_LIMIT}; +use zcash_client_backend::data_api::scanning::ScanPriority; use zcash_protocol::consensus::{NetworkUpgrade, Parameters}; use crate::wallet::scanning::priority_code; @@ -63,85 +61,52 @@ pub(super) const INDEX_ACCOUNTS_UIVK: &str = pub(super) const INDEX_HD_ACCOUNT: &str = r#"CREATE UNIQUE INDEX hd_account ON accounts (hd_seed_fingerprint, hd_account_index)"#; -/// Stores diversified Unified Addresses that have been generated from accounts in the -/// wallet. +/// Stores addresses that have been generated from accounts in the wallet. +/// +/// ### Columns /// -/// - The `cached_transparent_receiver_address` column contains the transparent receiver component -/// of the UA. It is cached directly in the table to make account lookups for transparent outputs -/// more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. +/// - `account_id`: the account whose IVK was used to derive this address. +/// - `diversifier_index_be`: the diversifier index at which this address was derived. +/// - `key_scope`: the key scope for which this address was derived. +/// - `address`: The Unified, Sapling, or transparent address. For Unified and Sapling addresses, +/// only external-key scoped addresses should be stored in this table; for purely transparent +/// addresses, this may be an internal-scope (change) address, so that we can provide +/// compatibility with HD-derived change addresses produced by transparent-only wallets. +/// - `transparent_child_index`: the diversifier index, if it is in the range of a non-hardened +/// transparent address index. This is used for gap limit handling and is always populated if the +/// diversifier index is in that range; since the diversifier index is stored as a byte array we +/// cannot use SQL integer operations on it and thus need it as an integer as well. +/// - `cached_transparent_receiver_address`: the transparent receiver component of address (which +/// may be the same as `address` in the case of an internal-scope transparent change address or a +/// ZIP 320 interstitial address). It is cached directly in the table to make account lookups for +/// transparent outputs more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. pub(super) const TABLE_ADDRESSES: &str = r#" CREATE TABLE "addresses" ( - account_id INTEGER NOT NULL, + id INTEGER NOT NULL PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES accounts(id), diversifier_index_be BLOB NOT NULL, + key_scope INTEGER NOT NULL DEFAULT 0, address TEXT NOT NULL, + transparent_child_index INTEGER, cached_transparent_receiver_address TEXT, - FOREIGN KEY (account_id) REFERENCES accounts(id), - CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be) + exposed_at_height INTEGER, + CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be, key_scope), + CONSTRAINT transparent_index_consistency CHECK ( + (transparent_child_index IS NOT NULL) == (cached_transparent_receiver_address IS NOT NULL) + ) )"#; pub(super) const INDEX_ADDRESSES_ACCOUNTS: &str = r#" -CREATE INDEX "addresses_accounts" ON "addresses" ( - "account_id" ASC +CREATE INDEX idx_addresses_accounts ON addresses ( + account_id ASC +)"#; +pub(super) const INDEX_ADDRESSES_INDICES: &str = r#" +CREATE INDEX idx_addresses_indices ON addresses ( + diversifier_index_be ASC +)"#; +pub(super) const INDEX_ADDRESSES_T_INDICES: &str = r#" +CREATE INDEX idx_addresses_t_indices ON addresses ( + transparent_child_index ASC )"#; - -/// Stores ephemeral transparent addresses used for ZIP 320. -/// -/// For each account, these addresses are allocated sequentially by address index under scope 2 -/// (`TransparentKeyScope::EPHEMERAL`) at the "change" level of the BIP 32 address hierarchy. -/// The ephemeral addresses stored in the table are exactly the "reserved" ephemeral addresses -/// (that is addresses that have been allocated for use in a ZIP 320 transaction proposal), plus -/// the addresses at the next [`GAP_LIMIT`] indices. -/// -/// Addresses are never removed. New ones should only be reserved via the -/// `WalletWrite::reserve_next_n_ephemeral_addresses` API. All of the addresses in the table -/// should be scanned for incoming funds. -/// -/// ### Columns -/// - `address` contains the string (Base58Check) encoding of a transparent P2PKH address. -/// - `used_in_tx` indicates that the address has been used by this wallet in a transaction (which -/// has not necessarily been mined yet). This should only be set once, when the txid is known. -/// - `seen_in_tx` is non-null iff an output to the address has been seed in a transaction observed -/// on the network and passed to `store_decrypted_tx`. The transaction may have been sent by this -// wallet or another one using the same seed, or by a TEX address recipient sending back the -/// funds. This is used to advance the "gap", as well as to heuristically reduce the chance of -/// address reuse collisions with another wallet using the same seed. -/// -/// It is an external invariant that within each account: -/// - the address indices are contiguous and start from 0; -/// - the last [`GAP_LIMIT`] addresses have `used_in_tx` and `seen_in_tx` both NULL. -/// -/// All but the last [`GAP_LIMIT`] addresses are defined to be "reserved" addresses. Since the next -/// index to reserve is determined by dead reckoning from the last stored address, we use dummy -/// entries having `NULL` for the value of the `address` column after the maximum valid index in -/// order to allow the last [`GAP_LIMIT`] addresses at the end of the index range to be used. -/// -/// Note that the fact that `used_in_tx` references a specific transaction is just a debugging aid. -/// The same is mostly true of `seen_in_tx`, but we also take into account whether the referenced -/// transaction is unmined in order to determine the last index that is safe to reserve. -pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" -CREATE TABLE ephemeral_addresses ( - account_id INTEGER NOT NULL, - address_index INTEGER NOT NULL, - -- nullability of this column is controlled by the index_range_and_address_nullity check - address TEXT, - used_in_tx INTEGER, - seen_in_tx INTEGER, - FOREIGN KEY (account_id) REFERENCES accounts(id), - FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), - FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), - PRIMARY KEY (account_id, address_index), - CONSTRAINT ephemeral_addr_uniq UNIQUE (address), - CONSTRAINT used_implies_seen CHECK ( - used_in_tx IS NULL OR seen_in_tx IS NOT NULL - ), - CONSTRAINT index_range_and_address_nullity CHECK ( - (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR - (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND seen_in_tx IS NULL) - ) -) WITHOUT ROWID"#; -// Hexadecimal integer literals were added in SQLite version 3.8.6 (2014-08-15). -// libsqlite3-sys requires at least version 3.14.0. -// "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. -const_assert_eq!(GAP_LIMIT, 20); /// Stores information about every block that the wallet has scanned. /// @@ -216,6 +181,7 @@ CREATE TABLE "sapling_received_notes" ( memo BLOB, commitment_tree_position INTEGER, recipient_key_scope INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT tx_output UNIQUE (tx, output_index) @@ -269,6 +235,7 @@ CREATE TABLE orchard_received_notes ( memo BLOB, commitment_tree_position INTEGER, recipient_key_scope INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT tx_output UNIQUE (tx, action_index) @@ -339,6 +306,7 @@ CREATE TABLE transparent_received_outputs ( script BLOB NOT NULL, value_zat INTEGER NOT NULL, max_observed_unspent_height INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) @@ -690,7 +658,8 @@ CREATE VIEW v_received_outputs AS sapling_received_notes.value, is_change, sapling_received_notes.memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + sapling_received_notes.address_id FROM sapling_received_notes LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -705,7 +674,8 @@ UNION orchard_received_notes.value, is_change, orchard_received_notes.memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + orchard_received_notes.address_id FROM orchard_received_notes LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -720,7 +690,8 @@ UNION u.value_zat AS value, 0 AS is_change, NULL AS memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + u.address_id FROM transparent_received_outputs u LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -1072,3 +1043,37 @@ GROUP BY subtree_start_height, subtree_end_height, contains_marked"; + +pub(super) const VIEW_ADDRESS_USES: &str = " +CREATE VIEW v_address_uses AS + SELECT orn.address_id, orn.account_id, orn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM orchard_received_notes orn + JOIN addresses a ON a.id = orn.address_id + JOIN transactions t ON t.id_tx = orn.tx +UNION + SELECT srn.address_id, srn.account_id, srn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM sapling_received_notes srn + JOIN addresses a ON a.id = srn.address_id + JOIN transactions t ON t.id_tx = srn.tx +UNION + SELECT tro.address_id, tro.account_id, tro.transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM transparent_received_outputs tro + JOIN addresses a ON a.id = tro.address_id + JOIN transactions t ON t.id_tx = tro.transaction_id"; + +pub(super) const VIEW_ADDRESS_FIRST_USE: &str = " + CREATE VIEW v_address_first_use AS + SELECT + address_id, + account_id, + key_scope, + diversifier_index_be, + transparent_child_index, + MIN(mined_height) AS first_use_height + FROM v_address_uses + GROUP BY + address_id, account_id, key_scope, + diversifier_index_be, transparent_child_index"; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index d9d25c2b9..77ab73348 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -220,12 +220,11 @@ fn sqlite_client_error_to_wallet_migration_error(e: SqliteClientError) -> Wallet unreachable!("we don't call methods that require a known chain height") } #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit(_, _) => { + SqliteClientError::ReachedGapLimit(_) => { unreachable!("we don't do ephemeral address tracking") } - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(_, _) => { - unreachable!("we don't do ephemeral address tracking") + SqliteClientError::AddressReuse(_, _) => { + unreachable!("we don't create transactions in migrations") } SqliteClientError::NoteFilterInvalid(_) => { unreachable!("we don't do note selection in migrations") @@ -487,7 +486,6 @@ mod tests { db::TABLE_ACCOUNTS, db::TABLE_ADDRESSES, db::TABLE_BLOCKS, - db::TABLE_EPHEMERAL_ADDRESSES, db::TABLE_NULLIFIER_MAP, db::TABLE_ORCHARD_RECEIVED_NOTE_SPENDS, db::TABLE_ORCHARD_RECEIVED_NOTES, @@ -529,6 +527,8 @@ mod tests { db::INDEX_ACCOUNTS_UUID, db::INDEX_HD_ACCOUNT, db::INDEX_ADDRESSES_ACCOUNTS, + db::INDEX_ADDRESSES_INDICES, + db::INDEX_ADDRESSES_T_INDICES, db::INDEX_NF_MAP_LOCATOR_IDX, db::INDEX_ORCHARD_RECEIVED_NOTES_ACCOUNT, db::INDEX_ORCHARD_RECEIVED_NOTES_TX, @@ -557,6 +557,8 @@ mod tests { } let expected_views = vec![ + db::VIEW_ADDRESS_FIRST_USE.to_owned(), + db::VIEW_ADDRESS_USES.to_owned(), db::view_orchard_shard_scan_ranges(st.network()), db::view_orchard_shard_unscanned_ranges(), db::VIEW_ORCHARD_SHARDS_SCAN_STATE.to_owned(), @@ -1079,6 +1081,11 @@ mod tests { let (account_id, _usk) = db_data .create_account("", &Secret::new(seed.to_vec()), &birthday, None) .unwrap(); + + // We have to have the chain tip height in order to allocate new addresses, to record the + // exposed-at height. + db_data.update_chain_tip(birthday.height()).unwrap(); + assert_matches!( db_data.get_account(account_id), Ok(Some(account)) if matches!( diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1e076736e..8aa056ced 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -19,6 +19,7 @@ mod sent_notes_to_internal; mod shardtree_support; mod spend_key_available; mod support_legacy_sqlite; +mod transparent_gap_limit_handling; mod tx_retrieval_queue; mod ufvk_support; mod utxos_table; @@ -84,8 +85,8 @@ pub(super) fn all_migrations( // support_legacy_sqlite // / \ // fix_broken_commitment_trees add_account_uuids - // | - // fix_bad_change_flagging + // | | + // fix_bad_change_flagging transparent_gap_limit_handling vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -149,6 +150,9 @@ pub(super) fn all_migrations( }), Box::new(fix_bad_change_flagging::Migration), Box::new(add_account_uuids::Migration), + Box::new(transparent_gap_limit_handling::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs index 6c737247d..213ec6e6e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs @@ -19,7 +19,7 @@ use { std::collections::HashMap, zcash_client_backend::wallet::TransparentAddressMetadata, zcash_keys::{address::Address, encoding::AddressCodec, keys::UnifiedFullViewingKey}, - zip32::{AccountId, DiversifierIndex, Scope}, + zip32::{AccountId, Scope}, }; /// This migration adds an account identifier column to the UTXOs table. @@ -132,6 +132,8 @@ fn get_transparent_receivers( params: &P, account: AccountId, ) -> Result>, SqliteClientError> { + use crate::wallet::decode_diversifier_index_be; + let mut ret: HashMap> = HashMap::new(); // Get all UAs derived @@ -141,11 +143,7 @@ fn get_transparent_receivers( while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; - let di_vec: Vec = row.get(1)?; - let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion + let di = decode_diversifier_index_be(&row.get::<_, Vec>(1)?)?; let ua = Address::decode(params, &ua_str) .ok_or_else(|| { @@ -160,13 +158,11 @@ fn get_transparent_receivers( })?; if let Some(taddr) = ua.transparent() { - let index = NonHardenedChildIndex::from_index( - DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_owned(), - ) - })?, - ) + let index = NonHardenedChildIndex::from_index(u32::try_from(di).map_err(|_| { + SqliteClientError::CorruptedData( + "Unable to get diversifier for transparent address.".to_owned(), + ) + })?) .ok_or_else(|| { SqliteClientError::CorruptedData( "Unexpected hardened index for transparent address.".to_owned(), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 109f307ff..c752184d4 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -7,13 +7,25 @@ use zcash_protocol::consensus; use crate::wallet::init::WalletMigrationError; -#[cfg(feature = "transparent-inputs")] -use crate::{wallet::transparent::ephemeral, AccountRef}; - use super::utxos_to_txos; +#[cfg(feature = "transparent-inputs")] +use { + crate::{error::SqliteClientError, AccountRef}, + rusqlite::named_params, + transparent::keys::NonHardenedChildIndex, + zcash_keys::{ + encoding::AddressCodec, + keys::{AddressGenerationError, UnifiedFullViewingKey}, + }, + zip32::DiversifierIndex, +}; + pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0e1d4274_1f8e_44e2_909d_689a4bc2967b); +#[cfg(feature = "transparent-inputs")] +const EPHEMERAL_GAP_LIMIT: u32 = 5; + const DEPENDENCIES: &[Uuid] = &[utxos_to_txos::MIGRATION_ID]; #[allow(dead_code)] @@ -35,6 +47,60 @@ impl

schemerz::Migration for Migration

{ } } +#[cfg(feature = "transparent-inputs")] +fn init_accounts( + transaction: &rusqlite::Transaction, + params: &P, +) -> Result<(), SqliteClientError> { + let mut stmt = transaction.prepare("SELECT id, ufvk FROM accounts")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let account_id = AccountRef(row.get(0)?); + let ufvk_str: Option = row.get(1)?; + if let Some(ufvk_str) = ufvk_str { + if let Some(tfvk) = UnifiedFullViewingKey::decode(params, &ufvk_str) + .map_err(SqliteClientError::CorruptedData)? + .transparent() + { + let ephemeral_ivk = tfvk.derive_ephemeral_ivk().map_err(|_| { + SqliteClientError::CorruptedData( + "Unexpected failure to derive ephemeral transparent IVK".to_owned(), + ) + })?; + + let mut ea_insert = transaction.prepare( + "INSERT INTO ephemeral_addresses (account_id, address_index, address) + VALUES (:account_id, :address_index, :address)", + )?; + + // NB: we have reduced the initial space of generated ephemeral addresses + // from 20 addresses to 5, as ephemeral addresses should always be used in + // a transaction immediatly after being reserved, and as a consequence + // there is no significant benefit in having a larger gap limit. + for i in 0..EPHEMERAL_GAP_LIMIT { + let address = ephemeral_ivk + .derive_ephemeral_address( + NonHardenedChildIndex::from_index(i).expect("index is valid"), + ) + .map_err(|_| { + AddressGenerationError::InvalidTransparentChildIndex( + DiversifierIndex::from(i), + ) + })?; + + ea_insert.execute(named_params! { + ":account_id": account_id.0, + ":address_index": i, + ":address": address.encode(params) + })?; + } + } + } + } + + Ok(()) +} + impl RusqliteMigration for Migration

{ type Error = WalletMigrationError; @@ -62,17 +128,13 @@ impl RusqliteMigration for Migration

{ ) WITHOUT ROWID;" )?; - // Make sure that at least `GAP_LIMIT` ephemeral transparent addresses are + // Make sure that at least `EPHEMERAL_GAP_LIMIT` ephemeral transparent addresses are // stored in each account. #[cfg(feature = "transparent-inputs")] { - let mut stmt = transaction.prepare("SELECT id FROM accounts")?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let account_id = AccountRef(row.get(0)?); - ephemeral::init_account(transaction, &self.params, account_id)?; - } + init_accounts(transaction, &self.params)?; } + Ok(()) } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs b/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs index 38b1d49ae..e25e0711e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs @@ -5,12 +5,11 @@ use std::collections::HashSet; use rusqlite::named_params; use schemerz_rusqlite::RusqliteMigration; use uuid::Uuid; -use zip32::Scope; use crate::{ wallet::{ init::{migrations::fix_broken_commitment_trees, WalletMigrationError}, - scope_code, + KeyScope, }, SAPLING_TABLES_PREFIX, }; @@ -52,7 +51,7 @@ impl RusqliteMigration for Migration { AND sn.from_account_id = {table_prefix}_received_notes.account_id AND {table_prefix}_received_notes.recipient_key_scope = :internal_scope" ), - named_params! {":internal_scope": scope_code(Scope::Internal)}, + named_params! {":internal_scope": KeyScope::INTERNAL.encode()}, ) }; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index 1e71ac501..d9ce72d67 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -29,7 +29,7 @@ use crate::{ chain_tip_height, commitment_tree::SqliteShardStore, init::{migrations::shardtree_support, WalletMigrationError}, - scope_code, + KeyScope, }, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; @@ -109,7 +109,7 @@ impl RusqliteMigration for Migration

{ transaction.execute_batch( &format!( "ALTER TABLE sapling_received_notes ADD COLUMN recipient_key_scope INTEGER NOT NULL DEFAULT {};", - scope_code(Scope::External) + KeyScope::EXTERNAL.encode() ) )?; @@ -204,7 +204,7 @@ impl RusqliteMigration for Migration

{ transaction.execute( "UPDATE sapling_received_notes SET recipient_key_scope = :scope WHERE id_note = :note_id", - named_params! {":scope": scope_code(Scope::Internal), ":note_id": note_id}, + named_params! {":scope": KeyScope::INTERNAL.encode(), ":note_id": note_id}, )?; } } else { @@ -261,7 +261,7 @@ impl RusqliteMigration for Migration

{ transaction.execute( "UPDATE sapling_received_notes SET recipient_key_scope = :scope WHERE id_note = :note_id", - named_params! {":scope": scope_code(Scope::Internal), ":note_id": note_id}, + named_params! {":scope": KeyScope::INTERNAL.encode(), ":note_id": note_id}, )?; } } @@ -326,8 +326,9 @@ mod tests { init_wallet_db_internal, migrations::{add_account_birthdays, shardtree_support, wallet_summaries}, }, - memo_repr, parse_scope, + memo_repr, sapling::ReceivedSaplingOutput, + KeyScope, }, AccountRef, TxRef, WalletDb, }; @@ -604,10 +605,10 @@ mod tests { while let Some(row) = rows.next().unwrap() { row_count += 1; let value: u64 = row.get(0).unwrap(); - let scope = parse_scope(row.get(1).unwrap()); + let scope = KeyScope::decode(row.get(1).unwrap()).unwrap(); match value { - EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), - INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), + EXTERNAL_VALUE => assert_eq!(scope, KeyScope::EXTERNAL), + INTERNAL_VALUE => assert_eq!(scope, KeyScope::INTERNAL), _ => { panic!( "(Value, Scope) pair {:?} is not expected to exist in the wallet.", @@ -782,10 +783,10 @@ mod tests { while let Some(row) = rows.next().unwrap() { row_count += 1; let value: u64 = row.get(0).unwrap(); - let scope = parse_scope(row.get(1).unwrap()); + let scope = KeyScope::decode(row.get(1).unwrap()).unwrap(); match value { - EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), - INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), + EXTERNAL_VALUE => assert_eq!(scope, KeyScope::EXTERNAL), + INTERNAL_VALUE => assert_eq!(scope, KeyScope::INTERNAL), _ => { panic!( "(Value, Scope) pair {:?} is not expected to exist in the wallet.", diff --git a/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs b/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs new file mode 100644 index 000000000..d41409e94 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs @@ -0,0 +1,491 @@ +//! Add support for general transparent gap limit handling, and unify the `addresses` and +//! `ephemeral_addresses` tables. + +use std::collections::HashSet; +use uuid::Uuid; + +use rusqlite::{named_params, Transaction}; +use schemerz_rusqlite::RusqliteMigration; + +use zcash_keys::keys::UnifiedIncomingViewingKey; +use zcash_protocol::consensus::{self, BlockHeight}; + +use super::add_account_uuids; +use crate::{ + wallet::{self, init::WalletMigrationError, KeyScope}, + AccountRef, +}; + +#[cfg(feature = "transparent-inputs")] +use { + crate::wallet::{decode_diversifier_index_be, encode_diversifier_index_be}, + ::transparent::keys::{IncomingViewingKey as _, NonHardenedChildIndex}, + zcash_keys::encoding::AddressCodec as _, + zip32::DiversifierIndex, +}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xc41dfc0e_e870_4859_be47_d2f572f5ca73); + +const DEPENDENCIES: &[Uuid] = &[add_account_uuids::MIGRATION_ID]; + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemerz::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + DEPENDENCIES.iter().copied().collect() + } + + fn description(&self) -> &'static str { + "Add support for general transparent gap limit handling, unifying the `addresses` and `ephemeral_addresses` tables." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, conn: &Transaction) -> Result<(), WalletMigrationError> { + let decode_uivk = |uivk_str: String| { + UnifiedIncomingViewingKey::decode(&self.params, &uivk_str).map_err(|e| { + WalletMigrationError::CorruptedData(format!( + "Invalid UIVK encoding {}: {}", + uivk_str, e + )) + }) + }; + + let external_scope_code = KeyScope::EXTERNAL.encode(); + + conn.execute_batch(&format!( + r#" + ALTER TABLE addresses ADD COLUMN key_scope INTEGER NOT NULL DEFAULT {external_scope_code}; + ALTER TABLE addresses ADD COLUMN transparent_child_index INTEGER; + "# + ))?; + + #[cfg(feature = "transparent-inputs")] + { + // If the diversifier index is in the valid range of non-hardened child indices, set + // `transparent_child_index` so that we can use it for gap limit handling. + let mut di_query = conn.prepare( + r#" + SELECT account_id, accounts.uivk AS uivk, diversifier_index_be + FROM addresses + JOIN accounts ON accounts.id = account_id + GROUP BY account_id, uivk, diversifier_index_be + "#, + )?; + let mut rows = di_query.query([])?; + while let Some(row) = rows.next()? { + let account_id: i64 = row.get("account_id")?; + let uivk = decode_uivk(row.get("uivk")?)?; + let di_be: Vec = row.get("diversifier_index_be")?; + let diversifier_index = decode_diversifier_index_be(&di_be)?; + + let transparent_external = NonHardenedChildIndex::try_from(diversifier_index) + .ok() + .and_then(|idx| { + uivk.transparent() + .as_ref() + .and_then(|external_ivk| external_ivk.derive_address(idx).ok()) + .map(|t_addr| (idx, t_addr)) + }); + + // Add transparent address index metadata and the transparent address corresponding + // to the index to the addresses table. We unconditionally set the cached + // transparent receiver address in order to simplify gap limit handling; even if a + // unified address is generated without a transparent receiver, we still assume + // that a transparent-only wallet for which we have imported the seed may have + // generated an address at that index. + if let Some((idx, t_addr)) = transparent_external { + conn.execute( + r#" + UPDATE addresses + SET transparent_child_index = :transparent_child_index, + cached_transparent_receiver_address = :t_addr + WHERE account_id = :account_id + AND diversifier_index_be = :diversifier_index_be + AND key_scope = :external_scope_code + "#, + named_params! { + ":account_id": account_id, + ":diversifier_index_be": &di_be[..], + ":external_scope_code": external_scope_code, + ":transparent_child_index": idx.index(), + ":t_addr": t_addr.encode(&self.params), + }, + )?; + } + } + } + + // We now have to re-create the `addresses` table in order to fix the constraints. + // Note that we do not include `used_in_tx` or `seen_in_tx` columns as these are + // duplicative of information that can be discovered via joins with the various + // `*_received_{notes|outputs}` tables, which we will create a view to perform below. + conn.execute_batch(&format!( + r#" + CREATE TABLE addresses_new ( + id INTEGER NOT NULL PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES accounts(id), + diversifier_index_be BLOB NOT NULL, + key_scope INTEGER NOT NULL DEFAULT {external_scope_code}, + address TEXT NOT NULL, + transparent_child_index INTEGER, + cached_transparent_receiver_address TEXT, + exposed_at_height INTEGER, + CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be, key_scope), + CONSTRAINT transparent_index_consistency CHECK ( + (transparent_child_index IS NOT NULL) == (cached_transparent_receiver_address IS NOT NULL) + ) + ); + + INSERT INTO addresses_new ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) + SELECT + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + FROM addresses; + "# + ))?; + + // Now, we add the ephemeral addresses to the newly unified `addresses` table. + #[cfg(feature = "transparent-inputs")] + { + let mut ea_insert = conn.prepare( + r#" + INSERT INTO addresses_new ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) VALUES ( + :account_id, :diversifier_index_be, :key_scope, :address, + :transparent_child_index, :cached_transparent_receiver_address + ) + "#, + )?; + + let mut ea_query = conn.prepare( + r#" + SELECT account_id, address_index, address + FROM ephemeral_addresses + "#, + )?; + let mut rows = ea_query.query([])?; + while let Some(row) = rows.next()? { + let account_id: i64 = row.get("account_id")?; + let transparent_child_index = row.get::<_, i64>("address_index")?; + let diversifier_index = DiversifierIndex::from( + u32::try_from(transparent_child_index).map_err(|_| { + WalletMigrationError::CorruptedData( + "ephermeral address indices must be in the range of `u32`".to_owned(), + ) + })?, + ); + let address: String = row.get("address")?; + + // We set both the `address` column and the `cached_transparent_receiver_address` + // column to the same value here; there is no Unified address that corresponds to + // this transparent address. + ea_insert.execute(named_params! { + ":account_id": account_id, + ":diversifier_index_be": encode_diversifier_index_be(diversifier_index), + ":key_scope": KeyScope::Ephemeral.encode(), + ":address": address, + ":transparent_child_index": transparent_child_index, + ":cached_transparent_receiver_address": address + })?; + } + } + + conn.execute_batch( + r#" + PRAGMA legacy_alter_table = ON; + + DROP TABLE addresses; + ALTER TABLE addresses_new RENAME TO addresses; + CREATE INDEX idx_addresses_accounts ON addresses ( + account_id ASC + ); + CREATE INDEX idx_addresses_indices ON addresses ( + diversifier_index_be ASC + ); + CREATE INDEX idx_addresses_t_indices ON addresses ( + transparent_child_index ASC + ); + + DROP TABLE ephemeral_addresses; + + PRAGMA legacy_alter_table = OFF; + "#, + )?; + + // Add foreign key references from the *_received_{notes|outputs} tables to the addresses + // table to make it possible to identify which address was involved. These foreign key + // columns must be nullable as for shielded account-internal. Ideally the foreign key + // relationship between `transparent_received_outputs` and `addresses` would not be + // nullable, but we allow it to be so here in order to avoid having to re-create that + // table. + // + // While it would be possible to only add the address reference to + // `transparent_received_outputs`, that would mean that a note received at a shielded + // component of a diversified Unified Address would not update the position of the + // transparent "address gap". Since we will include shielded address indices in the gap + // computation, transparent-only wallets may not be able to discover all transparent funds, + // but users of shielded wallets will be guaranteed to be able to recover all of their + // funds. + conn.execute_batch( + r#" + ALTER TABLE orchard_received_notes + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + ALTER TABLE sapling_received_notes + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + ALTER TABLE transparent_received_outputs + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + "#, + )?; + + // Ensure that an address exists for each received Orchard note, and populate the + // `address_id` column. + #[cfg(feature = "orchard")] + { + let mut stmt_rn_diversifiers = conn.prepare( + r#" + SELECT orn.id, orn.account_id, accounts.uivk, + orn.recipient_key_scope, orn.diversifier, t.mined_height + FROM orchard_received_notes orn + JOIN accounts ON accounts.id = account_id + JOIN transactions t on t.id_tx = orn.tx + "#, + )?; + + let mut rows = stmt_rn_diversifiers.query([])?; + while let Some(row) = rows.next()? { + let scope = KeyScope::decode(row.get("recipient_key_scope")?)?; + // for Orchard and Sapling, we only store addresses for externally-scoped keys. + if scope == KeyScope::EXTERNAL { + let row_id: i64 = row.get("id")?; + let account_id = AccountRef(row.get("account_id")?); + let mined_height = row + .get::<_, Option>("mined_height")? + .map(BlockHeight::from); + + let uivk = decode_uivk(row.get("uivk")?)?; + let diversifier = + orchard::keys::Diversifier::from_bytes(row.get("diversifier")?); + + // TODO: It's annoying that `IncomingViewingKey` doesn't expose the ability to + // decrypt the diversifier to find the index directly, and doesn't provide an + // accessor for `dk`. We already know we have the right IVK. + let ivk = uivk + .orchard() + .as_ref() + .expect("previously received an Orchard output"); + let di = ivk + .diversifier_index(&ivk.address(diversifier)) + .expect("roundtrip"); + let ua = uivk.address(di, None)?; + let address_id = wallet::upsert_address( + conn, + &self.params, + account_id, + di, + &ua, + mined_height, + )?; + + conn.execute( + "UPDATE orchard_received_notes + SET address_id = :address_id + WHERE id = :row_id", + named_params! { + ":address_id": address_id.0, + ":row_id": row_id + }, + )?; + } + } + } + + // Ensure that an address exists for each received Sapling note, and populate the + // `address_id` column. + { + let mut stmt_rn_diversifiers = conn.prepare( + r#" + SELECT srn.id, srn.account_id, accounts.uivk, + srn.recipient_key_scope, srn.diversifier, t.mined_height + FROM sapling_received_notes srn + JOIN accounts ON accounts.id = account_id + JOIN transactions t ON t.id_tx = srn.tx + "#, + )?; + + let mut rows = stmt_rn_diversifiers.query([])?; + while let Some(row) = rows.next()? { + let scope = KeyScope::decode(row.get("recipient_key_scope")?)?; + // for Orchard and Sapling, we only store addresses for externally-scoped keys. + if scope == KeyScope::EXTERNAL { + let row_id: i64 = row.get("id")?; + let account_id = AccountRef(row.get("account_id")?); + let mined_height = row + .get::<_, Option>("mined_height")? + .map(BlockHeight::from); + + let uivk = decode_uivk(row.get("uivk")?)?; + let diversifier = sapling::Diversifier(row.get("diversifier")?); + + // TODO: It's annoying that `IncomingViewingKey` doesn't expose the ability to + // decrypt the diversifier to find the index directly, and doesn't provide an + // accessor for `dk`. We already know we have the right IVK. + let ivk = uivk + .sapling() + .as_ref() + .expect("previously received a Sapling output"); + let di = ivk + .decrypt_diversifier( + &ivk.address(diversifier) + .expect("previously generated an address"), + ) + .expect("roundtrip"); + let ua = uivk.address(di, None)?; + let address_id = wallet::upsert_address( + conn, + &self.params, + account_id, + di, + &ua, + mined_height, + )?; + + conn.execute( + "UPDATE sapling_received_notes + SET address_id = :address_id + WHERE id = :row_id", + named_params! { + ":address_id": address_id.0, + ":row_id": row_id + }, + )?; + } + } + } + + // At this point, every address on which we've received a transparent output should have a + // corresponding row in the `addresses` table with a valid + // `cached_transparent_receiver_address` entry, because we will only have queried the light + // wallet server for outputs from exactly these addresses. So for transparent outputs, we + // join to the addresses table using the address itself in order to obtain the address index. + #[cfg(feature = "transparent-inputs")] + { + conn.execute( + r#" + UPDATE transparent_received_outputs + SET address_id = addresses.id + FROM addresses + WHERE addresses.cached_transparent_receiver_address = transparent_received_outputs.address + "#, + [] + )?; + } + + // Construct a view that identifies the minimum block height at which each address was + // first used + conn.execute_batch( + r#" + CREATE VIEW v_address_uses AS + SELECT orn.address_id, orn.account_id, orn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM orchard_received_notes orn + JOIN addresses a ON a.id = orn.address_id + JOIN transactions t ON t.id_tx = orn.tx + UNION + SELECT srn.address_id, srn.account_id, srn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM sapling_received_notes srn + JOIN addresses a ON a.id = srn.address_id + JOIN transactions t ON t.id_tx = srn.tx + UNION + SELECT tro.address_id, tro.account_id, tro.transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM transparent_received_outputs tro + JOIN addresses a ON a.id = tro.address_id + JOIN transactions t ON t.id_tx = tro.transaction_id; + + CREATE VIEW v_address_first_use AS + SELECT + address_id, + account_id, + key_scope, + diversifier_index_be, + transparent_child_index, + MIN(mined_height) AS first_use_height + FROM v_address_uses + GROUP BY + address_id, account_id, key_scope, + diversifier_index_be, transparent_child_index; + + DROP VIEW v_received_outputs; + CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id, + sapling_received_notes.address_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id, + orchard_received_notes.address_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) + UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id, + u.address_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index); + "#, + )?; + + Ok(()) + } + + fn down(&self, _: &Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs index 41f957202..57605a948 100644 --- a/zcash_client_sqlite/src/wallet/orchard.rs +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -5,14 +5,14 @@ use orchard::{ keys::Diversifier, note::{Note, Nullifier, RandomSeed, Rho}, }; -use rusqlite::{named_params, types::Value, Connection, Row, Transaction}; +use rusqlite::{named_params, types::Value, Connection, Row}; use zcash_client_backend::{ - data_api::NullifierQuery, + data_api::{Account as _, NullifierQuery}, wallet::{ReceivedNote, WalletOrchardOutput}, DecryptedOutput, TransferType, }; -use zcash_keys::keys::UnifiedFullViewingKey; +use zcash_keys::keys::{UnifiedAddressRequest, UnifiedFullViewingKey}; use zcash_primitives::transaction::TxId; use zcash_protocol::{ consensus::{self, BlockHeight}, @@ -22,9 +22,9 @@ use zcash_protocol::{ }; use zip32::Scope; -use crate::{error::SqliteClientError, AccountUuid, ReceivedNoteId, TxRef}; +use crate::{error::SqliteClientError, AccountRef, AccountUuid, AddressRef, ReceivedNoteId, TxRef}; -use super::{get_account_ref, memo_repr, parse_scope, scope_code}; +use super::{get_account, get_account_ref, memo_repr, upsert_address, KeyScope}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedOrchardOutput { @@ -160,9 +160,14 @@ fn to_spendable_note( let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; + let spending_key_scope = zip32::Scope::try_from(KeyScope::decode(scope_code)?) + .map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Invalid key scope code {}", + scope_code + )) + })?; + let recipient = ufvk .orchard() .map(|fvk| fvk.to_ivk(spending_key_scope).address(diversifier)) @@ -226,34 +231,82 @@ pub(crate) fn select_spendable_orchard_notes( ) } +pub(crate) fn ensure_address< + T: ReceivedOrchardOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, + output: &T, + exposure_height: Option, +) -> Result, SqliteClientError> { + if output.recipient_key_scope() != Some(Scope::Internal) { + let account = get_account(conn, params, output.account_id())? + .ok_or(SqliteClientError::AccountUnknown)?; + + let uivk = account.uivk(); + let ivk = uivk + .orchard() + .as_ref() + .expect("uivk decrypted this output."); + let to = output.note().recipient(); + let diversifier_index = ivk + .diversifier_index(&to) + .expect("address corresponds to account"); + + let ua = account + .uivk() + .address(diversifier_index, Some(UnifiedAddressRequest::ALLOW_ALL))?; + upsert_address( + conn, + params, + account.internal_id(), + diversifier_index, + &ua, + exposure_height, + ) + .map(Some) + } else { + Ok(None) + } +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note>( - conn: &Transaction, +/// +/// Returns the internal account identifier of the account that received the output. +pub(crate) fn put_received_note< + T: ReceivedOrchardOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, output: &T, tx_ref: TxRef, + target_or_mined_height: Option, spent_in: Option, -) -> Result<(), SqliteClientError> { +) -> Result { let account_id = get_account_ref(conn, output.account_id())?; + let address_id = ensure_address(conn, params, output, target_or_mined_height)?; let mut stmt_upsert_received_note = conn.prepare_cached( - "INSERT INTO orchard_received_notes - ( - tx, action_index, account_id, + "INSERT INTO orchard_received_notes ( + tx, action_index, account_id, address_id, diversifier, value, rho, rseed, memo, nf, is_change, commitment_tree_position, recipient_key_scope ) VALUES ( - :tx, :action_index, :account_id, + :tx, :action_index, :account_id, :address_id, :diversifier, :value, :rho, :rseed, :memo, :nf, :is_change, :commitment_tree_position, :recipient_key_scope ) ON CONFLICT (tx, action_index) DO UPDATE SET account_id = :account_id, + address_id = :address_id, diversifier = :diversifier, value = :value, rho = :rho, @@ -274,6 +327,7 @@ pub(crate) fn put_received_note( let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; + let spending_key_scope = zip32::Scope::try_from(KeyScope::decode(scope_code)?) + .map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Invalid key scope code {}", + scope_code + )) + })?; let recipient = match spending_key_scope { Scope::Internal => ufvk @@ -330,27 +333,79 @@ pub(crate) fn mark_sapling_note_spent( } } +pub(crate) fn ensure_address< + T: ReceivedSaplingOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, + output: &T, + exposure_height: Option, +) -> Result, SqliteClientError> { + if output.recipient_key_scope() != Some(Scope::Internal) { + let account = get_account(conn, params, output.account_id())? + .ok_or(SqliteClientError::AccountUnknown)?; + + let uivk = account.uivk(); + let ivk = uivk + .sapling() + .as_ref() + .expect("uivk decrypted this output."); + let to = output.note().recipient(); + let diversifier_index = ivk + .decrypt_diversifier(&to) + .expect("address corresponds to account"); + + let ua = account + .uivk() + .address(diversifier_index, Some(UnifiedAddressRequest::ALLOW_ALL))?; + + upsert_address( + conn, + params, + account.internal_id(), + diversifier_index, + &ua, + exposure_height, + ) + .map(Some) + } else { + Ok(None) + } +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note>( - conn: &Transaction, +/// +/// Returns the internal account identifier of the account that received the output. +pub(crate) fn put_received_note< + T: ReceivedSaplingOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, output: &T, tx_ref: TxRef, + target_or_mined_height: Option, spent_in: Option, -) -> Result<(), SqliteClientError> { +) -> Result { let account_id = get_account_ref(conn, output.account_id())?; + let address_id = ensure_address(conn, params, output, target_or_mined_height)?; let mut stmt_upsert_received_note = conn.prepare_cached( - "INSERT INTO sapling_received_notes - (tx, output_index, account_id, diversifier, value, rcm, memo, nf, - is_change, commitment_tree_position, - recipient_key_scope) + "INSERT INTO sapling_received_notes ( + tx, output_index, account_id, address_id, + diversifier, value, rcm, memo, nf, + is_change, commitment_tree_position, + recipient_key_scope + ) VALUES ( :tx, :output_index, :account_id, + :address_id, :diversifier, :value, :rcm, @@ -362,6 +417,7 @@ pub(crate) fn put_received_note( fn address_index_from_diversifier_index_be( diversifier_index_be: &[u8], ) -> Result { - let mut di: [u8; 11] = diversifier_index_be.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion + let di = decode_diversifier_index_be(diversifier_index_be)?; - NonHardenedChildIndex::from_index(DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_string(), - ) - })?) - .ok_or_else(|| { + NonHardenedChildIndex::try_from(di).map_err(|_| { SqliteClientError::CorruptedData( "Unexpected hardened index for transparent address.".to_string(), ) @@ -83,45 +87,48 @@ pub(crate) fn get_transparent_receivers( conn: &rusqlite::Connection, params: &P, account_uuid: AccountUuid, + scopes: &[KeyScope], ) -> Result>, SqliteClientError> { let mut ret: HashMap> = HashMap::new(); - // Get all UAs derived - let mut ua_query = conn.prepare( - "SELECT address, diversifier_index_be - FROM addresses + // Get all addresses with the provided scopes. + let mut addr_query = conn.prepare( + "SELECT address, diversifier_index_be, key_scope + FROM addresses JOIN accounts ON accounts.id = addresses.account_id - WHERE accounts.uuid = :account_uuid", + WHERE accounts.uuid = :account_uuid + AND key_scope IN rarray(:scopes_ptr)", )?; - let mut rows = ua_query.query(named_params![":account_uuid": account_uuid.0])?; + + let scope_values: Vec = scopes.iter().map(|s| Value::Integer(s.encode())).collect(); + let scopes_ptr = Rc::new(scope_values); + let mut rows = addr_query.query(named_params![ + ":account_uuid": account_uuid.0, + ":scopes_ptr": &scopes_ptr + ])?; while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; let di_vec: Vec = row.get(1)?; + let scope = KeyScope::decode(row.get(2)?)?; - let ua = Address::decode(params, &ua_str) + let taddr = Address::decode(params, &ua_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) - }) - .and_then(|addr| match addr { - Address::Unified(ua) => Ok(ua), - _ => Err(SqliteClientError::CorruptedData(format!( - "Addresses table contains {} which is not a unified address", - ua_str, - ))), - })?; - - if let Some(taddr) = ua.transparent() { + })? + .to_transparent_address(); + + if let Some(taddr) = taddr { let address_index = address_index_from_diversifier_index_be(&di_vec)?; - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); - ret.insert(*taddr, Some(metadata)); + let metadata = TransparentAddressMetadata::new(scope.into(), address_index); + ret.insert(taddr, Some(metadata)); } } if let Some((taddr, address_index)) = get_legacy_transparent_address(params, conn, account_uuid)? { - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + let metadata = TransparentAddressMetadata::new(KeyScope::EXTERNAL.into(), address_index); ret.insert(taddr, Some(metadata)); } @@ -183,6 +190,335 @@ pub(crate) fn get_legacy_transparent_address( Ok(None) } +/// Returns the transparent address index at the start of the first gap of at least `gap_limit` +/// indices in the given account, considering only addresses derived for the specified key scope. +/// +/// Returns `Ok(None)` if the gap would start at an index greater than the maximum valid +/// non-hardened transparent child index. +pub(crate) fn find_gap_start( + conn: &rusqlite::Connection, + account_id: AccountRef, + key_scope: KeyScope, + gap_limit: u32, +) -> Result, SqliteClientError> { + match conn + .query_row( + r#" + WITH offsets AS ( + SELECT + a.transparent_child_index, + LEAD(a.transparent_child_index) + OVER (ORDER BY a.transparent_child_index) + AS next_child_index + FROM v_address_first_use a + WHERE a.account_id = :account_id + AND a.key_scope = :key_scope + AND a.transparent_child_index IS NOT NULL + AND a.first_use_height IS NOT NULL + ) + SELECT + transparent_child_index + 1, + next_child_index - transparent_child_index - 1 AS gap_len + FROM offsets + WHERE gap_len >= :gap_limit OR next_child_index IS NULL + ORDER BY transparent_child_index + LIMIT 1 + "#, + named_params![ + ":account_id": account_id.0, + ":key_scope": key_scope.encode(), + ":gap_limit": gap_limit + ], + |row| row.get::<_, u32>(0), + ) + .optional()? + { + Some(i) => Ok(NonHardenedChildIndex::from_index(i)), + None => Ok(Some(NonHardenedChildIndex::ZERO)), + } +} + +pub(crate) fn decode_transparent_child_index( + value: i64, +) -> Result { + u32::try_from(value) + .ok() + .and_then(NonHardenedChildIndex::from_index) + .ok_or_else(|| { + SqliteClientError::CorruptedData(format!("Illegal transparent child index {value}")) + }) +} + +/// Returns a vector with the next `n` previously unreserved transparent addresses for +/// the given account. These addresses must have been previously generated using +/// `generate_gap_addresses`. +/// +/// # Errors +/// +/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. +/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses +/// within the gap limit after the last address in this account that is known to have an +/// output in a mined transaction. +/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, +/// if the limit on transparent address indices has been reached. +pub(crate) fn reserve_next_n_addresses( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountRef, + key_scope: KeyScope, + gap_limit: u32, + n: usize, +) -> Result, SqliteClientError> { + if n == 0 { + return Ok(vec![]); + } + + let gap_start = find_gap_start(conn, account_id, key_scope, gap_limit)?.ok_or( + SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted), + )?; + + let mut stmt_addrs_to_reserve = conn.prepare( + "SELECT id, transparent_child_index, cached_transparent_receiver_address + FROM addresses + WHERE account_id = :account_id + AND key_scope = :key_scope + AND transparent_child_index >= :gap_start + AND transparent_child_index < :gap_end + AND exposed_at_height IS NULL + ORDER BY transparent_child_index + LIMIT :n", + )?; + + let addresses_to_reserve = stmt_addrs_to_reserve + .query_and_then( + named_params! { + ":account_id": account_id.0, + ":key_scope": key_scope.encode(), + ":gap_start": gap_start.index(), + ":gap_end": gap_start.saturating_add(gap_limit).index(), + ":n": n + }, + |row| { + let address_id = row.get("id").map(AddressRef)?; + let transparent_child_index = row + .get::<_, Option>("transparent_child_index")? + .map(decode_transparent_child_index) + .transpose()?; + let address = row + .get::<_, Option>("cached_transparent_receiver_address")? + .map(|addr_str| TransparentAddress::decode(params, &addr_str)) + .transpose()?; + + Ok::<_, SqliteClientError>(transparent_child_index.zip(address).map(|(i, a)| { + ( + address_id, + a, + TransparentAddressMetadata::new(key_scope.into(), i), + ) + })) + }, + )? + .filter_map(|r| r.transpose()) + .collect::, _>>()?; + + if addresses_to_reserve.len() < n { + return Err(SqliteClientError::ReachedGapLimit( + gap_start.index() + gap_limit, + )); + } + + let current_chain_tip = chain_tip_height(conn)?.ok_or(SqliteClientError::ChainHeightUnknown)?; + + let reserve_id_values: Vec = addresses_to_reserve + .iter() + .map(|(id, _, _)| Value::Integer(id.0)) + .collect(); + let reserved_ptr = Rc::new(reserve_id_values); + conn.execute( + "UPDATE addresses + SET exposed_at_height = :chain_tip_height + WHERE id IN rarray(:reserved_ptr)", + named_params! { + ":chain_tip_height": u32::from(current_chain_tip), + ":reserved_ptr": &reserved_ptr + }, + )?; + + Ok(addresses_to_reserve) +} + +/// Extend the range of preallocated addresses in an account to ensure that a full `gap_limit` of +/// transparent addresses is available from the first gap in existing indices of addresses at which +/// a received transaction has been observed on the chain, for each key scope. +/// +/// The provided [`UnifiedAddressRequest`] is used to pregenerate unified addresses that correspond +/// to the transparent address index in question; such unified addresses need not internally +/// contain a transparent receiver, and may be overwritten when these addresses are exposed via the +/// [`WalletWrite::get_next_available_address`] or [`WalletWrite::get_address_for_index`] methods. +/// If no request is provided, each address so generated will contain a receiver for each possible +/// pool: i.e., a recevier for each data item in the account's UFVK or UIVK where the transparent +/// child index is valid. +/// +/// [`WalletWrite::get_next_available_address`]: zcash_client_backend::data_api::WalletWrite::get_next_available_address +/// [`WalletWrite::get_address_for_index`]: zcash_client_backend::data_api::WalletWrite::get_address_for_index +pub(crate) fn generate_gap_addresses( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountRef, + key_scope: KeyScope, + gap_limits: &GapLimits, + request: Option, +) -> Result<(), SqliteClientError> { + let account = get_account_internal(conn, params, account_id)? + .ok_or_else(|| SqliteClientError::AccountUnknown)?; + + let request = request.unwrap_or_else(|| { + use ReceiverRequirement::*; + UnifiedAddressRequest::unsafe_new(Allow, Allow, Require) + }); + + let gen_addrs = |key_scope: KeyScope, index: NonHardenedChildIndex| { + Ok::<_, SqliteClientError>(match key_scope { + KeyScope::Zip32(zip32::Scope::External) => { + let ua = account.uivk().address(index.into(), Some(request)); + let transparent_address = account + .uivk() + .transparent() + .as_ref() + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_address(index)?; + ( + ua.map_or_else( + |e| { + if matches!(e, AddressGenerationError::ShieldedReceiverRequired) { + // fall back to the transparent-only address + Ok(Address::from(transparent_address).to_zcash_address(params)) + } else { + // other address generation errors are allowed to propagate + Err(e) + } + }, + |addr| Ok(Address::from(addr).to_zcash_address(params)), + )?, + transparent_address, + ) + } + KeyScope::Zip32(zip32::Scope::Internal) => { + let internal_address = account + .ufvk() + .and_then(|k| k.transparent()) + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_internal_ivk()? + .derive_address(index)?; + ( + Address::from(internal_address).to_zcash_address(params), + internal_address, + ) + } + KeyScope::Ephemeral => { + let ephemeral_address = account + .ufvk() + .and_then(|k| k.transparent()) + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_ephemeral_ivk()? + .derive_ephemeral_address(index)?; + ( + Address::from(ephemeral_address).to_zcash_address(params), + ephemeral_address, + ) + } + }) + }; + + let gap_limit = match key_scope { + KeyScope::Zip32(zip32::Scope::External) => gap_limits.external(), + KeyScope::Zip32(zip32::Scope::Internal) => gap_limits.transparent_internal(), + KeyScope::Ephemeral => gap_limits.ephemeral(), + }; + + if let Some(gap_start) = find_gap_start(conn, account_id, key_scope, gap_limit)? { + let range_to_store = gap_start.index()..gap_start.saturating_add(gap_limit).index(); + if range_to_store.is_empty() { + return Ok(()); + } + // exposed_at_height is initially NULL + let mut stmt_insert_address = conn.prepare_cached( + "INSERT INTO addresses ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) + VALUES ( + :account_id, :diversifier_index_be, :key_scope, :address, + :transparent_child_index, :transparent_address + ) + ON CONFLICT (account_id, diversifier_index_be, key_scope) DO NOTHING", + )?; + + for raw_index in range_to_store { + let transparent_child_index = NonHardenedChildIndex::from_index(raw_index) + .expect("restricted to valid range above"); + let (zcash_address, transparent_address) = + gen_addrs(key_scope, transparent_child_index)?; + + stmt_insert_address.execute(named_params![ + ":account_id": account_id.0, + ":diversifier_index_be": encode_diversifier_index_be(transparent_child_index.into()), + ":key_scope": key_scope.encode(), + ":address": zcash_address.encode(), + ":transparent_child_index": raw_index, + ":transparent_address": transparent_address.encode(params) + ])?; + } + } + + Ok(()) +} + +/// If `address` is one of our addresses, check whether it was used as the recipient address for +/// any of our received outputs. +/// +/// If the address was already used in an output we received, this method will return +/// [`SqliteClientError::AddressReuse`]. +pub(crate) fn check_address_reuse( + conn: &rusqlite::Transaction, + params: &P, + address: &Address, +) -> Result<(), SqliteClientError> { + // TODO: ideally we would do something better than string matching here - the best would be to + // have the diversifier index for the address passed to us instead of the address itself, but + // not all call sites currently have a good way to obtain the diversifier index. We could + // trial-decrypt with each of the wallet's IVKs if we wanted to do it here, but a better + // approach is to restructure the call sites so that we don't discard diversifier index + // information in the process of passing it through to here. + let addr_str = address.encode(params); + let taddr_str = address.to_transparent_address().map(|a| a.encode(params)); + + let mut stmt = conn.prepare_cached( + "SELECT t.txid + FROM transactions t + JOIN v_received_outputs vro ON vro.transaction_id = t.id_tx + JOIN addresses a ON a.id = vro.address_id + WHERE a.address = :address + OR a.cached_transparent_receiver_address = :transparent_address", + )?; + + let txids = stmt + .query_and_then( + named_params![ + ":address": addr_str, + ":transparent_address": taddr_str, + ], + |row| Ok(TxId::from_bytes(row.get::<_, [u8; 32]>(0)?)), + )? + .collect::, SqliteClientError>>()?; + + if let Some(txids) = NonEmpty::from_vec(txids) { + return Err(SqliteClientError::AddressReuse(addr_str, txids)); + } + + Ok(()) +} + fn to_unspent_transparent_output(row: &Row) -> Result { let txid: Vec = row.get("txid")?; let mut txid_bytes = [0u8; 32]; @@ -559,28 +895,19 @@ pub(crate) fn mark_transparent_utxo_spent( /// Adds the given received UTXO to the datastore. pub(crate) fn put_received_transparent_utxo( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction, params: &P, output: &WalletTransparentOutput, -) -> Result { - let address = output.recipient_address(); - if let Some(receiving_account) = - find_account_uuid_for_transparent_address(conn, params, address)? - { - put_transparent_output( - conn, - params, - output.outpoint(), - output.txout(), - output.mined_height(), - address, - receiving_account, - true, - ) - } else { - // The UTXO was not for any of our transparent addresses. - Err(SqliteClientError::AddressNotRecognized(*address)) - } +) -> Result<(AccountRef, KeyScope, UtxoId), SqliteClientError> { + put_transparent_output( + conn, + params, + output.outpoint(), + output.txout(), + output.mined_height(), + output.recipient_address(), + true, + ) } /// Returns the vector of [`TransactionDataRequest`]s that represents the information needed by the @@ -634,21 +961,29 @@ pub(crate) fn get_transparent_address_metadata( address: &TransparentAddress, ) -> Result, SqliteClientError> { let address_str = address.encode(params); - - if let Some(di_vec) = conn + let addr_meta = conn .query_row( - "SELECT diversifier_index_be FROM addresses + "SELECT diversifier_index_be, key_scope + FROM addresses JOIN accounts ON addresses.account_id = accounts.id - WHERE accounts.uuid = :account_uuid + WHERE accounts.uuid = :account_uuid AND cached_transparent_receiver_address = :address", named_params![":account_uuid": account_uuid.0, ":address": &address_str], - |row| row.get::<_, Vec>(0), + |row| { + let di_be: Vec = row.get(0)?; + let scope_code = row.get(1)?; + Ok(KeyScope::decode(scope_code).and_then(|key_scope| { + address_index_from_diversifier_index_be(&di_be).map(|address_index| { + TransparentAddressMetadata::new(key_scope.into(), address_index) + }) + })) + }, ) .optional()? - { - let address_index = address_index_from_diversifier_index_be(&di_vec)?; - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); - return Ok(Some(metadata)); + .transpose()?; + + if addr_meta.is_some() { + return Ok(addr_meta); } if let Some((legacy_taddr, address_index)) = @@ -660,13 +995,6 @@ pub(crate) fn get_transparent_address_metadata( } } - // Search known ephemeral addresses. - if let Some(address_index) = - ephemeral::find_index_for_ephemeral_address_str(conn, account_uuid, &address_str)? - { - return Ok(Some(ephemeral::metadata(address_index))); - } - Ok(None) } @@ -684,27 +1012,27 @@ pub(crate) fn find_account_uuid_for_transparent_address Result, SqliteClientError> { +) -> Result, SqliteClientError> { let address_str = address.encode(params); - if let Some(account_id) = conn + if let Some((account_id, key_scope_code)) = conn .query_row( - "SELECT accounts.uuid - FROM addresses + "SELECT accounts.uuid, addresses.key_scope + FROM addresses JOIN accounts ON accounts.id = addresses.account_id WHERE cached_transparent_receiver_address = :address", named_params![":address": &address_str], - |row| Ok(AccountUuid(row.get(0)?)), + |row| Ok((AccountUuid(row.get(0)?), row.get(1)?)), ) .optional()? { - return Ok(Some(account_id)); + return Ok(Some((account_id, KeyScope::decode(key_scope_code)?))); } // Search known ephemeral addresses. if let Some(account_id) = ephemeral::find_account_for_ephemeral_address_str(conn, &address_str)? { - return Ok(Some(account_id)); + return Ok(Some((account_id, KeyScope::Ephemeral))); } let account_ids = get_account_ids(conn)?; @@ -718,7 +1046,7 @@ pub(crate) fn find_account_uuid_for_transparent_address( txout: &TxOut, output_height: Option, address: &TransparentAddress, - receiving_account_uuid: AccountUuid, known_unspent: bool, -) -> Result { +) -> Result<(AccountRef, KeyScope, UtxoId), SqliteClientError> { + let addr_str = address.encode(params); + + // Unlike the shielded pools, we only can receive transparent outputs on addresses for which we + // have an `addresses` table entry, so we can just query for that here. + let (address_id, account_id, key_scope_code) = conn.query_row( + "SELECT id, account_id, key_scope + FROM addresses + WHERE cached_transparent_receiver_address = :transparent_address", + named_params! {":transparent_address": addr_str}, + |row| { + Ok(( + row.get("id").map(AddressRef)?, + row.get("account_id").map(AccountRef)?, + row.get("key_scope")?, + )) + }, + )?; + + let key_scope = KeyScope::decode(key_scope_code)?; + let output_height = output_height.map(u32::from); - let receiving_account_id = super::get_account_ref(conn, receiving_account_uuid)?; // Check whether we have an entry in the blocks table for the output height; // if not, the transaction will be updated with its mined height when the @@ -810,16 +1156,17 @@ pub(crate) fn put_transparent_output( let mut stmt_upsert_transparent_output = conn.prepare_cached( "INSERT INTO transparent_received_outputs ( transaction_id, output_index, - account_id, address, script, + account_id, address_id, address, script, value_zat, max_observed_unspent_height ) VALUES ( :transaction_id, :output_index, - :account_id, :address, :script, + :account_id, :address_id, :address, :script, :value_zat, :max_observed_unspent_height ) ON CONFLICT (transaction_id, output_index) DO UPDATE SET account_id = :account_id, + address_id = :address_id, address = :address, script = :script, value_zat = :value_zat, @@ -830,7 +1177,8 @@ pub(crate) fn put_transparent_output( let sql_args = named_params![ ":transaction_id": id_tx, ":output_index": &outpoint.n(), - ":account_id": receiving_account_id.0, + ":account_id": account_id.0, + ":address_id": address_id.0, ":address": &address.encode(params), ":script": &txout.script_pubkey.0, ":value_zat": &i64::from(ZatBalance::from(txout.value)), @@ -862,7 +1210,7 @@ pub(crate) fn put_transparent_output( mark_transparent_utxo_spent(conn, spending_transaction_id, outpoint)?; } - Ok(utxo_id) + Ok((account_id, key_scope, utxo_id)) } /// Adds a request to retrieve transactions involving the specified address to the transparent @@ -903,14 +1251,19 @@ mod tests { use secrecy::Secret; use transparent::keys::NonHardenedChildIndex; use zcash_client_backend::{ - data_api::{testing::TestBuilder, Account as _, WalletWrite, GAP_LIMIT}, + data_api::{testing::TestBuilder, Account as _, WalletWrite}, wallet::TransparentAddressMetadata, }; use zcash_primitives::block::BlockHash; use crate::{ + error::SqliteClientError, testing::{db::TestDbFactory, BlockCache}, - wallet::{get_account_ref, transparent::ephemeral}, + wallet::{ + get_account_ref, + transparent::{ephemeral, find_gap_start, reserve_next_n_addresses}, + KeyScope, + }, WalletDb, }; @@ -949,21 +1302,59 @@ mod tests { let account0_uuid = st.test_account().unwrap().account().id(); let account0_id = get_account_ref(&st.wallet().db().conn, account0_uuid).unwrap(); + // The chain height must be known in order to reserve addresses, as we store the height at + // which the address was considered to be exposed. + st.wallet_mut() + .db_mut() + .update_chain_tip(birthday.height()) + .unwrap(); + let check = |db: &WalletDb<_, _>, account_id| { eprintln!("checking {account_id:?}"); - assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); - assert_matches!(ephemeral::first_unreserved_index(&db.conn, account_id), Ok(addr_index) if addr_index == 0); + assert_matches!( + find_gap_start(&db.conn, account_id, KeyScope::Ephemeral, db.gap_limits.ephemeral()), Ok(addr_index) + if addr_index == Some(NonHardenedChildIndex::ZERO) + ); + //assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); let known_addrs = ephemeral::get_known_ephemeral_addresses(&db.conn, &db.params, account_id, None) .unwrap(); - let expected_metadata: Vec = (0..GAP_LIMIT) + let expected_metadata: Vec = (0..db.gap_limits.ephemeral()) .map(|i| ephemeral::metadata(NonHardenedChildIndex::from_index(i).unwrap())) .collect(); let actual_metadata: Vec = known_addrs.into_iter().map(|(_, meta)| meta).collect(); assert_eq!(actual_metadata, expected_metadata); + + let transaction = &db.conn.unchecked_transaction().unwrap(); + // reserve half the addresses (rounding down) + let reserved = reserve_next_n_addresses( + transaction, + &db.params, + account_id, + KeyScope::Ephemeral, + db.gap_limits.ephemeral(), + (db.gap_limits.ephemeral() / 2) as usize, + ) + .unwrap(); + assert_eq!(reserved.len(), (db.gap_limits.ephemeral() / 2) as usize); + + // we have not yet used any of the addresses, so the maximum available address index + // should not have increased, and therefore attempting to reserve a full gap limit + // worth of addresses should fail. + assert_matches!( + reserve_next_n_addresses( + transaction, + &db.params, + account_id, + KeyScope::Ephemeral, + db.gap_limits.ephemeral(), + db.gap_limits.ephemeral() as usize + ), + Err(SqliteClientError::ReachedGapLimit(_)) + ); }; check(st.wallet().db(), account0_id); diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 293eaa955..d97706d3f 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -1,22 +1,17 @@ -//! Functions for wallet support of ephemeral transparent addresses. -use std::cmp::{max, min}; +//! Functikjns for wallet support of ephemeral transparent addresses. use std::ops::Range; use rusqlite::{named_params, OptionalExtension}; use ::transparent::{ address::TransparentAddress, - keys::{EphemeralIvk, NonHardenedChildIndex, TransparentKeyScope}, + keys::{NonHardenedChildIndex, TransparentKeyScope}, }; -use zcash_client_backend::{data_api::GAP_LIMIT, wallet::TransparentAddressMetadata}; -use zcash_keys::keys::UnifiedFullViewingKey; -use zcash_keys::{encoding::AddressCodec, keys::AddressGenerationError}; -use zcash_primitives::transaction::TxId; +use zcash_client_backend::wallet::TransparentAddressMetadata; +use zcash_keys::encoding::AddressCodec; use zcash_protocol::consensus; -use crate::wallet::{self, get_account_ref}; -use crate::AccountUuid; -use crate::{error::SqliteClientError, AccountRef, TxRef}; +use crate::{error::SqliteClientError, wallet::KeyScope, AccountRef, AccountUuid}; // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. @@ -24,161 +19,50 @@ pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddre TransparentAddressMetadata::new(TransparentKeyScope::EPHEMERAL, address_index) } -/// Returns the first unstored ephemeral address index in the given account. -pub(crate) fn first_unstored_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - match conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id - ORDER BY address_index DESC - LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - { - Some(i) if i >= (1 << 31) + GAP_LIMIT => { - unreachable!("violates constraint index_range_and_address_nullity") - } - Some(i) => Ok(i.checked_add(1).unwrap()), - None => Ok(0), - } -} - -/// Returns the first unreserved ephemeral address index in the given account. -pub(crate) fn first_unreserved_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - first_unstored_index(conn, account_id)? - .checked_sub(GAP_LIMIT) - .ok_or(SqliteClientError::CorruptedData( - "ephemeral_addresses table has not been initialized".to_owned(), - )) -} - -/// Returns the first ephemeral address index in the given account that -/// would violate the gap invariant if used. -pub(crate) fn first_unsafe_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - // The inner join with `transactions` excludes addresses for which - // `seen_in_tx` is NULL. The query also excludes addresses observed - // to have been mined in a transaction that we currently see as unmined. - // This is conservative in terms of avoiding violation of the gap - // invariant: it can only cause us to get to the end of the gap sooner. - // - // TODO: do we want to only consider transactions with a minimum number - // of confirmations here? - let first_unmined_index: u32 = match conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - JOIN transactions t ON t.id_tx = seen_in_tx - WHERE account_id = :account_id AND t.mined_height IS NOT NULL - ORDER BY address_index DESC - LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - { - Some(i) if i >= 1 << 31 => { - unreachable!("violates constraint index_range_and_address_nullity") - } - Some(i) => i.checked_add(1).unwrap(), - None => 0, - }; - Ok(min( - 1 << 31, - first_unmined_index.checked_add(GAP_LIMIT).unwrap(), - )) -} - -/// Utility function to return an `Range` that starts at `i` -/// and is of length up to `n`. The range is truncated if necessary -/// so that it contains no elements beyond the maximum valid address -/// index, `(1 << 31) - 1`. -pub(crate) fn range_from(i: u32, n: u32) -> Range { - let first = min(1 << 31, i); - let last = min(1 << 31, i.saturating_add(n)); - first..last -} - -/// Returns the ephemeral transparent IVK for a given account ID. -pub(crate) fn get_ephemeral_ivk( - conn: &rusqlite::Connection, - params: &P, - account_id: AccountRef, -) -> Result, SqliteClientError> { - let ufvk = conn - .query_row( - "SELECT ufvk FROM accounts WHERE id = :account_id", - named_params![":account_id": account_id.0], - |row| { - let ufvk_str: Option = row.get("ufvk")?; - Ok(ufvk_str.map(|s| { - UnifiedFullViewingKey::decode(params, &s[..]) - .map_err(SqliteClientError::BadAccountData) - })) - }, - ) - .optional()? - .ok_or(SqliteClientError::AccountUnknown)? - .transpose()?; - - let eivk = ufvk - .as_ref() - .and_then(|ufvk| ufvk.transparent()) - .map(|t| t.derive_ephemeral_ivk()) - .transpose()?; - - Ok(eivk) -} - -/// Returns a vector of ephemeral transparent addresses associated with the given -/// account controlled by this wallet, along with their metadata. The result includes -/// reserved addresses, and addresses for `GAP_LIMIT` additional indices (capped to -/// the maximum index). +/// Returns a vector of ephemeral transparent addresses associated with the given account +/// controlled by this wallet, along with their metadata. The result includes reserved addresses, +/// and addresses for the wallet's configured ephemeral address gap limit of additional indices +/// (capped to the maximum index). /// -/// If `index_range` is some `Range`, it limits the result to addresses with indices -/// in that range. +/// If `index_range` is some `Range`, it limits the result to addresses with indices in that range. pub(crate) fn get_known_ephemeral_addresses( conn: &rusqlite::Connection, params: &P, account_id: AccountRef, - index_range: Option>, + index_range: Option>, ) -> Result, SqliteClientError> { - let index_range = index_range.unwrap_or(0..(1 << 31)); - let mut stmt = conn.prepare( - "SELECT address, address_index - FROM ephemeral_addresses ea - WHERE ea.account_id = :account_id - AND address_index >= :start - AND address_index < :end - ORDER BY address_index", + "SELECT cached_transparent_receiver_address, transparent_child_index + FROM addresses + WHERE account_id = :account_id + AND transparent_child_index >= :start + AND transparent_child_index < :end + AND key_scope = :key_scope + ORDER BY transparent_child_index", )?; - let mut rows = stmt.query(named_params![ - ":account_id": account_id.0, - ":start": index_range.start, - ":end": min(1 << 31, index_range.end), - ])?; - let mut result = vec![]; + let results = stmt + .query_and_then( + named_params![ + ":account_id": account_id.0, + ":start": index_range.as_ref().map_or(NonHardenedChildIndex::ZERO, |i| i.start).index(), + ":end": index_range.as_ref().map_or(NonHardenedChildIndex::MAX, |i| i.end).index(), + ":key_scope": KeyScope::Ephemeral.encode() + ], + |row| { + let addr_str: String = row.get(0)?; + let raw_index: u32 = row.get(1)?; + let address_index = NonHardenedChildIndex::from_index(raw_index) + .expect("where clause ensures this is in range"); + Ok::<_, SqliteClientError>(( + TransparentAddress::decode(params, &addr_str)?, + metadata(address_index) + )) + }, + )? + .collect::, _>>()?; - while let Some(row) = rows.next()? { - let addr_str: String = row.get(0)?; - let raw_index: u32 = row.get(1)?; - let address_index = NonHardenedChildIndex::from_index(raw_index) - .expect("where clause ensures this is in range"); - let address = TransparentAddress::decode(params, &addr_str)?; - result.push((address, metadata(address_index))); - } - Ok(result) + Ok(results) } /// If this is a known ephemeral address in any account, return its account id. @@ -189,277 +73,15 @@ pub(crate) fn find_account_for_ephemeral_address_str( Ok(conn .query_row( "SELECT accounts.uuid - FROM ephemeral_addresses ea - JOIN accounts ON accounts.id = ea.account_id - WHERE address = :address", - named_params![":address": &address_str], + FROM addresses + JOIN accounts ON accounts.id = account_id + WHERE cached_transparent_receiver_address = :address + AND key_scope = :key_scope", + named_params![ + ":address": &address_str, + ":key_scope": KeyScope::Ephemeral.encode() + ], |row| Ok(AccountUuid(row.get(0)?)), ) .optional()?) } - -/// If this is a known ephemeral address in the given account, return its index. -pub(crate) fn find_index_for_ephemeral_address_str( - conn: &rusqlite::Connection, - account_uuid: AccountUuid, - address_str: &str, -) -> Result, SqliteClientError> { - let account_id = get_account_ref(conn, account_uuid)?; - Ok(conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id AND address = :address", - named_params![":account_id": account_id.0, ":address": &address_str], - |row| row.get::<_, u32>(0), - ) - .optional()? - .map(|index| { - NonHardenedChildIndex::from_index(index) - .expect("valid by constraint index_range_and_address_nullity") - })) -} - -/// Returns a vector with the next `n` previously unreserved ephemeral addresses for -/// the given account. -/// -/// # Errors -/// -/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. -/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and -/// it is not possible to derive new addresses for it. -/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses -/// within the gap limit after the last address in this account that is known to have an -/// output in a mined transaction. -/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, -/// if the limit on transparent address indices has been reached. -pub(crate) fn reserve_next_n_ephemeral_addresses( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, - n: usize, -) -> Result, SqliteClientError> { - if n == 0 { - return Ok(vec![]); - } - - let first_unreserved = first_unreserved_index(conn, account_id)?; - let first_unsafe = first_unsafe_index(conn, account_id)?; - let allocation = range_from( - first_unreserved, - u32::try_from(n).map_err(|_| AddressGenerationError::DiversifierSpaceExhausted)?, - ); - - if allocation.len() < n { - return Err(AddressGenerationError::DiversifierSpaceExhausted.into()); - } - if allocation.end > first_unsafe { - let account_uuid = wallet::get_account_uuid(conn, account_id)?; - return Err(SqliteClientError::ReachedGapLimit( - account_uuid, - max(first_unreserved, first_unsafe), - )); - } - reserve_until(conn, params, account_id, allocation.end)?; - get_known_ephemeral_addresses(conn, params, account_id, Some(allocation)) -} - -/// Initialize the `ephemeral_addresses` table. This must be called when -/// creating or migrating an account. -pub(crate) fn init_account( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, -) -> Result<(), SqliteClientError> { - reserve_until(conn, params, account_id, 0) -} - -/// Extend the range of stored addresses in an account if necessary so that the index of the next -/// address to reserve will be *at least* `next_to_reserve`. If no transparent key exists for the -/// given account or it would already have been at least `next_to_reserve`, then do nothing. -/// -/// Note that this is called from database migration code. -/// -/// # Panics -/// -/// Panics if the precondition `next_to_reserve <= (1 << 31)` does not hold. -fn reserve_until( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, - next_to_reserve: u32, -) -> Result<(), SqliteClientError> { - assert!(next_to_reserve <= 1 << 31); - - if let Some(ephemeral_ivk) = get_ephemeral_ivk(conn, params, account_id)? { - let first_unstored = first_unstored_index(conn, account_id)?; - let range_to_store = first_unstored..(next_to_reserve.checked_add(GAP_LIMIT).unwrap()); - if range_to_store.is_empty() { - return Ok(()); - } - - // used_in_tx and seen_in_tx are initially NULL - let mut stmt_insert_ephemeral_address = conn.prepare_cached( - "INSERT INTO ephemeral_addresses (account_id, address_index, address) - VALUES (:account_id, :address_index, :address)", - )?; - - for raw_index in range_to_store { - // The range to store may contain indicies that are out of the valid range of non hardened - // child indices; we still store explicit rows in the ephemeral_addresses table for these - // so that it's possible to find the first unused address using dead reckoning with the gap - // limit. - let address_str_opt = NonHardenedChildIndex::from_index(raw_index) - .map(|address_index| { - ephemeral_ivk - .derive_ephemeral_address(address_index) - .map(|addr| addr.encode(params)) - }) - .transpose()?; - - stmt_insert_ephemeral_address.execute(named_params![ - ":account_id": account_id.0, - ":address_index": raw_index, - ":address": address_str_opt, - ])?; - } - } - - Ok(()) -} - -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -fn ephemeral_address_reuse_check( - conn: &rusqlite::Transaction, - address_str: &str, -) -> Result<(), SqliteClientError> { - // It is intentional that we don't require `t.mined_height` to be non-null. - // That is, we conservatively treat an ephemeral address as potentially - // reused even if we think that the transaction where we had evidence of - // its use is at present unmined. This should never occur in supported - // situations where only a single correctly operating wallet instance is - // using a given seed, because such a wallet will not reuse an address that - // it ever reserved. - // - // `COALESCE(used_in_tx, seen_in_tx)` can only differ from `used_in_tx` - // if the address was reserved, an error occurred in transaction creation - // before calling `mark_ephemeral_address_as_used`, and then we saw the - // address in another transaction (presumably created by another wallet - // instance, or as a result of a bug) anyway. - let res = conn - .query_row( - "SELECT t.txid FROM ephemeral_addresses - LEFT OUTER JOIN transactions t - ON t.id_tx = COALESCE(used_in_tx, seen_in_tx) - WHERE address = :address", - named_params![":address": address_str], - |row| row.get::<_, Option>>(0), - ) - .optional()? - .flatten(); - - if let Some(txid_bytes) = res { - let txid = TxId::from_bytes( - txid_bytes - .try_into() - .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, - ); - Err(SqliteClientError::EphemeralAddressReuse( - address_str.to_owned(), - txid, - )) - } else { - Ok(()) - } -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in a transaction that we have just created. This has no effect if `address` is -/// not one of our ephemeral addresses. -/// -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -pub(crate) fn mark_ephemeral_address_as_used( - conn: &rusqlite::Transaction, - params: &P, - ephemeral_address: &TransparentAddress, - tx_ref: TxRef, -) -> Result<(), SqliteClientError> { - let address_str = ephemeral_address.encode(params); - ephemeral_address_reuse_check(conn, &address_str)?; - - // We update both `used_in_tx` and `seen_in_tx` here, because a used address has - // necessarily been seen in a transaction. We will not treat this as extending the - // range of addresses that are safe to reserve unless and until the transaction is - // observed as mined. - let update_result = conn - .query_row( - "UPDATE ephemeral_addresses - SET used_in_tx = :tx_ref, seen_in_tx = :tx_ref - WHERE address = :address - RETURNING account_id, address_index", - named_params![":tx_ref": tx_ref.0, ":address": address_str], - |row| Ok((AccountRef(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), - ) - .optional()?; - - // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. - if let Some((account_id, address_index)) = update_result { - let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); - reserve_until(conn, params, account_id, next_to_reserve)?; - } - Ok(()) -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in the given mined transaction (which may or may not be a transaction we sent). -/// -/// `tx_ref` must be a valid transaction reference. This call has no effect if -/// `address` is not one of our ephemeral addresses. -pub(crate) fn mark_ephemeral_address_as_seen( - conn: &rusqlite::Transaction, - params: &P, - address: &TransparentAddress, - tx_ref: TxRef, -) -> Result<(), SqliteClientError> { - let address_str = address.encode(params); - - // Figure out which transaction was mined earlier: `tx_ref`, or any existing - // tx referenced by `seen_in_tx` for the given address. Prefer the existing - // reference in case of a tie or if both transactions are unmined. - // This slightly reduces the chance of unnecessarily reaching the gap limit - // too early in some corner cases (because the earlier transaction is less - // likely to be unmined). - // - // The query should always return a value if `tx_ref` is valid. - let earlier_ref = conn.query_row( - "SELECT id_tx FROM transactions - LEFT OUTER JOIN ephemeral_addresses e - ON id_tx = e.seen_in_tx - WHERE id_tx = :tx_ref OR e.address = :address - ORDER BY mined_height ASC NULLS LAST, - tx_index ASC NULLS LAST, - e.seen_in_tx ASC NULLS LAST - LIMIT 1", - named_params![":tx_ref": tx_ref.0, ":address": address_str], - |row| row.get::<_, i64>(0), - )?; - - let update_result = conn - .query_row( - "UPDATE ephemeral_addresses - SET seen_in_tx = :seen_in_tx - WHERE address = :address - RETURNING account_id, address_index", - named_params![":seen_in_tx": &earlier_ref, ":address": address_str], - |row| Ok((AccountRef(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), - ) - .optional()?; - - // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. - if let Some((account_id, address_index)) = update_result { - let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); - reserve_until(conn, params, account_id, next_to_reserve)?; - } - Ok(()) -} diff --git a/zcash_transparent/CHANGELOG.md b/zcash_transparent/CHANGELOG.md index 46dc2a245..b902cdd78 100644 --- a/zcash_transparent/CHANGELOG.md +++ b/zcash_transparent/CHANGELOG.md @@ -13,6 +13,11 @@ and this library adheres to Rust's notion of ### Added - `zcash_transparent::pczt::Bip32Derivation::extract_bip_44_fields` +- `zcash_transparent::keys::NonHardenedChildIndex::saturating_sub` +- `zcash_transparent::keys::NonHardenedChildIndex::saturating_add` +- `zcash_transparent::keys::NonHardenedChildIndex::MAX` +- `impl From for zip32::DiversifierIndex` +- `impl TryFrom for NonHardenedChildIndex` ## [0.1.0] - 2024-12-16 diff --git a/zcash_transparent/src/keys.rs b/zcash_transparent/src/keys.rs index 74c6dccb3..4f56264b0 100644 --- a/zcash_transparent/src/keys.rs +++ b/zcash_transparent/src/keys.rs @@ -2,6 +2,7 @@ use bip32::ChildNumber; use subtle::{Choice, ConstantTimeEq}; +use zip32::DiversifierIndex; #[cfg(feature = "transparent-inputs")] use { @@ -67,7 +68,7 @@ impl From for ChildNumber { /// A child index for a derived transparent address. /// /// Only NON-hardened derivation is supported. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct NonHardenedChildIndex(u32); impl ConstantTimeEq for NonHardenedChildIndex { @@ -77,13 +78,17 @@ impl ConstantTimeEq for NonHardenedChildIndex { } impl NonHardenedChildIndex { + /// The minimum valid non-hardened child index. pub const ZERO: NonHardenedChildIndex = NonHardenedChildIndex(0); + /// The maximum valid non-hardened child index. + pub const MAX: NonHardenedChildIndex = NonHardenedChildIndex((1 << 31) - 1); + /// Parses the given ZIP 32 child index. /// /// Returns `None` if the hardened bit is set. - pub fn from_index(i: u32) -> Option { - if i < (1 << 31) { + pub const fn from_index(i: u32) -> Option { + if i <= Self::MAX.0 { Some(NonHardenedChildIndex(i)) } else { None @@ -91,15 +96,32 @@ impl NonHardenedChildIndex { } /// Returns the index as a 32-bit integer. - pub fn index(&self) -> u32 { + pub const fn index(&self) -> u32 { self.0 } - pub fn next(&self) -> Option { + /// Returns the successor to this index. + pub const fn next(&self) -> Option { // overflow cannot happen because self.0 is 31 bits, and the next index is at most 32 bits // which in that case would lead from_index to return None. Self::from_index(self.0 + 1) } + + /// Subtracts the given delta from this index. + pub const fn saturating_sub(&self, delta: u32) -> Self { + NonHardenedChildIndex(self.0.saturating_sub(delta)) + } + + /// Adds the given delta to this index, returning a maximum possible value of + /// [`NonHardenedChildIndex::MAX`]. + pub const fn saturating_add(&self, delta: u32) -> Self { + let idx = self.0 + delta; + if idx > Self::MAX.0 { + Self::MAX + } else { + NonHardenedChildIndex(idx) + } + } } impl TryFrom for NonHardenedChildIndex { @@ -120,6 +142,21 @@ impl From for ChildNumber { } } +impl TryFrom for NonHardenedChildIndex { + type Error = (); + + fn try_from(value: DiversifierIndex) -> Result { + let idx = u32::try_from(value).map_err(|_| ())?; + NonHardenedChildIndex::from_index(idx).ok_or(()) + } +} + +impl From for DiversifierIndex { + fn from(value: NonHardenedChildIndex) -> Self { + DiversifierIndex::from(value.0) + } +} + /// A [BIP44] private key at the account path level `m/44'/'/'`. /// /// [BIP44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki