From 964ef8114371afb3ea4a53ca3c12a2e6d9a230b0 Mon Sep 17 00:00:00 2001 From: Daira-Emma Hopwood Date: Mon, 3 Jun 2024 21:14:11 +0100 Subject: [PATCH] ZIP 320 implementation work-in-progress. Co-authored-by: Kris Nuttycombe Co-authored-by: Jack Grigg Signed-off-by: Daira-Emma Hopwood --- zcash_client_backend/CHANGELOG.md | 21 +- zcash_client_backend/proto/proposal.proto | 2 + zcash_client_backend/src/data_api.rs | 73 +++- zcash_client_backend/src/data_api/error.rs | 2 + zcash_client_backend/src/data_api/wallet.rs | 353 +++++++++++------- .../src/data_api/wallet/input_selection.rs | 159 +++++++- zcash_client_backend/src/proto/proposal.rs | 2 + zcash_client_backend/src/wallet.rs | 72 +++- zcash_client_sqlite/CHANGELOG.md | 19 + zcash_client_sqlite/src/error.rs | 9 +- zcash_client_sqlite/src/lib.rs | 79 +++- zcash_client_sqlite/src/testing/pool.rs | 4 +- zcash_client_sqlite/src/wallet.rs | 304 ++++++++++++--- zcash_client_sqlite/src/wallet/init.rs | 15 + .../src/wallet/init/migrations.rs | 10 +- .../init/migrations/ephemeral_addresses.rs | 66 ++++ .../wallet/init/migrations/ufvk_support.rs | 4 +- zcash_primitives/CHANGELOG.md | 5 + zcash_primitives/src/legacy/keys.rs | 27 ++ 19 files changed, 989 insertions(+), 237 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index fb5910f140..8eb1323f5b 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -6,6 +6,23 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Notable changes +`zcash_client_backend` now supports TEX (transparent-source-only) addresses as specified +in ZIP 320. Sending to one or more TEX addresses will automatically create a multi-step +proposal that uses two transactions. This is intended to be used in conjunction with +`zcash_client_sqlite` 0.11 or later. + +In order to take advantage of this support, client wallets will need to be able to send +multiple transactions created from `zcash_client_backend::data_api::wallet::create_proposed_transactions`. +This API was added in `zcash_client_backend` 0.11.0 but previously could only return a +single transaction. + +**Note:** This feature changes the use of transparent addresses in ways that are relevant +to security and access to funds, and that may interact with other wallet behaviour. In +particular it exposes new ephemeral transparent addresses belonging to the wallet, which +need to be scanned in order to recover funds if the first transaction of the proposal is +mined but the second is not, or if someone (e.g. the TEX-address recipient) sends back +funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for details. ### Added - `zcash_client_backend::data_api`: @@ -33,7 +50,9 @@ and this library adheres to Rust's notion of - `zcash_client_backend::input_selection::GreedyInputSelectorError` has a new variant `UnsupportedTexAddress`. - `zcash_client_backend::wallet`: - - `Recipient` variants have changed. Instead of wrapping protocol-address + - `Recipient` variants have changed. It has a new `EphemeralTransparent` + variant, and an extra generic parameter giving the type of metadata about + an ephemeral transparent outpoint. Instead of wrapping protocol-address types, the `External` and `InternalAccount` variants now wrap a `zcash_address::ZcashAddress`. This simplifies the process of tracking the original address to which value was sent. diff --git a/zcash_client_backend/proto/proposal.proto b/zcash_client_backend/proto/proposal.proto index 950bb3406c..ef1c0732bc 100644 --- a/zcash_client_backend/proto/proposal.proto +++ b/zcash_client_backend/proto/proposal.proto @@ -113,6 +113,8 @@ enum FeeRule { // The proposed change outputs and fee value. message TransactionBalance { // A list of change output values. + // Any `ChangeValue`s for the transparent value pool represent ephemeral + // outputs that will each be given a unique t-address. repeated ChangeValue proposedChange = 1; // The fee to be paid by the proposed transaction, in zatoshis. uint64 feeRequired = 2; diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 2721219ae5..a3bb541354 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -88,16 +88,16 @@ use zcash_primitives::{ consensus::BlockHeight, memo::{Memo, MemoBytes}, transaction::{ - components::amount::{BalanceError, NonNegativeAmount}, + components::{ + amount::{BalanceError, NonNegativeAmount}, + OutPoint, + }, Transaction, TxId, }, }; #[cfg(feature = "transparent-inputs")] -use { - crate::wallet::TransparentAddressMetadata, - zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint}, -}; +use {crate::wallet::TransparentAddressMetadata, zcash_primitives::legacy::TransparentAddress}; #[cfg(any(test, feature = "test-dependencies"))] use zcash_primitives::consensus::NetworkUpgrade; @@ -909,6 +909,16 @@ pub trait WalletRead { ) -> Result, Self::Error> { Ok(HashMap::new()) } + + /// Returns the set of reserved ephemeral addresses controlled by this wallet. + #[cfg(feature = "transparent-inputs")] + fn get_reserved_ephemeral_addresses( + &self, + _account: Self::AccountId, + _for_sync: bool, + ) -> Result, Self::Error> { + Ok(vec![]) + } } /// The relevance of a seed to a given wallet. @@ -1239,7 +1249,7 @@ impl<'a, AccountId> SentTransaction<'a, AccountId> { /// This type is capable of representing both shielded and transparent outputs. pub struct SentTransactionOutput { output_index: usize, - recipient: Recipient, + recipient: Recipient, value: NonNegativeAmount, memo: Option, } @@ -1256,7 +1266,7 @@ impl SentTransactionOutput { /// * `memo` - the memo that was sent with this output pub fn from_parts( output_index: usize, - recipient: Recipient, + recipient: Recipient, value: NonNegativeAmount, memo: Option, ) -> Self { @@ -1278,8 +1288,8 @@ impl SentTransactionOutput { self.output_index } /// Returns the recipient address of the transaction, or the account id and - /// resulting note for wallet-internal outputs. - pub fn recipient(&self) -> &Recipient { + /// resulting note/outpoint for wallet-internal outputs. + pub fn recipient(&self) -> &Recipient { &self.recipient } /// Returns the value of the newly created output. @@ -1514,8 +1524,11 @@ pub trait WalletWrite: WalletRead { received_tx: DecryptedTransaction, ) -> Result<(), Self::Error>; - /// Saves information about a transaction that was constructed and sent by the wallet to the - /// persistent wallet store. + /// Saves information about a transaction constructed by the wallet to the persistent + /// wallet store. + /// + /// The name `store_sent_tx` is somewhat misleading; this must be called *before* the + /// transaction is sent to the network. fn store_sent_tx( &mut self, sent_tx: &SentTransaction, @@ -1535,6 +1548,26 @@ pub trait WalletWrite: WalletRead { /// /// There may be restrictions on heights to which it is possible to truncate. fn truncate_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error>; + + /// Reserves the next `n` available ephemeral addresses for the given account. + /// This cannot be undone, so as far as possible, errors associated with transaction + /// construction should have been reported before calling this method. + /// + /// To ensure that sufficient information is stored on-chain to allow recovering + /// funds sent back to any of the used addresses, a "gap limit" of 20 addresses + /// should be observed as described in [BIP 44]. + /// + /// Returns an error if there is insufficient space within the gap limit to allocate + /// the given number of addresses, or if the account identifier does not correspond + /// to a known account. + /// + /// [BIP 44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#user-content-Address_gap_limit + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + account_id: Self::AccountId, + n: u32, + ) -> Result, Self::Error>; } /// This trait describes a capability for manipulating wallet note commitment trees. @@ -1862,6 +1895,15 @@ pub mod testing { ) -> Result, Self::Error> { Ok(HashMap::new()) } + + #[cfg(feature = "transparent-inputs")] + fn get_reserved_ephemeral_addresses( + &self, + _account: Self::AccountId, + _for_sync: bool, + ) -> Result, Self::Error> { + Ok(vec![]) + } } impl WalletWrite for MockWalletDb { @@ -1924,6 +1966,15 @@ pub mod testing { ) -> Result { Ok(0) } + + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + _account_id: Self::AccountId, + _n: u32, + ) -> Result, Self::Error> { + Err(()) + } } impl WalletCommitmentTrees for MockWalletDb { diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 2c10db70ca..2fcfaf4a11 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -85,6 +85,8 @@ pub enum Error { /// An error occurred parsing the address from a payment request. Address(ConversionError<&'static str>), + /// The address associated with a record being inserted was not recognized as + /// belonging to the wallet #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index deea7e9057..996f2892aa 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -58,7 +58,7 @@ use crate::{ }; use zcash_primitives::transaction::{ builder::{BuildConfig, BuildResult, Builder}, - components::{amount::NonNegativeAmount, sapling::zip212_enforcement}, + components::{amount::NonNegativeAmount, sapling::zip212_enforcement, OutPoint}, fees::{zip317::FeeError as Zip317FeeError, FeeRule, StandardFeeRule}, Transaction, TxId, }; @@ -70,11 +70,9 @@ use zip32::Scope; #[cfg(feature = "transparent-inputs")] use { - input_selection::ShieldingSelector, - std::convert::Infallible, - zcash_keys::encoding::AddressCodec, - zcash_primitives::legacy::TransparentAddress, - zcash_primitives::transaction::components::{OutPoint, TxOut}, + input_selection::ShieldingSelector, std::convert::Infallible, + zcash_keys::encoding::AddressCodec, zcash_primitives::legacy::TransparentAddress, + zcash_primitives::transaction::components::TxOut, }; pub mod input_selection; @@ -659,40 +657,49 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - // TODO: Spending shielded outputs of prior multi-step transaction steps is not yet - // supported. Maybe support this at some point? Doing so would require a higher-level - // approach in the wallet that waits for transactions with shielded outputs to be - // mined and only then attempts to perform the next step. + type ErrorT = Error< + ::Error, + ::Error, + InputsErrT, + ::Error, + >; + + // Spending shielded outputs of prior multi-step transaction steps (either payments or change) + // is not supported. + // + // TODO: Maybe support this at some point? Doing so would require a higher-level approach in + // the wallet that waits for transactions with shielded outputs to be mined and only then + // attempts to perform the next step. for s_ref in proposal_step.prior_step_inputs() { prior_step_results.get(s_ref.step_index()).map_or_else( || { - // Return an error in case the step index doesn't match up with a step + // Return an error in case the step index doesn't match up with a step. Err(Error::Proposal(ProposalError::ReferenceError(*s_ref))) }, - |step| match s_ref.output_index() { - proposal::StepOutputIndex::Payment(i) => { - let prior_pool = step - .0 - .payment_pools() - .get(&i) - .ok_or(Error::Proposal(ProposalError::ReferenceError(*s_ref)))?; - - if matches!(prior_pool, PoolType::Shielded(_)) { - Err(Error::ProposalNotSupported) - } else { - Ok(()) + |step| { + let prior_pool = match s_ref.output_index() { + proposal::StepOutputIndex::Payment(i) => { + step.0.payment_pools().get(&i).cloned() } + proposal::StepOutputIndex::Change(i) => step + .0 + .balance() + .proposed_change() + .get(i) + .map(|change| change.output_pool()), } - proposal::StepOutputIndex::Change(_) => { - // Only shielded change is supported by zcash_client_backend, so multi-step - // transactions cannot yet spend prior transactions' change outputs. - Err(Error::ProposalNotSupported) + .ok_or(Error::Proposal(ProposalError::ReferenceError(*s_ref)))?; + + // Return an error on trying to spend a prior shielded output. + match prior_pool { + PoolType::Shielded(_) => Err(Error::ProposalNotSupported), + PoolType::Transparent => Ok(()), } }, )?; } - let account = wallet_db + let account_id = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? .ok_or(Error::KeyNotRecognized)? @@ -789,34 +796,38 @@ where }, ); + #[cfg(feature = "transparent-inputs")] + let mut has_shielded_inputs = false; + for (sapling_key, sapling_note, merkle_path) in sapling_inputs.into_iter() { builder.add_sapling_spend(&sapling_key, sapling_note.clone(), merkle_path)?; + #[cfg(feature = "transparent-inputs")] + { + has_shielded_inputs = true; + } } #[cfg(feature = "orchard")] for (orchard_note, merkle_path) in orchard_inputs.into_iter() { builder.add_orchard_spend(usk.orchard(), *orchard_note, merkle_path.into())?; + #[cfg(feature = "transparent-inputs")] + { + has_shielded_inputs = true; + } } #[cfg(feature = "transparent-inputs")] let utxos_spent = { + // FIXME: We need this to include transparent ephemeral addresses. let known_addrs = wallet_db - .get_transparent_receivers(account) + .get_transparent_receivers(account_id) .map_err(Error::DataSource)?; let mut utxos_spent: Vec = vec![]; let mut add_transparent_input = |addr: &TransparentAddress, outpoint: OutPoint, utxo: TxOut| - -> Result< - (), - Error< - ::Error, - ::Error, - InputsErrT, - FeeRuleT::Error, - >, - > { + -> Result<(), ErrorT> { let address_metadata = known_addrs .get(addr) .ok_or(Error::AddressNotRecognized(*addr))? @@ -858,19 +869,22 @@ where .clone() .convert_if_network(params.network_type())?; + // Address::Tex should not be able to occur here for a proposal made by propose_transaction. + // They can occur for a deserialized proposal and are invalid in that case. let recipient_taddr = match recipient_address { Address::Transparent(t) => Some(t), Address::Unified(uaddr) => uaddr.transparent(), _ => None, } .ok_or(Error::ProposalNotSupported)?; + let outpoint = OutPoint::new( result.transaction().txid().into(), u32::try_from( prior_step .payment_pools() .iter() - .filter(|(_, pool)| pool == &&PoolType::Transparent) + .filter(|(_, pool)| pool == &&PoolType::TRANSPARENT) .take_while(|(j, _)| j <= &&i) .count() - 1, @@ -957,99 +971,106 @@ where (payment, output_pool) }) { - let recipient_address: Address = payment - .recipient_address() - .clone() - .convert_if_network(params.network_type())?; - - match recipient_address { - Address::Unified(ua) => { - let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); + let recipient_address = payment.recipient_address(); + + let add_sapling = |builder: &mut Builder<_, _>, + meta: &mut Vec<_>, + to| + -> Result<(), ErrorT> { + let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); + builder.add_sapling_output(sapling_external_ovk, to, payment.amount(), memo.clone())?; + meta.push(( + Recipient::External(recipient_address.clone(), PoolType::SAPLING), + payment.amount(), + Some(memo), + )); + Ok(()) + }; - match output_pool { - #[cfg(not(feature = "orchard"))] - PoolType::Shielded(ShieldedProtocol::Orchard) => { - return Err(Error::ProposalNotSupported); - } - #[cfg(feature = "orchard")] - PoolType::Shielded(ShieldedProtocol::Orchard) => { - builder.add_orchard_output( - orchard_external_ovk.clone(), - *ua.orchard().expect("The mapping between payment pool and receiver is checked in step construction"), - payment.amount().into(), - memo.clone(), - )?; - orchard_output_meta.push(( - Recipient::External( - payment.recipient_address().clone(), - PoolType::Shielded(ShieldedProtocol::Orchard), - ), - payment.amount(), - Some(memo), - )); - } + #[cfg(feature = "orchard")] + let add_orchard = |builder: &mut Builder<_, _>, + meta: &mut Vec<_>, + to| + -> Result<(), ErrorT> { + let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); + builder.add_orchard_output( + orchard_external_ovk.clone(), + to, + payment.amount().into(), + memo.clone(), + )?; + meta.push(( + Recipient::External(recipient_address.clone(), PoolType::ORCHARD), + payment.amount(), + Some(memo), + )); + Ok(()) + }; - PoolType::Shielded(ShieldedProtocol::Sapling) => { - builder.add_sapling_output( - sapling_external_ovk, - *ua.sapling().expect("The mapping between payment pool and receiver is checked in step construction"), - payment.amount(), - memo.clone(), - )?; - sapling_output_meta.push(( - Recipient::External( - payment.recipient_address().clone(), - PoolType::Shielded(ShieldedProtocol::Sapling), - ), - payment.amount(), - Some(memo), - )); - } + let add_transparent = |builder: &mut Builder<_, _>, + meta: &mut Vec<_>, + to| + -> Result<(), ErrorT> { + if payment.memo().is_some() { + return Err(Error::MemoForbidden); + } else { + builder.add_transparent_output(&to, payment.amount())?; + } + meta.push(( + Recipient::External(recipient_address.clone(), PoolType::TRANSPARENT), + to, + payment.amount(), + )); + Ok(()) + }; - PoolType::Transparent => { - if payment.memo().is_some() { - return Err(Error::MemoForbidden); - } else { - builder.add_transparent_output( - ua.transparent().expect("The mapping between payment pool and receiver is checked in step construction."), - payment.amount() - )?; - } - } + match recipient_address + .clone() + .convert_if_network(params.network_type())? + { + Address::Unified(ua) => match output_pool { + #[cfg(not(feature = "orchard"))] + PoolType::Shielded(ShieldedProtocol::Orchard) => { + return Err(Error::ProposalNotSupported); } - } - Address::Sapling(addr) => { - let memo = payment.memo().map_or_else(MemoBytes::empty, |m| m.clone()); - builder.add_sapling_output( - sapling_external_ovk, - addr, - payment.amount(), - memo.clone(), - )?; - sapling_output_meta.push(( - Recipient::External(payment.recipient_address().clone(), PoolType::SAPLING), - payment.amount(), - Some(memo), - )); + #[cfg(feature = "orchard")] + PoolType::Shielded(ShieldedProtocol::Orchard) => { + let to = *ua.orchard().expect("The mapping between payment pool and receiver is checked in step construction"); + add_orchard(&mut builder, &mut orchard_output_meta, to)?; + } + PoolType::Shielded(ShieldedProtocol::Sapling) => { + let to = *ua.sapling().expect("The mapping between payment pool and receiver is checked in step construction"); + add_sapling(&mut builder, &mut sapling_output_meta, to)?; + } + PoolType::Transparent => { + let to = *ua.transparent().expect("The mapping between payment pool and receiver is checked in step construction"); + add_transparent(&mut builder, &mut transparent_output_meta, to)?; + } + }, + Address::Sapling(to) => { + add_sapling(&mut builder, &mut sapling_output_meta, to)?; } Address::Transparent(to) => { - if payment.memo().is_some() { - return Err(Error::MemoForbidden); - } else { - builder.add_transparent_output(&to, payment.amount())?; - } - transparent_output_meta.push(( - Recipient::External(payment.recipient_address().clone(), PoolType::TRANSPARENT), - to, - payment.amount(), - )); + add_transparent(&mut builder, &mut transparent_output_meta, to)?; } + #[cfg(not(feature = "transparent-inputs"))] Address::Tex(_) => { return Err(Error::ProposalNotSupported); } + #[cfg(feature = "transparent-inputs")] + Address::Tex(data) => { + if has_shielded_inputs { + return Err(Error::ProposalNotSupported); + } + let to = TransparentAddress::PublicKeyHash(data); + add_transparent(&mut builder, &mut transparent_output_meta, to)?; + } } } + #[cfg(feature = "transparent-inputs")] + let mut num_ephemeral_addresses: u32 = 0; + for change_value in proposal_step.balance().proposed_change() { let memo = change_value .memo() @@ -1065,7 +1086,7 @@ where )?; sapling_output_meta.push(( Recipient::InternalAccount { - receiving_account: account, + receiving_account: account_id, external_address: None, note: output_pool, }, @@ -1087,7 +1108,7 @@ where )?; orchard_output_meta.push(( Recipient::InternalAccount { - receiving_account: account, + receiving_account: account_id, external_address: None, note: output_pool, }, @@ -1097,7 +1118,52 @@ where } } PoolType::Transparent => { + #[cfg(not(feature = "transparent-inputs"))] return Err(Error::UnsupportedChangeType(output_pool)); + + #[cfg(feature = "transparent-inputs")] + { + num_ephemeral_addresses = num_ephemeral_addresses + .checked_add(1) + .ok_or(Error::ProposalNotSupported)?; + } + } + } + } + + // This reserves the ephemeral addresses even if transaction construction fails. + // It is not worth the complexity of being able to unreserve them, because there + // are few failure modes after this point that would allow us to do so. + #[cfg(feature = "transparent-inputs")] + let ephemeral_addresses = wallet_db + .reserve_next_n_ephemeral_addresses(account_id, num_ephemeral_addresses) + .map_err(Error::DataSource)?; + + #[cfg(feature = "transparent-inputs")] + { + assert!(ephemeral_addresses.len() == num_ephemeral_addresses.try_into().unwrap()); + + let mut addrs = ephemeral_addresses.iter(); + for change_value in proposal_step.balance().proposed_change() { + let output_pool = change_value.output_pool(); + if matches!(output_pool, PoolType::Transparent) { + // This is intended for an ephemeral transparent output, rather than a + // non-ephemeral transparent change output. That is, there should be another + // request, although we cannot check that here. + let ephemeral_address = addrs + .next() + .expect("ephemeral_addresses is the right length"); + + builder.add_transparent_output(ephemeral_address, change_value.value())?; + transparent_output_meta.push(( + Recipient::EphemeralTransparent { + receiving_account: account_id, + ephemeral_address: *ephemeral_address, + outpoint: output_pool, + }, + *ephemeral_address, + change_value.value(), + )) } } } @@ -1120,7 +1186,7 @@ where let recipient = recipient .map_internal_account_note(|pool| { - assert!(pool == PoolType::Shielded(ShieldedProtocol::Orchard)); + assert!(pool == PoolType::ORCHARD); build_result .transaction() .orchard_bundle() @@ -1150,7 +1216,7 @@ where let recipient = recipient .map_internal_account_note(|pool| { - assert!(pool == PoolType::Shielded(ShieldedProtocol::Sapling)); + assert!(pool == PoolType::SAPLING); build_result .transaction() .sapling_bundle() @@ -1169,29 +1235,26 @@ where SentTransactionOutput::from_parts(output_index, recipient, value, memo) }); - let transparent_outputs = - transparent_output_meta - .into_iter() - .map(|(recipient, addr, value)| { - let script = addr.script(); - let output_index = build_result - .transaction() - .transparent_bundle() - .and_then(|b| { - b.vout - .iter() - .enumerate() - .find(|(_, tx_out)| tx_out.script_pubkey == script) - }) - .map(|(index, _)| index) - .expect( - "An output should exist in the transaction for each transparent payment.", - ); - - SentTransactionOutput::from_parts(output_index, recipient, value, None) + let txid: [u8; 32] = build_result.transaction().txid().into(); + let num_txouts = build_result + .transaction() + .transparent_bundle() + .map_or(0, |b| b.vout.len()); + assert!(num_txouts <= u32::MAX as usize); + + let transparent_outputs = transparent_output_meta.into_iter().enumerate().map( + |(output_index, (recipient, _addr, value))| { + assert!(output_index < num_txouts); + let recipient = recipient.map_ephemeral_transparent_outpoint(|pool: PoolType| { + assert!(pool == PoolType::TRANSPARENT); + OutPoint::new(txid, output_index as u32) }); - let mut outputs = vec![]; + SentTransactionOutput::from_parts(output_index, recipient, value, None) + }, + ); + + let mut outputs: Vec::AccountId>> = vec![]; #[cfg(feature = "orchard")] outputs.extend(orchard_outputs); outputs.extend(sapling_outputs); @@ -1201,7 +1264,7 @@ where .store_sent_tx(&SentTransaction { tx: build_result.transaction(), created: time::OffsetDateTime::now_utc(), - account, + account: account_id, outputs, fee_amount: proposal_step.balance().fee_required(), #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 8ac49c92e3..49acb18d60 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -32,9 +32,16 @@ use crate::{ #[cfg(feature = "transparent-inputs")] use { - std::collections::BTreeSet, std::convert::Infallible, - zcash_primitives::legacy::TransparentAddress, - zcash_primitives::transaction::components::OutPoint, + crate::{ + fees::{ChangeValue, TransactionBalance}, + proposal::{Step, StepOutput, StepOutputIndex}, + zip321::Payment, + }, + std::collections::BTreeSet, + std::convert::Infallible, + zcash_primitives::{ + legacy::TransparentAddress, transaction::components::OutPoint, transaction::fees::zip317, + }, }; #[cfg(feature = "orchard")] @@ -364,6 +371,16 @@ where #[cfg(feature = "orchard")] let mut orchard_outputs = vec![]; let mut payment_pools = BTreeMap::new(); + + #[cfg(feature = "transparent-inputs")] + let mut tr2_transparent_outputs = vec![]; + #[cfg(feature = "transparent-inputs")] + let mut tr2_payments = vec![]; + #[cfg(feature = "transparent-inputs")] + let mut tr2_payment_pools = BTreeMap::new(); + #[cfg(feature = "transparent-inputs")] + let mut total_ephemeral_plus_fee = Some(NonNegativeAmount::ZERO); // None means overflow + for (idx, payment) in transaction_request.payments() { let recipient_address: Address = payment .recipient_address() @@ -378,6 +395,30 @@ where script_pubkey: addr.script(), }); } + #[cfg(feature = "transparent-inputs")] + Address::Tex(data) => { + let p2pkh_addr = TransparentAddress::PublicKeyHash(data); + + tr2_payment_pools.insert(*idx, PoolType::TRANSPARENT); + tr2_transparent_outputs.push(TxOut { + value: payment.amount(), + script_pubkey: p2pkh_addr.script(), + }); + tr2_payments.push( + Payment::new( + payment.recipient_address().clone(), + payment.amount(), + None, + payment.label().cloned(), + payment.message().cloned(), + payment.other_params().to_vec(), + ) + .expect("cannot fail because memo is None"), + ); + total_ephemeral_plus_fee = + total_ephemeral_plus_fee + payment.amount() + zip317::MARGINAL_FEE; + } + #[cfg(not(feature = "transparent-inputs"))] Address::Tex(_) => { return Err(InputSelectorError::Selection( GreedyInputSelectorError::UnsupportedTexAddress, @@ -417,10 +458,30 @@ where } } + #[cfg(feature = "transparent-inputs")] + let total_ephemeral_plus_fee = + total_ephemeral_plus_fee.ok_or(InputSelectorError::Selection( + GreedyInputSelectorError::Balance(BalanceError::Overflow), + ))?; + + #[cfg(feature = "transparent-inputs")] + if !tr2_transparent_outputs.is_empty() { + // Push a fake output of total_ephemeral_plus_fee to an arbitrary t-address. + // This is *only* used to compute the balance and will not appear in any + // `TransactionRequest`, so it is fine to use a burn address. This assumes + // that the fake output will have the same effect on the fee for the first + // transaction as the real ephemeral output. + transparent_outputs.push(TxOut { + value: total_ephemeral_plus_fee, + script_pubkey: TransparentAddress::PublicKeyHash([0u8; 20]).script(), + }); + } + let mut shielded_inputs = SpendableNotes::empty(); let mut prior_available = NonNegativeAmount::ZERO; let mut amount_required = NonNegativeAmount::ZERO; let mut exclude: Vec = vec![]; + // This loop is guaranteed to terminate because on each iteration we check that the amount // of funds selected is strictly increasing. The loop will either return a successful // result or the wallet will eventually run out of funds to select. @@ -434,12 +495,12 @@ where shielded_inputs.orchard_value()?, ); - // Use Sapling inputs if there are no Orchard outputs or there are not sufficient - // Orchard outputs to cover the amount required. + // Use Sapling inputs if there are no Orchard outputs or if there are insufficient + // funds from Orchard inputs to cover the amount required. let use_sapling = orchard_outputs.is_empty() || amount_required > orchard_input_total; - // Use Orchard inputs if there are insufficient Sapling funds to cover the amount - // reqiuired. + // Use Orchard inputs if there are insufficient funds from Sapling inputs to cover + // the amount required. let use_orchard = !use_sapling || amount_required > sapling_input_total; (use_sapling, use_orchard) @@ -487,18 +548,90 @@ where match balance { Ok(balance) => { - return Proposal::single_step( - transaction_request, - payment_pools, - vec![], + let fee_rule = (*self.change_strategy.fee_rule()).clone(); + let shielded_inputs = NonEmpty::from_vec(shielded_inputs.into_vec(&SimpleNoteRetention { sapling: use_sapling, #[cfg(feature = "orchard")] orchard: use_orchard, })) - .map(|notes| ShieldedInputs::from_parts(anchor_height, notes)), + .map(|notes| ShieldedInputs::from_parts(anchor_height, notes)); + + #[cfg(feature = "transparent-inputs")] + if !tr2_transparent_outputs.is_empty() { + // Construct two new `TransactionRequest`s: + // * `tr1` excludes the TEX outputs, and in their place includes + // a single additional "change" output to the transparent pool. + // * `tr2` spends from that change output to each TEX output. + + let tr1 = TransactionRequest::from_indexed( + transaction_request + .payments() + .iter() + .filter_map(|(idx, payment)| { + if tr2_payment_pools.contains_key(idx) { + None + } else { + Some((*idx, payment.clone())) + } + }) + .collect(), + ) + .expect("removing payments from a TransactionRequest preserves validity"); + + // Create a TransactionBalance for `tr1` that adds the ephemeral output + // as an extra change output. + let mut tr1_change: Vec<_> = balance.proposed_change().into(); + let ephemeral_output = + StepOutput::new(0, StepOutputIndex::Change(tr1_change.len())); + tr1_change.push(ChangeValue::transparent(total_ephemeral_plus_fee)); + let tr1_balance = + TransactionBalance::new(tr1_change, balance.fee_required()).map_err( + |_| InputSelectorError::Proposal(ProposalError::Overflow), + )?; + + let tr2 = + TransactionRequest::new(tr2_payments).expect("valid by construction"); + + let step1 = Step::from_parts( + &[], + tr1, + payment_pools, + vec![], + shielded_inputs, + vec![], + tr1_balance, + false, + ) + .map_err(InputSelectorError::Proposal)?; + + let step2 = Step::from_parts( + &[step1.clone()], + tr2, + tr2_payment_pools, + vec![], + None, + vec![ephemeral_output], + balance, + false, + ) + .map_err(InputSelectorError::Proposal)?; + + return Proposal::multi_step( + fee_rule, + target_height, + NonEmpty::from((step1, vec![step2])), + ) + .map_err(InputSelectorError::Proposal); + } + + return Proposal::single_step( + transaction_request, + payment_pools, + vec![], + shielded_inputs, balance, - (*self.change_strategy.fee_rule()).clone(), + fee_rule, target_height, false, ) diff --git a/zcash_client_backend/src/proto/proposal.rs b/zcash_client_backend/src/proto/proposal.rs index a17b83bf8b..c94aada37e 100644 --- a/zcash_client_backend/src/proto/proposal.rs +++ b/zcash_client_backend/src/proto/proposal.rs @@ -116,6 +116,8 @@ pub mod proposed_input { #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionBalance { /// A list of change output values. + /// Any `ChangeValue`s for the transparent value pool represent ephemeral + /// outputs that will each be given a unique t-address. #[prost(message, repeated, tag = "1")] pub proposed_change: ::prost::alloc::vec::Vec, /// The fee to be paid by the proposed transaction, in zatoshis. diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index 53e0be8247..dcf2f6957f 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -62,13 +62,19 @@ impl NoteId { } } -/// A type that represents the recipient of a transaction output: a recipient address (and, for -/// unified addresses, the pool to which the payment is sent) in the case of an outgoing output, or an -/// internal account ID and the pool to which funds were sent in the case of a wallet-internal -/// output. +/// A type that represents the recipient of a transaction output: +/// * a recipient address; +/// * for external unified addresses, the pool to which the payment is sent; +/// * for ephemeral transparent addresses, the internal account ID and metadata about the outpoint; +/// * for wallet-internal outputs, the internal account ID and metadata about the note. #[derive(Debug, Clone)] -pub enum Recipient { +pub enum Recipient { External(ZcashAddress, PoolType), + EphemeralTransparent { + receiving_account: AccountId, + ephemeral_address: TransparentAddress, + outpoint: O, + }, InternalAccount { receiving_account: AccountId, external_address: Option, @@ -76,10 +82,22 @@ pub enum Recipient { }, } -impl Recipient { - pub fn map_internal_account_note B>(self, f: F) -> Recipient { +impl Recipient { + pub fn map_internal_account_note B>( + self, + f: F, + ) -> Recipient { match self { Recipient::External(addr, pool) => Recipient::External(addr, pool), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint, + } => Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint, + }, Recipient::InternalAccount { receiving_account, external_address, @@ -91,12 +109,48 @@ impl Recipient { }, } } + + pub fn map_ephemeral_transparent_outpoint B>( + self, + f: F, + ) -> Recipient { + match self { + Recipient::External(addr, pool) => Recipient::External(addr, pool), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint, + } => Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint: f(outpoint), + }, + Recipient::InternalAccount { + receiving_account, + external_address, + note, + } => Recipient::InternalAccount { + receiving_account, + external_address, + note, + }, + } + } } -impl Recipient> { - pub fn internal_account_note_transpose_option(self) -> Option> { +impl Recipient, O> { + pub fn internal_account_note_transpose_option(self) -> Option> { match self { Recipient::External(addr, pool) => Some(Recipient::External(addr, pool)), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint, + } => Some(Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint, + }), Recipient::InternalAccount { receiving_account, external_address, diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index b15f86a721..893aa459c3 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -6,7 +6,26 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Notable changes +`zcash_client_sqlite` now supports TEX (transparent-source-only) addresses as specified +in ZIP 320. Sending to one or more TEX addresses will automatically create a multi-step +proposal that uses two transactions. + +In order to take advantage of this support, client wallets will need to be able to send +multiple transactions created from `zcash_client_backend::data_api::wallet::create_proposed_transactions`. +This API was added in `zcash_client_backend` 0.11.0 but previously could only return a +single transaction. + +**Note:** This feature changes the use of transparent addresses in ways that are relevant +to security and access to funds, and that may interact with other wallet behaviour. In +particular it exposes new ephemeral transparent addresses belonging to the wallet, which +need to be scanned in order to recover funds if the first transaction of the proposal is +mined but the second is not, or if someone (e.g. the TEX-address recipient) sends back +funds to those addresses. See [ZIP 320](https://zips.z.cash/zip-0320) for details. + ### Changed +- `zcash_client_sqlite::error::SqliteClientError` has a new `ReachedGapLimit` variant + when the "transparent-inputs" features is enabled. - The result of the `v_tx_outputs` SQL query could now include transparent outputs with unknown height. - MSRV is now 1.66.0. diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 2f961853c0..51d861c0a1 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -89,7 +89,7 @@ pub enum SqliteClientError { AccountIdOutOfRange, /// The address associated with a record being inserted was not recognized as - /// belonging to the wallet + /// belonging to the wallet. #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), @@ -112,6 +112,11 @@ pub enum SqliteClientError { /// An error occurred in computing wallet balance BalanceError(BalanceError), + + /// The proposal cannot be constructed until transactions with previously reserved + /// ephemeral address outputs have been mined. + #[cfg(feature = "transparent-inputs")] + ReachedGapLimit, } impl error::Error for SqliteClientError { @@ -162,6 +167,8 @@ impl fmt::Display for SqliteClientError { SqliteClientError::ChainHeightUnknown => write!(f, "Chain height unknown; please call `update_chain_tip`"), SqliteClientError::UnsupportedPoolType(t) => write!(f, "Pool type is not currently supported: {}", t), SqliteClientError::BalanceError(e) => write!(f, "Balance error: {}", e), + #[cfg(feature = "transparent-inputs")] + SqliteClientError::ReachedGapLimit => write!(f, "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined."), } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index d7bbb13166..5f9693bf72 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -88,7 +88,10 @@ use { #[cfg(feature = "transparent-inputs")] use { zcash_client_backend::wallet::TransparentAddressMetadata, - zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint}, + zcash_primitives::{ + legacy::TransparentAddress, + transaction::components::{OutPoint, TxOut}, + }, }; #[cfg(feature = "unstable")] @@ -529,6 +532,20 @@ impl, P: consensus::Parameters> WalletRead for W ) -> Result, Self::Error> { wallet::get_transparent_balances(self.conn.borrow(), &self.params, account, max_height) } + + #[cfg(feature = "transparent-inputs")] + fn get_reserved_ephemeral_addresses( + &self, + account: Self::AccountId, + for_sync: bool, + ) -> Result, Self::Error> { + wallet::get_reserved_ephemeral_addresses( + self.conn.borrow(), + &self.params, + account, + for_sync, + ) + } } impl WalletWrite for WalletDb { @@ -1079,6 +1096,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1098,6 +1116,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1128,6 +1147,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, account_id, tx_ref, output.index(), @@ -1160,6 +1180,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1179,6 +1200,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, *output.account(), tx_ref, output.index(), @@ -1210,6 +1232,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, account_id, tx_ref, output.index(), @@ -1262,6 +1285,11 @@ impl WalletWrite for WalletDb .enumerate() { if let Some(address) = txout.recipient_address() { + // TODO: we really want to only mark outputs when a transaction has been + // *reliably* mined. + #[cfg(feature = "transparent-inputs")] + wallet::mark_ephemeral_address_as_mined(wdb.conn.0, &wdb.params, &address, tx_ref).map_err(SqliteClientError::from)?; + let receiver = Receiver::Transparent(address); #[cfg(feature = "transparent-inputs")] @@ -1281,6 +1309,7 @@ impl WalletWrite for WalletDb wallet::put_sent_output( wdb.conn.0, + &wdb.params, account_id, tx_ref, output_index, @@ -1343,7 +1372,13 @@ impl WalletWrite for WalletDb } for output in sent_tx.outputs() { - wallet::insert_sent_output(wdb.conn.0, tx_ref, *sent_tx.account_id(), output)?; + wallet::insert_sent_output( + wdb.conn.0, + &wdb.params, + tx_ref, + *sent_tx.account_id(), + output, + )?; match output.recipient() { Recipient::InternalAccount { @@ -1387,6 +1422,33 @@ impl WalletWrite for WalletDb None, )?; } + #[cfg(feature = "transparent-inputs")] + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + outpoint, + } => { + wallet::put_legacy_transparent_utxo( + wdb.conn.0, + &wdb.params, + &WalletTransparentOutput::from_parts( + outpoint.clone(), + TxOut { + value: output.value(), + script_pubkey: ephemeral_address.script(), + }, + None, + ) + .expect("txout is correct"), + *receiving_account, + )?; + wallet::mark_ephemeral_address_as_used( + wdb.conn.0, + &wdb.params, + ephemeral_address, + tx_ref, + )?; + } _ => (), } } @@ -1396,9 +1458,16 @@ impl WalletWrite for WalletDb } fn truncate_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { - self.transactionally(|wdb| { - wallet::truncate_to_height(wdb.conn.0, &wdb.params, block_height) - }) + self.transactionally(|wdb| wallet::truncate_to_height(wdb, block_height)) + } + + #[cfg(feature = "transparent-inputs")] + fn reserve_next_n_ephemeral_addresses( + &mut self, + account_id: Self::AccountId, + n: u32, + ) -> Result, Self::Error> { + self.transactionally(|wdb| wallet::reserve_next_n_ephemeral_addresses(wdb, account_id, n)) } } diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 9ae749e6e6..2dd9ef4a07 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -2017,7 +2017,7 @@ pub(crate) fn data_db_truncation() { // "Rewind" to height of last scanned block (this is a no-op) st.wallet_mut() - .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h + 1)) + .transactionally(|wdb| truncate_to_height(wdb, h + 1)) .unwrap(); // Spendable balance should be unaltered @@ -2028,7 +2028,7 @@ pub(crate) fn data_db_truncation() { // Rewind so that one block is dropped st.wallet_mut() - .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h)) + .transactionally(|wdb| truncate_to_height(wdb, h)) .unwrap(); // Spendable balance should only contain the first received note; diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 228ab20c97..0928d4c961 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -68,6 +68,7 @@ use incrementalmerkletree::Retention; use rusqlite::{self, named_params, OptionalExtension}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; +use zcash_keys::encoding::encode_transparent_address_p; use zip32::fingerprint::SeedFingerprint; use std::collections::{HashMap, HashSet}; @@ -102,7 +103,7 @@ use zcash_primitives::{ memo::{Memo, MemoBytes}, merkle_tree::read_commitment_tree, transaction::{ - components::{amount::NonNegativeAmount, Amount}, + components::{amount::NonNegativeAmount, Amount, OutPoint}, Transaction, TransactionData, TxId, }, }; @@ -129,10 +130,10 @@ use { zcash_client_backend::wallet::{TransparentAddressMetadata, WalletTransparentOutput}, zcash_primitives::{ legacy::{ - keys::{IncomingViewingKey, NonHardenedChildIndex}, + keys::{EphemeralIvk, IncomingViewingKey, NonHardenedChildIndex}, Script, TransparentAddress, }, - transaction::components::{OutPoint, TxOut}, + transaction::components::TxOut, }, }; @@ -622,9 +623,7 @@ pub(crate) fn get_transparent_receivers( let ua_str: String = row.get(0)?; let di_vec: Vec = row.get(1)?; let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Diverisifier index is not an 11-byte value".to_owned(), - ) + SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) })?; di.reverse(); // BE -> LE conversion @@ -1986,22 +1985,25 @@ pub(crate) fn get_min_unspent_height( /// /// This should only be executed inside a transactional context. pub(crate) fn truncate_to_height( - conn: &rusqlite::Transaction, - params: &P, + wdb: &mut WalletDb, P>, block_height: BlockHeight, ) -> Result<(), SqliteClientError> { - let sapling_activation_height = params + let sapling_activation_height = wdb + .params .activation_height(NetworkUpgrade::Sapling) .expect("Sapling activation height must be available."); // Recall where we synced up to previously. - let last_scanned_height = conn.query_row("SELECT MAX(height) FROM blocks", [], |row| { - row.get::<_, Option>(0) - .map(|opt| opt.map_or_else(|| sapling_activation_height - 1, BlockHeight::from)) - })?; + let last_scanned_height = + wdb.conn + .0 + .query_row("SELECT MAX(height) FROM blocks", [], |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map_or_else(|| sapling_activation_height - 1, BlockHeight::from)) + })?; if block_height < last_scanned_height - PRUNING_DEPTH { - if let Some(h) = get_min_unspent_height(conn)? { + if let Some(h) = get_min_unspent_height(wdb.conn.0)? { if block_height > h { return Err(SqliteClientError::RequestedRewindInvalid(h, block_height)); } @@ -2012,12 +2014,12 @@ pub(crate) fn truncate_to_height( // truncation height, and then truncate any remaining range by setting the end // equal to the truncation height + 1. This sets our view of the chain tip back // to the retained height. - conn.execute( + wdb.conn.0.execute( "DELETE FROM scan_queue WHERE block_range_start >= :new_end_height", named_params![":new_end_height": u32::from(block_height + 1)], )?; - conn.execute( + wdb.conn.0.execute( "UPDATE scan_queue SET block_range_end = :new_end_height WHERE block_range_end > :new_end_height", @@ -2029,10 +2031,6 @@ pub(crate) fn truncate_to_height( // database. if block_height < last_scanned_height { // Truncate the note commitment trees - let mut wdb = WalletDb { - conn: SqlTransaction(conn), - params: params.clone(), - }; wdb.with_sapling_tree_mut(|tree| { tree.truncate_removing_checkpoint(&block_height).map(|_| ()) })?; @@ -2049,28 +2047,29 @@ pub(crate) fn truncate_to_height( // do not count towards spendability or transaction balalnce. // Rewind utxos. It is currently necessary to delete these because we do - // not have the full transaction data for the received output. - conn.execute( + // not have the full transaction data for the received output. FIXME: this + // does not delete utxos with an unknown height (sentinel -1); is that correct? + wdb.conn.0.execute( "DELETE FROM utxos WHERE height > ?", [u32::from(block_height)], )?; // Un-mine transactions. - conn.execute( + wdb.conn.0.execute( "UPDATE transactions SET block = NULL, tx_index = NULL WHERE block IS NOT NULL AND block > ?", [u32::from(block_height)], )?; // Now that they aren't depended on, delete un-mined blocks. - conn.execute( + wdb.conn.0.execute( "DELETE FROM blocks WHERE height > ?", [u32::from(block_height)], )?; // Delete from the nullifier map any entries with a locator referencing a block // height greater than the truncation height. - conn.execute( + wdb.conn.0.execute( "DELETE FROM tx_locator_map WHERE block_height > :block_height", named_params![":block_height": u32::from(block_height)], @@ -2256,6 +2255,149 @@ pub(crate) fn get_transparent_balances( Ok(res) } +#[cfg(feature = "transparent-inputs")] +pub(crate) fn get_ephemeral_ivk( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, +) -> Result { + use zcash_client_backend::data_api::Account; + + Ok(get_account(conn, params, account_id)? + .ok_or(SqliteClientError::AccountUnknown)? + .ufvk() + .and_then(|ufvk| ufvk.transparent()) + .ok_or(SqliteClientError::UnknownZip32Derivation)? + .derive_ephemeral_ivk()?) +} + +// Same as Bitcoin. +#[cfg(feature = "transparent-inputs")] +const GAP_LIMIT: i32 = 20; + +/// Returns a vector with all ephemeral transparent addresses potentially belonging to this wallet. +/// If `for_sync` is true, this includes addresses for an additional GAP_LIMIT indices. +#[cfg(feature = "transparent-inputs")] +pub(crate) fn get_reserved_ephemeral_addresses( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountId, + for_sync: bool, +) -> Result, SqliteClientError> { + let mut stmt = conn.prepare( + "SELECT address, index FROM ephemeral_addresses WHERE account_id = :account ORDER BY index", + )?; + let mut rows = stmt.query(named_params! { ":account": account_id.0 })?; + + let mut result = Vec::new(); + let mut first_unused_index: Option = Some(0); + while let Some(row) = rows.next()? { + let addr_str: String = row.get(0)?; + let raw_index: u32 = row.get(1)?; + first_unused_index = i32::try_from(raw_index) + .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? + .checked_add(1); + result.push(TransparentAddress::decode(params, &addr_str)?); + } + + if for_sync { + if let Some(first) = first_unused_index { + let ephemeral_ivk = get_ephemeral_ivk(conn, params, account_id)?; + + for index in first..=first.saturating_add(GAP_LIMIT - 1) { + let child_index = + NonHardenedChildIndex::from_index(index as u32).expect("valid by construction"); + result.push(ephemeral_ivk.derive_address(child_index)?); + } + } + } + Ok(result) +} + +/// Returns a vector with the next `n` previously unreserved ephemeral addresses for +/// the given account. +/// +/// # Errors +/// +/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. +/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and +/// it is not possible to derive new addresses for it. +/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses +/// within the gap limit after the last address in this account that is known to have an +/// output in a mined transaction. +/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, +/// if the limit on transparent address indices has been reached. +#[cfg(feature = "transparent-inputs")] +pub(crate) fn reserve_next_n_ephemeral_addresses( + wdb: &mut WalletDb, P>, + account_id: AccountId, + n: u32, +) -> Result, SqliteClientError> { + if n == 0 { + return Ok(vec![]); + } + assert!(n > 0); + + let ephemeral_ivk = get_ephemeral_ivk(wdb.conn.0, &wdb.params, account_id)?; + + let last_gap_index: i32 = wdb + .conn + .0 + .query_row( + "SELECT index FROM ephemeral_addresses WHERE account = :account_id AND mined_in_tx IS NOT NULL + ORDER BY index DESC LIMIT 1", + named_params![":account_id": account_id.0], + |row| row.get::<_, u32>(0), + ) + .optional()? + .map_or(Ok(-1i32), |i| + i32::try_from(i).map_err(|e| SqliteClientError::CorruptedData(e.to_string())))? + .saturating_add(GAP_LIMIT); + + let (first_index, last_index) = wdb + .conn + .0 + .query_row( + "SELECT index FROM ephemeral_addresses WHERE account = :account_id + ORDER BY index DESC LIMIT 1", + named_params![":account_id": account_id.0], + |row| row.get::<_, u32>(0), + ) + .optional()? + .map_or(Ok(-1i32), |i| { + i32::try_from(i).map_err(|e| SqliteClientError::CorruptedData(e.to_string())) + }) + .map(|i: i32| i.checked_add(1).zip(i.checked_add(n.try_into().ok()?)))? + .ok_or(SqliteClientError::AddressGeneration( + AddressGenerationError::DiversifierSpaceExhausted, + ))?; + + assert!(last_index >= first_index); + if last_index > last_gap_index { + return Err(SqliteClientError::ReachedGapLimit); + } + + // used_in_tx and mined_in_tx are initially NULL + let mut stmt_insert_ephemeral_address = wdb.conn.0.prepare_cached( + "INSERT INTO ephemeral_addresses (account_id, index, address) + VALUES (:account_id, :index, :address)", + )?; + + (first_index..=last_index) + .map(|index| { + let child_index = + NonHardenedChildIndex::from_index(index as u32).expect("valid by construction"); + let address = ephemeral_ivk.derive_address(child_index)?; + stmt_insert_ephemeral_address.execute(named_params![ + ":account_id": account_id.0, + ":index": index, + ":address": encode_transparent_address_p(&wdb.params, &address) + ])?; + Ok(address) + }) + .collect() +} + /// Returns a vector with the IDs of all accounts known to this wallet. pub(crate) fn get_account_ids( conn: &rusqlite::Connection, @@ -2491,6 +2633,32 @@ pub(crate) fn put_received_transparent_utxo( params: &P, output: &WalletTransparentOutput, ) -> Result { + if let Some(account) = find_account_for_transparent_output(conn, params, output)? { + let utxo_id = put_legacy_transparent_utxo(conn, params, output, account)?; + Ok(utxo_id) + } else { + // The UTXO was not for any of the legacy transparent addresses. + Err(SqliteClientError::AddressNotRecognized( + *output.recipient_address(), + )) + } +} + +/// Attempts to determine the account that received the given transparent output. +/// +/// The following two locations in the wallet's key tree are searched: +/// - Transparent receivers that have been generated as part of a Unified Address. +/// - "Legacy transparent addresses" (at BIP 44 address index 0 within an account). +/// FIXME: also look here in the set of ephemeral addresses we have generated. +/// +/// Returns `Ok(None)` if the transparent output's recipient address is not in any of the +/// above locations. This means the wallet considers the output "not interesting". +#[cfg(feature = "transparent-inputs")] +pub(crate) fn find_account_for_transparent_output( + conn: &rusqlite::Connection, + params: &P, + output: &WalletTransparentOutput, +) -> Result, SqliteClientError> { let address_str = output.recipient_address().encode(params); let account_id = conn .query_row( @@ -2501,7 +2669,7 @@ pub(crate) fn put_received_transparent_utxo( .optional()?; if let Some(account) = account_id { - Ok(put_legacy_transparent_utxo(conn, params, output, account)?) + Ok(Some(account)) } else { // If the UTXO is received at the legacy transparent address (at BIP 44 address // index 0 within its particular account, which we specifically ensure is returned @@ -2514,21 +2682,13 @@ pub(crate) fn put_received_transparent_utxo( .find_map( |account| match get_legacy_transparent_address(params, conn, account) { Ok(Some((legacy_taddr, _))) if &legacy_taddr == output.recipient_address() => { - Some( - put_legacy_transparent_utxo(conn, params, output, account) - .map_err(SqliteClientError::from), - ) + Some(Ok(account)) } Ok(_) => None, Err(e) => Some(Err(e)), }, ) - // The UTXO was not for any of the legacy transparent addresses. - .unwrap_or_else(|| { - Err(SqliteClientError::AddressNotRecognized( - *output.recipient_address(), - )) - }) + .transpose() } } @@ -2571,13 +2731,69 @@ pub(crate) fn put_legacy_transparent_utxo( stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } +/// If `address` is one of our ephemeral addresses, mark it as having an output +/// in a transaction that we have just created. This has no effect if `address` is +/// not one of our ephemeral addresses, although that case should not occur. +#[cfg(feature = "transparent-inputs")] +pub(crate) fn mark_ephemeral_address_as_used( + conn: &rusqlite::Connection, + params: &P, + ephemeral_address: &TransparentAddress, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let mut stmt_mark_ephemeral_address_as_used = conn.prepare_cached( + "UPDATE ephemeral_addresses SET used_in_tx = :used_in_tx WHERE address = :address", + )?; + + let sql_args = named_params![ + ":used_in_tx": &tx_ref, + ":address": encode_transparent_address_p(params, ephemeral_address), + ]; + + stmt_mark_ephemeral_address_as_used.execute(sql_args)?; + Ok(()) +} + +/// If `address` is one of our ephemeral addresses, mark it as having an output +/// in the given mined transaction (which may or may not be a transaction we sent). +/// This has no effect if `address` is not one of our ephemeral addresses. +#[cfg(feature = "transparent-inputs")] +pub(crate) fn mark_ephemeral_address_as_mined( + conn: &rusqlite::Connection, + params: &P, + address: &TransparentAddress, + tx_ref: i64, +) -> Result<(), SqliteClientError> { + let mut stmt_mark_ephemeral_address_as_mined = conn.prepare_cached( + "UPDATE ephemeral_addresses SET mined_in_tx = :mined_in_tx WHERE address = :address", + )?; + + let sql_args = named_params![ + ":mined_in_tx": &tx_ref, + ":address": encode_transparent_address_p(params, address), + ]; + + stmt_mark_ephemeral_address_as_mined.execute(sql_args)?; + Ok(()) +} + // A utility function for creation of parameters for use in `insert_sent_output` // and `put_sent_output` -fn recipient_params( - to: &Recipient, +fn recipient_params( + params: &P, + to: &Recipient, ) -> (Option, Option, PoolType) { match to { Recipient::External(addr, pool) => (Some(addr.encode()), None, *pool), + Recipient::EphemeralTransparent { + receiving_account, + ephemeral_address, + .. + } => ( + Some(encode_transparent_address_p(params, ephemeral_address)), + Some(*receiving_account), + PoolType::Transparent, + ), Recipient::InternalAccount { receiving_account, external_address, @@ -2591,8 +2807,9 @@ fn recipient_params( } /// Records information about a transaction output that your wallet created. -pub(crate) fn insert_sent_output( +pub(crate) fn insert_sent_output( conn: &rusqlite::Connection, + params: &P, tx_ref: i64, from_account: AccountId, output: &SentTransactionOutput, @@ -2606,7 +2823,7 @@ pub(crate) fn insert_sent_output( :to_address, :to_account_id, :value, :memo)", )?; - let (to_address, to_account_id, pool_type) = recipient_params(output.recipient()); + let (to_address, to_account_id, pool_type) = recipient_params(params, output.recipient()); let sql_args = named_params![ ":tx": &tx_ref, ":output_pool": &pool_code(pool_type), @@ -2635,12 +2852,13 @@ pub(crate) fn insert_sent_output( /// - If `recipient` is an internal account, `output_index` is an index into the Sapling outputs of /// the transaction. #[allow(clippy::too_many_arguments)] -pub(crate) fn put_sent_output( +pub(crate) fn put_sent_output( conn: &rusqlite::Connection, + params: &P, from_account: AccountId, tx_ref: i64, output_index: usize, - recipient: &Recipient, + recipient: &Recipient, value: NonNegativeAmount, memo: Option<&MemoBytes>, ) -> Result<(), SqliteClientError> { @@ -2659,7 +2877,7 @@ pub(crate) fn put_sent_output( memo = IFNULL(:memo, memo)", )?; - let (to_address, to_account_id, pool_type) = recipient_params(recipient); + let (to_address, to_account_id, pool_type) = recipient_params(params, recipient); let sql_args = named_params![ ":tx": &tx_ref, ":output_pool": &pool_code(pool_type), diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index ab69c69c29..d6a2100616 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -174,6 +174,10 @@ fn sqlite_client_error_to_wallet_migration_error(e: SqliteClientError) -> Wallet SqliteClientError::ChainHeightUnknown => { unreachable!("we don't call methods that require a known chain height") } + #[cfg(feature = "transparent-inputs")] + SqliteClientError::ReachedGapLimit => { + unreachable!("we don't do ephemeral address tracking") + } } } @@ -411,6 +415,17 @@ mod tests { orchard_commitment_tree_size INTEGER, sapling_output_count INTEGER, orchard_action_count INTEGER)", + "CREATE TABLE ephemeral_addresses ( + account_id INTEGER NOT NULL, + index INTEGER NOT NULL, + address TEXT NOT NULL, + used_in_tx INTEGER, + mined_in_tx INTEGER, + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), + FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), + PRIMARY KEY (account_id, index) + )", "CREATE TABLE nullifier_map ( spend_pool INTEGER NOT NULL, nf BLOB NOT NULL, diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 0cfba40e93..fa7300f205 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -3,6 +3,7 @@ mod add_transaction_views; mod add_utxo_account; mod addresses_table; mod ensure_orchard_ua_receiver; +mod ephemeral_addresses; mod full_account_ids; mod initial_setup; mod nullifier_map; @@ -61,10 +62,10 @@ pub(super) fn all_migrations( // \ \ | v_transactions_note_uniqueness // \ \ | / // -------------------- full_account_ids - // | - // orchard_received_notes - // | - // ensure_orchard_ua_receiver + // / \ + // orchard_received_notes ephemeral_addresses + // | + // ensure_orchard_ua_receiver vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -114,5 +115,6 @@ pub(super) fn all_migrations( Box::new(ensure_orchard_ua_receiver::Migration { params: params.clone(), }), + Box::new(ephemeral_addresses::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs new file mode 100644 index 0000000000..098e3c4289 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -0,0 +1,66 @@ +//! The migration that records ephemeral addresses for each account. +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::WalletMigrationError; + +use super::full_account_ids; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0e1d4274_1f8e_44e2_909d_689a4bc2967b); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [full_account_ids::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Record ephemeral addresses for each account." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // This table should include all ephemeral addresses that we will scan for + // incoming funds. Some subset of these will have been seen in a transaction, + // either because we created the transaction and are about to send it, or + // when we see an output to this address on-chain (possibly sent by another + // wallet or a previous iteration of this wallet, or by a TEX address recipient + // sending back the funds). + // + // Addresses are never removed from the table. Note that the fact that `used_in_tx` + // and `mined_in_tx` reference specific transactions is primarily a debugging aid. + // We only really care which addresses have been used, and whether we can allocate + // a new address within the gap limit. + transaction.execute_batch( + "CREATE TABLE ephemeral_addresses ( + account_id INTEGER NOT NULL, + index INTEGER NOT NULL, + address TEXT NOT NULL, + used_in_tx INTEGER, + mined_in_tx INTEGER, + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), + FOREIGN KEY (mined_in_tx) REFERENCES transactions(id_tx), + PRIMARY KEY (account_id, index) + )", + )?; + Ok(()) + } + + fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch("DROP TABLE ephemeral_addresses;")?; + Ok(()) + } +} diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs index 2d88745deb..c8d14f4225 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ufvk_support.rs @@ -7,9 +7,7 @@ use schemer_rusqlite::RusqliteMigration; use secrecy::{ExposeSecret, SecretVec}; use uuid::Uuid; -use zcash_client_backend::{ - address::Address, keys::UnifiedSpendingKey, PoolType, -}; +use zcash_client_backend::{address::Address, keys::UnifiedSpendingKey, PoolType}; use zcash_keys::keys::UnifiedAddressRequest; use zcash_primitives::{consensus, zip32::AccountId}; diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index c9fc8895ae..c2ccd0a8cf 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -6,6 +6,11 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- `zcash_primitives::legacy::keys`: + - `EphemeralIvk` + - `AccountPubKey::derive_ephemeral_ivk` + ### Changed - MSRV is now 1.66.0. diff --git a/zcash_primitives/src/legacy/keys.rs b/zcash_primitives/src/legacy/keys.rs index fbed55028e..989f091393 100644 --- a/zcash_primitives/src/legacy/keys.rs +++ b/zcash_primitives/src/legacy/keys.rs @@ -205,6 +205,14 @@ impl AccountPubKey { .map(InternalIvk) } + /// Derives the public key at the "ephemeral" path + /// `m/44'/'/'/2`. + pub fn derive_ephemeral_ivk(&self) -> Result { + self.0 + .derive_public_key(KeyIndex::Normal(2)) + .map(EphemeralIvk) + } + /// Derives the internal ovk and external ovk corresponding to this /// transparent fvk. As specified in [ZIP 316][transparent-ovk]. /// @@ -367,6 +375,25 @@ impl private::SealedChangeLevelKey for InternalIvk { impl IncomingViewingKey for InternalIvk {} +/// An incoming viewing key at the "ephemeral" path +/// `m/44'/'/'/2`. +/// +/// This allows derivation of ephemeral addresses for use within the wallet. +#[derive(Clone, Debug)] +pub struct EphemeralIvk(ExtendedPubKey); + +impl private::SealedChangeLevelKey for EphemeralIvk { + fn extended_pubkey(&self) -> &ExtendedPubKey { + &self.0 + } + + fn from_extended_pubkey(key: ExtendedPubKey) -> Self { + EphemeralIvk(key) + } +} + +impl IncomingViewingKey for EphemeralIvk {} + /// Internal outgoing viewing key used for autoshielding. pub struct InternalOvk([u8; 32]);