diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38597e615b..6263019a60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,26 +8,40 @@ on: jobs: test: name: > - Test on ${{ matrix.os }}${{ - matrix.extra_flags != 'NOT_A_PUZZLE' && format(' with --features {0}', matrix.extra_flags) || '' - }} + Test${{ + matrix.state != 'NOT_A_PUZZLE' && format(' {0}', matrix.state) || '' + }} on ${{ matrix.target }} runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.state != 'NOT_A_PUZZLE' }} strategy: matrix: - os: [ubuntu-latest-8cores, windows-latest-8cores, macOS-latest] - extra_flags: + target: + - Linux + - macOS + - Windows + state: - NOT_A_PUZZLE - - orchard - - unstable-nu6 - - zfuture + - Orchard + - NU6 + include: - - extra_flags: orchard + - target: Linux + os: ubuntu-latest-8cores + - target: macOS + os: macOS-latest + - target: Windows + os: windows-latest-8cores + + - state: Orchard + extra_flags: orchard rustflags: '--cfg zcash_unstable="orchard"' + - state: NU6 + rustflags: '--cfg zcash_unstable="nu6"' + exclude: - - os: macOS-latest - extra_flags: unstable-nu6 - - os: macOS-latest - extra_flags: zfuture + - target: macOS + state: NU6 + env: RUSTFLAGS: ${{ matrix.rustflags }} RUSTDOCFLAGS: ${{ matrix.rustflags }} @@ -37,7 +51,7 @@ jobs: - id: prepare uses: ./.github/actions/prepare with: - extra-features: ${{ matrix.extra_flags != 'NOT_A_PUZZLE' && matrix.extra_flags || '' }} + extra-features: ${{ matrix.state != 'NOT_A_PUZZLE' && matrix.extra_flags || '' }} - uses: actions/cache@v4 with: path: | @@ -63,6 +77,63 @@ jobs: - name: Verify working directory is clean run: git diff --exit-code + # States that we want to ensure can be built, but that we don't actively run tests for. + check-msrv: + name: > + Check${{ + matrix.state != 'NOT_A_PUZZLE' && format(' {0}', matrix.state) || '' + }} build on ${{ matrix.target }} + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.state != 'NOT_A_PUZZLE' }} + strategy: + matrix: + target: + - Linux + - macOS + - Windows + state: + - ZFuture + + include: + - target: Linux + os: ubuntu-latest + - target: macOS + os: macOS-latest + - target: Windows + os: windows-latest + + - state: ZFuture + rustflags: '--cfg zcash_unstable="zfuture"' + + env: + RUSTFLAGS: ${{ matrix.rustflags }} + RUSTDOCFLAGS: ${{ matrix.rustflags }} + + steps: + - uses: actions/checkout@v4 + - id: prepare + uses: ./.github/actions/prepare + with: + extra-features: ${{ matrix.state != 'NOT_A_PUZZLE' && matrix.extra_flags || '' }} + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-msrv-${{ hashFiles('**/Cargo.lock') }} + - name: Run check + run: > + cargo check + --release + --workspace + --tests + ${{ steps.prepare.outputs.feature-flags }} + - name: Verify working directory is clean + run: git diff --exit-code + build-latest: name: Latest build on ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/Cargo.lock b/Cargo.lock index 04e3cfc778..eaa12f5a36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1087,8 +1087,7 @@ dependencies = [ [[package]] name = "incrementalmerkletree" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "361c467824d4d9d4f284be4b2608800839419dccc4d4608f28345237fe354623" +source = "git+https://github.com/nuttycom/incrementalmerkletree?rev=fa147c89c6c98a03bba745538f4e68d4eaed5146#fa147c89c6c98a03bba745538f4e68d4eaed5146" dependencies = [ "either", "proptest", @@ -1476,8 +1475,7 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "orchard" version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb255c3ffdccd3c84fe9ebed72aef64fdc72e6a3e4180dd411002d47abaad42" +source = "git+https://github.com/zcash/orchard?rev=e74879dd0ad0918f4ffe0826e03905cd819981bd#e74879dd0ad0918f4ffe0826e03905cd819981bd" dependencies = [ "aes", "bitvec", @@ -2246,8 +2244,7 @@ dependencies = [ [[package]] name = "shardtree" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf20c7a2747d9083092e3a3eeb9a7ed75577ae364896bebbc5e0bdcd4e97735" +source = "git+https://github.com/nuttycom/incrementalmerkletree?rev=fa147c89c6c98a03bba745538f4e68d4eaed5146#fa147c89c6c98a03bba745538f4e68d4eaed5146" dependencies = [ "assert_matches", "bitflags 2.4.1", @@ -3007,7 +3004,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.11.0" +version = "0.11.1" dependencies = [ "assert_matches", "base64", @@ -3053,9 +3050,10 @@ dependencies = [ [[package]] name = "zcash_client_sqlite" -version = "0.9.0" +version = "0.9.1" dependencies = [ "assert_matches", + "bls12_381", "bs58", "byteorder", "document-features", @@ -3069,6 +3067,7 @@ dependencies = [ "pasta_curves", "proptest", "prost", + "rand_chacha", "rand_core", "regex", "rusqlite", diff --git a/Cargo.toml b/Cargo.toml index 612c8f7dd8..69897715e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ lazy_static = "1" assert_matches = "1.5" criterion = "0.4" proptest = "1" +rand_chacha = "0.3" rand_xorshift = "0.3" # ZIP 32 @@ -119,3 +120,8 @@ zip32 = "0.1" lto = true panic = 'abort' codegen-units = 1 + +[patch.crates-io] +orchard = { git = "https://github.com/zcash/orchard", rev = "e74879dd0ad0918f4ffe0826e03905cd819981bd" } +incrementalmerkletree = { git = "https://github.com/nuttycom/incrementalmerkletree", rev = "fa147c89c6c98a03bba745538f4e68d4eaed5146" } +shardtree = { git = "https://github.com/nuttycom/incrementalmerkletree", rev = "fa147c89c6c98a03bba745538f4e68d4eaed5146" } diff --git a/components/zcash_protocol/CHANGELOG.md b/components/zcash_protocol/CHANGELOG.md index 2d20e03cce..362b921850 100644 --- a/components/zcash_protocol/CHANGELOG.md +++ b/components/zcash_protocol/CHANGELOG.md @@ -11,6 +11,11 @@ and this library adheres to Rust's notion of - `zcash_protocol::memo`: - `impl TryFrom<&MemoBytes> for Memo` +### Removed +- `unstable-nu6` and `zfuture` feature flags (use `--cfg zcash_unstable=\"nu6\"` + or `--cfg zcash_unstable=\"zfuture\"` in `RUSTFLAGS` and `RUSTDOCFLAGS` + instead). + ## [0.1.0] - 2024-03-06 The entries below are relative to the `zcash_primitives` crate as of the tag `zcash_primitives-0.14.0`. diff --git a/components/zcash_protocol/Cargo.toml b/components/zcash_protocol/Cargo.toml index 7a8114dad1..e85b95834f 100644 --- a/components/zcash_protocol/Cargo.toml +++ b/components/zcash_protocol/Cargo.toml @@ -45,14 +45,3 @@ test-dependencies = [ ## Exposes support for working with a local consensus (e.g. regtest). local-consensus = [] - -#! ### Experimental features -#! -#! ⚠️ Enabling these features will likely make your code incompatible with current Zcash -#! consensus rules! - -## Exposes the in-development NU6 features. -unstable-nu6 = [] - -## Exposes early in-development features that are not yet planned for any network upgrade. -zfuture = [] diff --git a/components/zcash_protocol/src/consensus.rs b/components/zcash_protocol/src/consensus.rs index 7793b15f63..32af0be70b 100644 --- a/components/zcash_protocol/src/consensus.rs +++ b/components/zcash_protocol/src/consensus.rs @@ -354,9 +354,9 @@ impl Parameters for MainNetwork { NetworkUpgrade::Heartwood => Some(BlockHeight(903_000)), NetworkUpgrade::Canopy => Some(BlockHeight(1_046_400)), NetworkUpgrade::Nu5 => Some(BlockHeight(1_687_104)), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6 => None, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] NetworkUpgrade::ZFuture => None, } } @@ -384,9 +384,9 @@ impl Parameters for TestNetwork { NetworkUpgrade::Heartwood => Some(BlockHeight(903_800)), NetworkUpgrade::Canopy => Some(BlockHeight(1_028_500)), NetworkUpgrade::Nu5 => Some(BlockHeight(1_842_420)), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6 => None, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] NetworkUpgrade::ZFuture => None, } } @@ -452,14 +452,14 @@ pub enum NetworkUpgrade { /// The [Nu6] network upgrade. /// /// [Nu6]: https://z.cash/upgrade/nu6/ - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] Nu6, /// The ZFUTURE network upgrade. /// /// This upgrade is expected never to activate on mainnet; /// it is intended for use in integration testing of functionality /// that is a candidate for integration in a future network upgrade. - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] ZFuture, } @@ -474,9 +474,9 @@ impl fmt::Display for NetworkUpgrade { NetworkUpgrade::Heartwood => write!(f, "Heartwood"), NetworkUpgrade::Canopy => write!(f, "Canopy"), NetworkUpgrade::Nu5 => write!(f, "Nu5"), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6 => write!(f, "Nu6"), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] NetworkUpgrade::ZFuture => write!(f, "ZFUTURE"), } } @@ -491,9 +491,9 @@ impl NetworkUpgrade { NetworkUpgrade::Heartwood => BranchId::Heartwood, NetworkUpgrade::Canopy => BranchId::Canopy, NetworkUpgrade::Nu5 => BranchId::Nu5, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6 => BranchId::Nu6, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] NetworkUpgrade::ZFuture => BranchId::ZFuture, } } @@ -510,7 +510,7 @@ const UPGRADES_IN_ORDER: &[NetworkUpgrade] = &[ NetworkUpgrade::Heartwood, NetworkUpgrade::Canopy, NetworkUpgrade::Nu5, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6, ]; @@ -549,11 +549,11 @@ pub enum BranchId { /// The consensus rules deployed by [`NetworkUpgrade::Nu5`]. Nu5, /// The consensus rules deployed by [`NetworkUpgrade::Nu6`]. - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] Nu6, /// Candidates for future consensus rules; this branch will never /// activate on mainnet. - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] ZFuture, } @@ -571,9 +571,9 @@ impl TryFrom for BranchId { 0xf5b9_230b => Ok(BranchId::Heartwood), 0xe9ff_75a6 => Ok(BranchId::Canopy), 0xc2d6_d0b4 => Ok(BranchId::Nu5), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] 0xc8e7_1055 => Ok(BranchId::Nu6), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] 0xffff_ffff => Ok(BranchId::ZFuture), _ => Err("Unknown consensus branch ID"), } @@ -590,9 +590,9 @@ impl From for u32 { BranchId::Heartwood => 0xf5b9_230b, BranchId::Canopy => 0xe9ff_75a6, BranchId::Nu5 => 0xc2d6_d0b4, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] BranchId::Nu6 => 0xc8e7_1055, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] BranchId::ZFuture => 0xffff_ffff, } } @@ -658,15 +658,15 @@ impl BranchId { .activation_height(NetworkUpgrade::Canopy) .map(|lower| (lower, params.activation_height(NetworkUpgrade::Nu5))), BranchId::Nu5 => params.activation_height(NetworkUpgrade::Nu5).map(|lower| { - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let upper = params.activation_height(NetworkUpgrade::ZFuture); - #[cfg(not(feature = "zfuture"))] + #[cfg(not(zcash_unstable = "zfuture"))] let upper = None; (lower, upper) }), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] BranchId::Nu6 => None, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] BranchId::ZFuture => params .activation_height(NetworkUpgrade::ZFuture) .map(|lower| (lower, None)), @@ -694,9 +694,9 @@ pub mod testing { BranchId::Heartwood, BranchId::Canopy, BranchId::Nu5, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] BranchId::Nu6, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] BranchId::ZFuture, ]) } diff --git a/components/zcash_protocol/src/local_consensus.rs b/components/zcash_protocol/src/local_consensus.rs index 994154240c..5d277cf1be 100644 --- a/components/zcash_protocol/src/local_consensus.rs +++ b/components/zcash_protocol/src/local_consensus.rs @@ -26,10 +26,6 @@ use crate::consensus::{BlockHeight, NetworkType, NetworkUpgrade, Parameters}; /// heartwood: Some(BlockHeight::from_u32(1)), /// canopy: Some(BlockHeight::from_u32(1)), /// nu5: Some(BlockHeight::from_u32(1)), -/// #[cfg(feature = "unstable-nu6")] -/// nu6: Some(BlockHeight::from_u32(1)), -/// #[cfg(feature = "zfuture")] -/// z_future: Some(BlockHeight::from_u32(1)), /// }; /// ``` /// @@ -41,9 +37,9 @@ pub struct LocalNetwork { pub heartwood: Option, pub canopy: Option, pub nu5: Option, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] pub nu6: Option, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub z_future: Option, } @@ -61,9 +57,9 @@ impl Parameters for LocalNetwork { NetworkUpgrade::Heartwood => self.heartwood, NetworkUpgrade::Canopy => self.canopy, NetworkUpgrade::Nu5 => self.nu5, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6 => self.nu6, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] NetworkUpgrade::ZFuture => self.z_future, } } @@ -85,9 +81,9 @@ mod tests { let expected_heartwood = BlockHeight::from_u32(4); let expected_canopy = BlockHeight::from_u32(5); let expected_nu5 = BlockHeight::from_u32(6); - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] let expected_nu6 = BlockHeight::from_u32(7); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let expected_z_future = BlockHeight::from_u32(7); let regtest = LocalNetwork { @@ -97,9 +93,9 @@ mod tests { heartwood: Some(expected_heartwood), canopy: Some(expected_canopy), nu5: Some(expected_nu5), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] nu6: Some(expected_nu6), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] z_future: Some(expected_z_future), }; @@ -109,9 +105,9 @@ mod tests { assert!(regtest.is_nu_active(NetworkUpgrade::Heartwood, expected_heartwood)); assert!(regtest.is_nu_active(NetworkUpgrade::Canopy, expected_canopy)); assert!(regtest.is_nu_active(NetworkUpgrade::Nu5, expected_nu5)); - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] assert!(regtest.is_nu_active(NetworkUpgrade::Nu6, expected_nu6)); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] assert!(!regtest.is_nu_active(NetworkUpgrade::ZFuture, expected_nu5)); } @@ -123,9 +119,9 @@ mod tests { let expected_heartwood = BlockHeight::from_u32(4); let expected_canopy = BlockHeight::from_u32(5); let expected_nu5 = BlockHeight::from_u32(6); - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] let expected_nu6 = BlockHeight::from_u32(7); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let expected_z_future = BlockHeight::from_u32(7); let regtest = LocalNetwork { @@ -135,9 +131,9 @@ mod tests { heartwood: Some(expected_heartwood), canopy: Some(expected_canopy), nu5: Some(expected_nu5), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] nu6: Some(expected_nu6), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] z_future: Some(expected_z_future), }; @@ -165,7 +161,7 @@ mod tests { regtest.activation_height(NetworkUpgrade::Nu5), Some(expected_nu5) ); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] assert_eq!( regtest.activation_height(NetworkUpgrade::ZFuture), Some(expected_z_future) @@ -180,9 +176,9 @@ mod tests { let expected_heartwood = BlockHeight::from_u32(4); let expected_canopy = BlockHeight::from_u32(5); let expected_nu5 = BlockHeight::from_u32(6); - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] let expected_nu6 = BlockHeight::from_u32(7); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let expected_z_future = BlockHeight::from_u32(7); let regtest = LocalNetwork { @@ -192,9 +188,9 @@ mod tests { heartwood: Some(expected_heartwood), canopy: Some(expected_canopy), nu5: Some(expected_nu5), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] nu6: Some(expected_nu6), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] z_future: Some(expected_z_future), }; diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 28c500e3de..7aa804d5f4 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -13,6 +13,7 @@ and this library adheres to Rust's notion of changes related to `Orchard` below are introduced under this feature flag. - `zcash_client_backend::data_api`: + - `Account` - `AccountBalance::with_orchard_balance_mut` - `AccountBirthday::orchard_frontier` - `BlockMetadata::orchard_tree_size` @@ -22,6 +23,7 @@ and this library adheres to Rust's notion of - `SentTransaction::new` - `ORCHARD_SHARD_HEIGHT` - `BlockMetadata::orchard_tree_size` + - `WalletSummary::next_orchard_subtree_index` - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}` - `zcash_client_backend::fees`: - `orchard` @@ -48,7 +50,11 @@ and this library adheres to Rust's notion of - Arguments to `BlockMetadata::from_parts` have changed. - Arguments to `ScannedBlock::from_parts` have changed. - Changes to the `WalletRead` trait: - - Added `get_orchard_nullifiers` + - Added `Account` associated type. + - Added `get_orchard_nullifiers` method. + - `get_account_for_ufvk` now returns an `Self::Account` instead of a bare + `AccountId` + - Added `get_seed_account` method. - Changes to the `InputSource` trait: - `select_spendable_notes` now takes its `target_value` argument as a `NonNegativeAmount`. Also, the values of the returned map are also @@ -64,6 +70,8 @@ and this library adheres to Rust's notion of - `fn put_orchard_subtree_roots` - Added method `WalletRead::validate_seed` - Removed `Error::AccountNotFound` variant. + - `WalletSummary::new` now takes an additional `next_orchard_subtree_index` + argument when the `orchard` feature flag is enabled. - `zcash_client_backend::decrypt`: - Fields of `DecryptedOutput` are now private. Use `DecryptedOutput::new` and the newly provided accessors instead. @@ -73,6 +81,10 @@ and this library adheres to Rust's notion of constraint on its `` parameter has been strengthened to `Copy`. - `zcash_client_backend::fees`: - Arguments to `ChangeStrategy::compute_balance` have changed. + - `ChangeError::DustInputs` now has an `orchard` field behind the `orchard` + feature flag. +- `zcash_client_backend::proto`: + - `ProposalDecodingError` has a new variant `TransparentMemo`. - `zcash_client_backend::zip321::render::amount_str` now takes a `NonNegativeAmount` rather than a signed `Amount` as its argument. - `zcash_client_backend::zip321::parse::parse_amount` now parses a @@ -89,6 +101,11 @@ and this library adheres to Rust's notion of allowed amounts having a decimal point but no decimal value to be parsed as valid. +## [0.11.1] - 2024-03-09 + +### Fixed +- Documentation now correctly builds with all feature flags. + ## [0.11.0] - 2024-03-01 ### Added diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index a9dce2f686..41a6639a37 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_backend" description = "APIs for creating shielded Zcash light clients" -version = "0.11.0" +version = "0.11.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " @@ -21,7 +21,16 @@ exclude = ["*.proto"] development = ["zcash_proofs"] [package.metadata.docs.rs] -all-features = true +# Manually specify features while `orchard` is not in the public API. +#all-features = true +features = [ + "lightwalletd-tonic", + "transparent-inputs", + "test-dependencies", + "unstable", + "unstable-serialization", + "unstable-spanning-tree", +] rustdoc-args = ["--cfg", "docsrs"] [dependencies] diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 362c2bf113..be91777667 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -66,8 +66,12 @@ use std::{ use incrementalmerkletree::{frontier::Frontier, Retention}; use secrecy::SecretVec; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; +use zcash_keys::keys::HdSeedFingerprint; -use self::{chain::CommitmentTreeRoot, scanning::ScanRange}; +use self::{ + chain::{ChainState, CommitmentTreeRoot}, + scanning::ScanRange, +}; use crate::{ address::UnifiedAddress, decrypt::DecryptedOutput, @@ -92,6 +96,9 @@ use { zcash_primitives::{legacy::TransparentAddress, transaction::components::OutPoint}, }; +#[cfg(feature = "test-dependencies")] +use zcash_primitives::consensus::NetworkUpgrade; + pub mod chain; pub mod error; pub mod scanning; @@ -308,6 +315,35 @@ impl AccountBalance { } } +/// A set of capabilities that a client account must provide. +pub trait Account { + /// Returns the unique identifier for the account. + fn id(&self) -> AccountId; + + /// Returns the UFVK that the wallet backend has stored for the account, if any. + fn ufvk(&self) -> Option<&UnifiedFullViewingKey>; +} + +impl Account for (A, UnifiedFullViewingKey) { + fn id(&self) -> A { + self.0 + } + + fn ufvk(&self) -> Option<&UnifiedFullViewingKey> { + Some(&self.1) + } +} + +impl Account for (A, Option) { + fn id(&self) -> A { + self.0 + } + + fn ufvk(&self) -> Option<&UnifiedFullViewingKey> { + self.1.as_ref() + } +} + /// A polymorphic ratio type, usually used for rational numbers. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Ratio { @@ -350,6 +386,8 @@ pub struct WalletSummary { fully_scanned_height: BlockHeight, scan_progress: Option>, next_sapling_subtree_index: u64, + #[cfg(feature = "orchard")] + next_orchard_subtree_index: u64, } impl WalletSummary { @@ -359,14 +397,17 @@ impl WalletSummary { chain_tip_height: BlockHeight, fully_scanned_height: BlockHeight, scan_progress: Option>, - next_sapling_subtree_idx: u64, + next_sapling_subtree_index: u64, + #[cfg(feature = "orchard")] next_orchard_subtree_index: u64, ) -> Self { Self { account_balances, chain_tip_height, fully_scanned_height, scan_progress, - next_sapling_subtree_index: next_sapling_subtree_idx, + next_sapling_subtree_index, + #[cfg(feature = "orchard")] + next_orchard_subtree_index, } } @@ -402,6 +443,13 @@ impl WalletSummary { self.next_sapling_subtree_index } + /// Returns the Orchard subtree index that should start the next range of subtree + /// roots passed to [`WalletCommitmentTrees::put_orchard_subtree_roots`]. + #[cfg(feature = "orchard")] + pub fn next_orchard_subtree_index(&self) -> u64 { + self.next_orchard_subtree_index + } + /// Returns whether or not wallet scanning is complete. pub fn is_synced(&self) -> bool { self.chain_tip_height == self.fully_scanned_height @@ -412,7 +460,7 @@ impl WalletSummary { /// belonging to a wallet. pub trait InputSource { /// The type of errors produced by a wallet backend. - type Error; + type Error: Debug; /// Backend-specific account identifier. /// @@ -483,7 +531,7 @@ pub trait InputSource { /// be abstracted away from any particular data storage substrate. pub trait WalletRead { /// The type of errors that may be generated when querying a wallet data store. - type Error; + type Error: Debug; /// The type of the account identifier. /// @@ -492,6 +540,9 @@ pub trait WalletRead { /// will be interpreted as belonging to that account. type AccountId: Copy + Debug + Eq + Hash; + /// The concrete account type used by this wallet backend. + type Account: Account; + /// Verifies that the given seed corresponds to the viewing key for the specified account. /// /// Returns: @@ -602,11 +653,19 @@ pub trait WalletRead { &self, ) -> Result, Self::Error>; - /// Returns the account id corresponding to a given [`UnifiedFullViewingKey`], if any. + /// Returns the account corresponding to a given [`UnifiedFullViewingKey`], if any. fn get_account_for_ufvk( &self, ufvk: &UnifiedFullViewingKey, - ) -> Result, Self::Error>; + ) -> Result, Self::Error>; + + /// Returns the account corresponding to a given [`HdSeedFingerprint`] and + /// [`zip32::AccountId`], if any. + fn get_seed_account( + &self, + seed: &HdSeedFingerprint, + account_id: zip32::AccountId, + ) -> Result, Self::Error>; /// Returns the wallet balances and sync status for an account given the specified minimum /// number of confirmations, or `Ok(None)` if the wallet has no balance data available. @@ -1154,25 +1213,37 @@ impl AccountBirthday { } #[cfg(feature = "test-dependencies")] - /// Constructs a new [`AccountBirthday`] at Sapling activation, with no - /// "recover until" height. + /// Constructs a new [`AccountBirthday`] at the given network upgrade's activation, + /// with no "recover until" height. /// /// # Panics /// - /// Panics if the Sapling activation height is not set. - pub fn from_sapling_activation( + /// Panics if the activation height for the given network upgrade is not set. + pub fn from_activation( params: &P, + network_upgrade: NetworkUpgrade, ) -> AccountBirthday { - use zcash_primitives::consensus::NetworkUpgrade; - AccountBirthday::from_parts( - params.activation_height(NetworkUpgrade::Sapling).unwrap(), + params.activation_height(network_upgrade).unwrap(), Frontier::empty(), #[cfg(feature = "orchard")] Frontier::empty(), None, ) } + + #[cfg(feature = "test-dependencies")] + /// Constructs a new [`AccountBirthday`] at Sapling activation, with no + /// "recover until" height. + /// + /// # Panics + /// + /// Panics if the Sapling activation height is not set. + pub fn from_sapling_activation( + params: &P, + ) -> AccountBirthday { + Self::from_activation(params, NetworkUpgrade::Sapling) + } } /// This trait encapsulates the write capabilities required to update stored @@ -1232,9 +1303,15 @@ pub trait WalletWrite: WalletRead { /// along with the note commitments that were detected when scanning the block for transactions /// pertaining to this wallet. /// - /// `blocks` must be sequential, in order of increasing block height - fn put_blocks(&mut self, blocks: Vec>) - -> Result<(), Self::Error>; + /// ### Arguments + /// - `from_state` must be the chain state for the block height prior to the first + /// block in `blocks`. + /// - `blocks` must be sequential, in order of increasing block height. + fn put_blocks( + &mut self, + from_state: &ChainState, + blocks: Vec>, + ) -> Result<(), Self::Error>; /// Updates the wallet's view of the blockchain. /// @@ -1285,7 +1362,8 @@ pub trait WalletWrite: WalletRead { /// At present, this only serves the Sapling protocol, but it will be modified to /// also provide operations related to Orchard note commitment trees in the future. pub trait WalletCommitmentTrees { - type Error; + type Error: Debug; + /// The type of the backing [`ShardStore`] for the Sapling note commitment tree. type SaplingShardStore<'a>: ShardStore< H = sapling::Node, @@ -1356,6 +1434,7 @@ pub mod testing { use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree}; use std::{collections::HashMap, convert::Infallible, num::NonZeroU32}; + use zcash_keys::keys::HdSeedFingerprint; use zcash_primitives::{ block::BlockHash, @@ -1372,9 +1451,11 @@ pub mod testing { }; use super::{ - chain::CommitmentTreeRoot, scanning::ScanRange, AccountBirthday, BlockMetadata, - DecryptedTransaction, InputSource, NullifierQuery, ScannedBlock, SentTransaction, - WalletCommitmentTrees, WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT, + chain::{ChainState, CommitmentTreeRoot}, + scanning::ScanRange, + AccountBirthday, BlockMetadata, DecryptedTransaction, InputSource, NullifierQuery, + ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletSummary, + WalletWrite, SAPLING_SHARD_HEIGHT, }; #[cfg(feature = "transparent-inputs")] @@ -1438,6 +1519,7 @@ pub mod testing { impl WalletRead for MockWalletDb { type Error = (); type AccountId = u32; + type Account = (Self::AccountId, UnifiedFullViewingKey); fn validate_seed( &self, @@ -1523,7 +1605,15 @@ pub mod testing { fn get_account_for_ufvk( &self, _ufvk: &UnifiedFullViewingKey, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { + Ok(None) + } + + fn get_seed_account( + &self, + _seed: &HdSeedFingerprint, + _account_id: zip32::AccountId, + ) -> Result, Self::Error> { Ok(None) } @@ -1605,6 +1695,7 @@ pub mod testing { #[allow(clippy::type_complexity)] fn put_blocks( &mut self, + _from_state: &ChainState, _blocks: Vec>, ) -> Result<(), Self::Error> { Ok(()) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 5cd911c522..cb4e14e403 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -58,9 +58,12 @@ //! // the first element of the vector of suggested ranges. //! match scan_ranges.first() { //! Some(scan_range) if scan_range.priority() == ScanPriority::Verify => { +//! // Download the chain state for the block prior to the start of the range you want +//! // to scan. +//! let chain_state = unimplemented!("get_chain_state(scan_range.block_range().start - 1)?;"); //! // Download the blocks in `scan_range` into the block source, overwriting any //! // existing blocks in this range. -//! unimplemented!(); +//! unimplemented!("cache_blocks(scan_range)?;"); //! //! // Scan the downloaded blocks //! let scan_result = scan_cached_blocks( @@ -68,6 +71,7 @@ //! &block_source, //! &mut wallet_db, //! scan_range.block_range().start, +//! chain_state, //! scan_range.len() //! ); //! @@ -118,6 +122,9 @@ //! // encountered, this process should be repeated starting at step (3). //! let scan_ranges = wallet_db.suggest_scan_ranges().map_err(Error::Wallet)?; //! for scan_range in scan_ranges { +//! // Download the chain state for the block prior to the start of the range you want +//! // to scan. +//! let chain_state = unimplemented!("get_chain_state(scan_range.block_range().start - 1)?;"); //! // Download the blocks in `scan_range` into the block source. While in this example this //! // step is performed in-line, it's fine for the download of scan ranges to be asynchronous //! // and for the scanner to process the downloaded ranges as they become available in a @@ -125,7 +132,7 @@ //! // appropriate, and for ranges with priority `Historic` it can be useful to download and //! // scan the range in reverse order (to discover more recent unspent notes sooner), or from //! // the start and end of the range inwards. -//! unimplemented!(); +//! unimplemented!("cache_blocks(scan_range)?;"); //! //! // Scan the downloaded blocks. //! let scan_result = scan_cached_blocks( @@ -133,6 +140,7 @@ //! &block_source, //! &mut wallet_db, //! scan_range.block_range().start, +//! chain_state, //! scan_range.len() //! )?; //! @@ -145,6 +153,7 @@ use std::ops::Range; +use incrementalmerkletree::frontier::Frontier; use subtle::ConditionallySelectable; use zcash_primitives::consensus::{self, BlockHeight}; @@ -278,12 +287,78 @@ impl ScanSummary { } } +/// The final note commitment tree state for each shielded pool, as of a particular block height. +#[derive(Debug, Clone)] +pub struct ChainState { + block_height: BlockHeight, + final_sapling_tree: Frontier, + #[cfg(feature = "orchard")] + final_orchard_tree: + Frontier, +} + +impl ChainState { + /// Construct a new empty chain state. + pub fn empty(block_height: BlockHeight) -> Self { + Self { + block_height, + final_sapling_tree: Frontier::empty(), + #[cfg(feature = "orchard")] + final_orchard_tree: Frontier::empty(), + } + } + + /// Construct a new [`ChainState`] from its constituent parts. + pub fn new( + block_height: BlockHeight, + final_sapling_tree: Frontier, + #[cfg(feature = "orchard")] final_orchard_tree: Frontier< + orchard::tree::MerkleHashOrchard, + { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 }, + >, + ) -> Self { + Self { + block_height, + final_sapling_tree, + #[cfg(feature = "orchard")] + final_orchard_tree, + } + } + + /// Returns the block height to which this chain state applies. + pub fn block_height(&self) -> BlockHeight { + self.block_height + } + + /// Returns the frontier of the Sapling note commitment tree as of the end of the block at + /// [`Self::block_height`]. + pub fn final_sapling_tree( + &self, + ) -> &Frontier { + &self.final_sapling_tree + } + + /// Returns the frontier of the Orchard note commitment tree as of the end of the block at + /// [`Self::block_height`]. + #[cfg(feature = "orchard")] + pub fn final_orchard_tree( + &self, + ) -> &Frontier + { + &self.final_orchard_tree + } +} + /// Scans at most `limit` blocks from the provided block source for in order to find transactions /// received by the accounts tracked in the provided wallet database. /// /// This function will return after scanning at most `limit` new blocks, to enable the caller to /// update their UI with scanning progress. Repeatedly calling this function with `from_height == /// None` will process sequential ranges of blocks. +/// +/// ## Panics +/// +/// This method will panic if `from_height != from_state.block_height() + 1`. #[tracing::instrument(skip(params, block_source, data_db))] #[allow(clippy::type_complexity)] pub fn scan_cached_blocks( @@ -291,6 +366,7 @@ pub fn scan_cached_blocks( block_source: &BlockSourceT, data_db: &mut DbT, from_height: BlockHeight, + from_state: &ChainState, limit: usize, ) -> Result> where @@ -299,6 +375,8 @@ where DbT: WalletWrite, ::AccountId: ConditionallySelectable + Default + Send + 'static, { + assert_eq!(from_height, from_state.block_height + 1); + // Fetch the UnifiedFullViewingKeys we are tracking let account_ufvks = data_db .get_unified_full_viewing_keys() @@ -392,7 +470,9 @@ where }, )?; - data_db.put_blocks(scanned_blocks).map_err(Error::Wallet)?; + data_db + .put_blocks(from_state, scanned_blocks) + .map_err(Error::Wallet)?; Ok(scan_summary) } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index c2bc244626..aa73bd3f8c 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -45,8 +45,8 @@ use super::InputSource; use crate::{ address::Address, data_api::{ - error::Error, SentTransaction, SentTransactionOutput, WalletCommitmentTrees, WalletRead, - WalletWrite, + error::Error, Account, SentTransaction, SentTransactionOutput, WalletCommitmentTrees, + WalletRead, WalletWrite, }, decrypt_transaction, fees::{self, DustOutputPolicy}, @@ -269,7 +269,7 @@ where wallet_db, params, StandardFeeRule::PreZip313, - account, + account.id(), min_confirmations, to, amount, @@ -380,7 +380,7 @@ where let proposal = propose_transfer( wallet_db, params, - account, + account.id(), input_selector, request, min_confirmations, @@ -694,7 +694,8 @@ where let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? - .ok_or(Error::KeyNotRecognized)?; + .ok_or(Error::KeyNotRecognized)? + .id(); let (sapling_anchor, sapling_inputs) = if proposal_step.involves(PoolType::Shielded(ShieldedProtocol::Sapling)) { diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 1627698bab..23c7808c73 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -403,8 +403,9 @@ where ::sapling::builder::BundleType::DEFAULT, &shielded_inputs .iter() + .cloned() .filter_map(|i| { - i.clone().traverse_opt(|wn| match wn { + i.traverse_opt(|wn| match wn { Note::Sapling(n) => Some(n), #[cfg(feature = "orchard")] _ => None, @@ -445,8 +446,15 @@ where ) .map_err(InputSelectorError::Proposal); } - Err(ChangeError::DustInputs { mut sapling, .. }) => { + Err(ChangeError::DustInputs { + mut sapling, + #[cfg(feature = "orchard")] + mut orchard, + .. + }) => { exclude.append(&mut sapling); + #[cfg(feature = "orchard")] + exclude.append(&mut orchard); } Err(ChangeError::InsufficientFunds { required, .. }) => { amount_required = required; diff --git a/zcash_client_backend/src/decrypt.rs b/zcash_client_backend/src/decrypt.rs index fa197ad050..0db6c94813 100644 --- a/zcash_client_backend/src/decrypt.rs +++ b/zcash_client_backend/src/decrypt.rs @@ -185,7 +185,7 @@ pub fn decrypt_transaction<'a, P: consensus::Parameters, AccountId: Copy>( .iter() .enumerate() .flat_map(move |(index, action)| { - let domain = OrchardDomain::for_nullifier(*action.nullifier()); + let domain = OrchardDomain::for_action(action); let account = account; try_note_decryption(&domain, &ivk_external, action) .map(|ret| (ret, TransferType::Incoming)) diff --git a/zcash_client_backend/src/fees.rs b/zcash_client_backend/src/fees.rs index 6c512a2043..9106a3883a 100644 --- a/zcash_client_backend/src/fees.rs +++ b/zcash_client_backend/src/fees.rs @@ -149,6 +149,9 @@ pub enum ChangeError { transparent: Vec, /// The identifiers for Sapling inputs having no current economic value sapling: Vec, + /// The identifiers for Orchard inputs having no current economic value + #[cfg(feature = "orchard")] + orchard: Vec, }, /// An error occurred that was specific to the change selection strategy in use. StrategyError(E), @@ -169,9 +172,13 @@ impl ChangeError { ChangeError::DustInputs { transparent, sapling, + #[cfg(feature = "orchard")] + orchard, } => ChangeError::DustInputs { transparent, sapling, + #[cfg(feature = "orchard")] + orchard, }, ChangeError::StrategyError(e) => ChangeError::StrategyError(f(e)), ChangeError::BundleError(e) => ChangeError::BundleError(e), @@ -194,10 +201,21 @@ impl fmt::Display for ChangeError { ChangeError::DustInputs { transparent, sapling, + #[cfg(feature = "orchard")] + orchard, } => { + #[cfg(feature = "orchard")] + let orchard_len = orchard.len(); + #[cfg(not(feature = "orchard"))] + let orchard_len = 0; + // we can't encode the UA to its string representation because we // don't have network parameters here - write!(f, "Insufficient funds: {} dust inputs were present, but would cost more to spend than they are worth.", transparent.len() + sapling.len()) + write!( + f, + "Insufficient funds: {} dust inputs were present, but would cost more to spend than they are worth.", + transparent.len() + sapling.len() + orchard_len, + ) } ChangeError::StrategyError(err) => { write!(f, "{}", err) diff --git a/zcash_client_backend/src/fees/common.rs b/zcash_client_backend/src/fees/common.rs index 8807a12f76..a8a791a527 100644 --- a/zcash_client_backend/src/fees/common.rs +++ b/zcash_client_backend/src/fees/common.rs @@ -17,30 +17,26 @@ use super::{ #[cfg(feature = "orchard")] use super::orchard as orchard_fees; +pub(crate) struct NetFlows { + t_in: NonNegativeAmount, + t_out: NonNegativeAmount, + sapling_in: NonNegativeAmount, + sapling_out: NonNegativeAmount, + orchard_in: NonNegativeAmount, + orchard_out: NonNegativeAmount, +} + #[allow(clippy::too_many_arguments)] -pub(crate) fn single_change_output_balance< - P: consensus::Parameters, - NoteRefT: Clone, - F: FeeRule, - E, ->( - params: &P, - fee_rule: &F, - target_height: BlockHeight, +pub(crate) fn calculate_net_flows( transparent_inputs: &[impl transparent::InputView], transparent_outputs: &[impl transparent::OutputView], sapling: &impl sapling_fees::BundleView, #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, - dust_output_policy: &DustOutputPolicy, - default_dust_threshold: NonNegativeAmount, - change_memo: Option, - _fallback_change_pool: ShieldedProtocol, -) -> Result> +) -> Result> where E: From + From, { let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); - let underflow = || ChangeError::StrategyError(E::from(BalanceError::Underflow)); let t_in = transparent_inputs .iter() @@ -85,13 +81,30 @@ where #[cfg(not(feature = "orchard"))] let orchard_out = NonNegativeAmount::ZERO; + Ok(NetFlows { + t_in, + t_out, + sapling_in, + sapling_out, + orchard_in, + orchard_out, + }) +} + +pub(crate) fn single_change_output_policy( + _net_flows: &NetFlows, + _fallback_change_pool: ShieldedProtocol, +) -> Result<(ShieldedProtocol, usize, usize), ChangeError> +where + E: From + From, +{ // TODO: implement a less naive strategy for selecting the pool to which change will be sent. #[cfg(feature = "orchard")] let (change_pool, sapling_change, orchard_change) = - if orchard_in.is_positive() || orchard_out.is_positive() { + if _net_flows.orchard_in.is_positive() || _net_flows.orchard_out.is_positive() { // Send change to Orchard if we're spending any Orchard inputs or creating any Orchard outputs (ShieldedProtocol::Orchard, 0, 1) - } else if sapling_in.is_positive() || sapling_out.is_positive() { + } else if _net_flows.sapling_in.is_positive() || _net_flows.sapling_out.is_positive() { // Otherwise, send change to Sapling if we're spending any Sapling inputs or creating any // Sapling outputs, so that we avoid pool-crossing. (ShieldedProtocol::Sapling, 1, 0) @@ -104,18 +117,68 @@ where } }; #[cfg(not(feature = "orchard"))] - let (change_pool, sapling_change) = (ShieldedProtocol::Sapling, 1); + let (change_pool, sapling_change, orchard_change) = (ShieldedProtocol::Sapling, 1, 0); + + Ok((change_pool, sapling_change, orchard_change)) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn single_change_output_balance< + P: consensus::Parameters, + NoteRefT: Clone, + F: FeeRule, + E, +>( + params: &P, + fee_rule: &F, + target_height: BlockHeight, + transparent_inputs: &[impl transparent::InputView], + transparent_outputs: &[impl transparent::OutputView], + sapling: &impl sapling_fees::BundleView, + #[cfg(feature = "orchard")] orchard: &impl orchard_fees::BundleView, + dust_output_policy: &DustOutputPolicy, + default_dust_threshold: NonNegativeAmount, + change_memo: Option, + _fallback_change_pool: ShieldedProtocol, +) -> Result> +where + E: From + From, +{ + let overflow = || ChangeError::StrategyError(E::from(BalanceError::Overflow)); + let underflow = || ChangeError::StrategyError(E::from(BalanceError::Underflow)); + + let net_flows = calculate_net_flows::( + transparent_inputs, + transparent_outputs, + sapling, + #[cfg(feature = "orchard")] + orchard, + )?; + let (change_pool, sapling_change, _orchard_change) = + single_change_output_policy::(&net_flows, _fallback_change_pool)?; + + let sapling_input_count = sapling + .bundle_type() + .num_spends(sapling.inputs().len()) + .map_err(ChangeError::BundleError)?; + let sapling_output_count = sapling + .bundle_type() + .num_outputs( + sapling.inputs().len(), + sapling.outputs().len() + sapling_change, + ) + .map_err(ChangeError::BundleError)?; #[cfg(feature = "orchard")] - let orchard_num_actions = orchard + let orchard_action_count = orchard .bundle_type() .num_actions( orchard.inputs().len(), - orchard.outputs().len() + orchard_change, + orchard.outputs().len() + _orchard_change, ) .map_err(ChangeError::BundleError)?; #[cfg(not(feature = "orchard"))] - let orchard_num_actions = 0; + let orchard_action_count = 0; let fee_amount = fee_rule .fee_required( @@ -123,23 +186,16 @@ where target_height, transparent_inputs, transparent_outputs, - sapling - .bundle_type() - .num_spends(sapling.inputs().len()) - .map_err(ChangeError::BundleError)?, - sapling - .bundle_type() - .num_outputs( - sapling.inputs().len(), - sapling.outputs().len() + sapling_change, - ) - .map_err(ChangeError::BundleError)?, - orchard_num_actions, + sapling_input_count, + sapling_output_count, + orchard_action_count, ) .map_err(|fee_error| ChangeError::StrategyError(E::from(fee_error)))?; - let total_in = (t_in + sapling_in + orchard_in).ok_or_else(overflow)?; - let total_out = (t_out + sapling_out + orchard_out + fee_amount).ok_or_else(overflow)?; + let total_in = + (net_flows.t_in + net_flows.sapling_in + net_flows.orchard_in).ok_or_else(overflow)?; + let total_out = (net_flows.t_out + net_flows.sapling_out + net_flows.orchard_out + fee_amount) + .ok_or_else(overflow)?; let proposed_change = (total_in - total_out).ok_or(ChangeError::InsufficientFunds { available: total_in, diff --git a/zcash_client_backend/src/fees/zip317.rs b/zcash_client_backend/src/fees/zip317.rs index 3b703f67e3..ebc2aec867 100644 --- a/zcash_client_backend/src/fees/zip317.rs +++ b/zcash_client_backend/src/fees/zip317.rs @@ -16,8 +16,8 @@ use zcash_primitives::{ use crate::ShieldedProtocol; use super::{ - common::single_change_output_balance, sapling as sapling_fees, ChangeError, ChangeStrategy, - DustOutputPolicy, TransactionBalance, + common::{calculate_net_flows, single_change_output_balance, single_change_output_policy}, + sapling as sapling_fees, ChangeError, ChangeStrategy, DustOutputPolicy, TransactionBalance, }; #[cfg(feature = "orchard")] @@ -93,32 +93,73 @@ impl ChangeStrategy for SingleOutputChangeStrategy { }) .collect(); + #[cfg(feature = "orchard")] + let mut orchard_dust: Vec = orchard + .inputs() + .iter() + .filter_map(|i| { + if orchard_fees::InputView::::value(i) < self.fee_rule.marginal_fee() { + Some(orchard_fees::InputView::::note_id(i).clone()) + } else { + None + } + }) + .collect(); + #[cfg(not(feature = "orchard"))] + let mut orchard_dust: Vec = vec![]; + // Depending on the shape of the transaction, we may be able to spend up to // `grace_actions - 1` dust inputs. If we don't have any dust inputs though, // we don't need to worry about any of that. - if !(transparent_dust.is_empty() && sapling_dust.is_empty()) { + if !(transparent_dust.is_empty() && sapling_dust.is_empty() && orchard_dust.is_empty()) { let t_non_dust = transparent_inputs.len() - transparent_dust.len(); let t_allowed_dust = transparent_outputs.len().saturating_sub(t_non_dust); - // We add one to the sapling outputs for the (single) change output Note that this - // means that wallet-internal shielding transactions are an opportunity to spend a dust - // note. + // We add one to either the Sapling or Orchard outputs for the (single) + // change output. Note that this means that wallet-internal shielding + // transactions are an opportunity to spend a dust note. + let net_flows = calculate_net_flows::( + transparent_inputs, + transparent_outputs, + sapling, + #[cfg(feature = "orchard")] + orchard, + )?; + let (_, sapling_change, orchard_change) = + single_change_output_policy::( + &net_flows, + self.fallback_change_pool, + )?; + let s_non_dust = sapling.inputs().len() - sapling_dust.len(); - let s_allowed_dust = (sapling.outputs().len() + 1).saturating_sub(s_non_dust); + let s_allowed_dust = + (sapling.outputs().len() + sapling_change).saturating_sub(s_non_dust); + + #[cfg(feature = "orchard")] + let (orchard_inputs_len, orchard_outputs_len) = + (orchard.inputs().len(), orchard.outputs().len()); + #[cfg(not(feature = "orchard"))] + let (orchard_inputs_len, orchard_outputs_len) = (0, 0); + + let o_non_dust = orchard_inputs_len - orchard_dust.len(); + let o_allowed_dust = (orchard_outputs_len + orchard_change).saturating_sub(o_non_dust); let available_grace_inputs = self .fee_rule .grace_actions() .saturating_sub(t_non_dust) - .saturating_sub(s_non_dust); + .saturating_sub(s_non_dust) + .saturating_sub(o_non_dust); let mut t_disallowed_dust = transparent_dust.len().saturating_sub(t_allowed_dust); let mut s_disallowed_dust = sapling_dust.len().saturating_sub(s_allowed_dust); + let mut o_disallowed_dust = orchard_dust.len().saturating_sub(o_allowed_dust); if available_grace_inputs > 0 { // If we have available grace inputs, allocate them first to transparent dust - // and then to Sapling dust. The caller has provided inputs that it is willing - // to spend, so we don't need to consider privacy effects at this layer. + // and then to Sapling dust followed by Orchard dust. The caller has provided + // inputs that it is willing to spend, so we don't need to consider privacy + // effects at this layer. let t_grace_dust = available_grace_inputs.saturating_sub(t_disallowed_dust); t_disallowed_dust = t_disallowed_dust.saturating_sub(t_grace_dust); @@ -126,6 +167,12 @@ impl ChangeStrategy for SingleOutputChangeStrategy { .saturating_sub(t_grace_dust) .saturating_sub(s_disallowed_dust); s_disallowed_dust = s_disallowed_dust.saturating_sub(s_grace_dust); + + let o_grace_dust = available_grace_inputs + .saturating_sub(t_grace_dust) + .saturating_sub(s_grace_dust) + .saturating_sub(o_disallowed_dust); + o_disallowed_dust = o_disallowed_dust.saturating_sub(o_grace_dust); } // Truncate the lists of inputs to be disregarded in input selection to just the @@ -135,11 +182,16 @@ impl ChangeStrategy for SingleOutputChangeStrategy { transparent_dust.truncate(t_disallowed_dust); sapling_dust.reverse(); sapling_dust.truncate(s_disallowed_dust); + orchard_dust.reverse(); + orchard_dust.truncate(o_disallowed_dust); - if !(transparent_dust.is_empty() && sapling_dust.is_empty()) { + if !(transparent_dust.is_empty() && sapling_dust.is_empty() && orchard_dust.is_empty()) + { return Err(ChangeError::DustInputs { transparent: transparent_dust, sapling: sapling_dust, + #[cfg(feature = "orchard")] + orchard: orchard_dust, }); } } diff --git a/zcash_client_backend/src/proto.rs b/zcash_client_backend/src/proto.rs index 50cf4b09f4..f020e731d6 100644 --- a/zcash_client_backend/src/proto.rs +++ b/zcash_client_backend/src/proto.rs @@ -326,6 +326,8 @@ pub enum ProposalDecodingError { ProposalInvalid(ProposalError), /// An inputs field for the given protocol was present, but contained no input note references. EmptyShieldedInputs(ShieldedProtocol), + /// A memo field was provided for a transparent output. + TransparentMemo, /// Change outputs to the specified pool are not supported. InvalidChangeRecipient(PoolType), } @@ -378,6 +380,9 @@ impl Display for ProposalDecodingError { "An inputs field was present for {:?}, but contained no note references.", protocol ), + ProposalDecodingError::TransparentMemo => { + write!(f, "Transparent outputs cannot have memos.") + } ProposalDecodingError::InvalidChangeRecipient(pool_type) => write!( f, "Change outputs to the {} pool are not supported.", @@ -704,20 +709,26 @@ impl proposal::Proposal { .proposed_change .iter() .map(|cv| -> Result> { + let value = NonNegativeAmount::from_u64(cv.value) + .map_err(|_| ProposalDecodingError::BalanceInvalid)?; + let memo = cv + .memo + .as_ref() + .map(|bytes| { + MemoBytes::from_bytes(&bytes.value) + .map_err(ProposalDecodingError::MemoInvalid) + }) + .transpose()?; match cv.pool_type()? { PoolType::Shielded(ShieldedProtocol::Sapling) => { - Ok(ChangeValue::sapling( - NonNegativeAmount::from_u64(cv.value).map_err( - |_| ProposalDecodingError::BalanceInvalid, - )?, - cv.memo - .as_ref() - .map(|bytes| { - MemoBytes::from_bytes(&bytes.value) - .map_err(ProposalDecodingError::MemoInvalid) - }) - .transpose()?, - )) + Ok(ChangeValue::sapling(value, memo)) + } + #[cfg(feature = "orchard")] + PoolType::Shielded(ShieldedProtocol::Orchard) => { + Ok(ChangeValue::orchard(value, memo)) + } + PoolType::Transparent if memo.is_some() => { + Err(ProposalDecodingError::TransparentMemo) } t => Err(ProposalDecodingError::InvalidChangeRecipient(t)), } diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 4a99f87547..33df8a938f 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -624,7 +624,7 @@ where self.orchard.add_outputs( block_hash, txid, - |action| OrchardDomain::for_nullifier(action.nullifier()), + OrchardDomain::for_compact_action, &tx.actions .iter() .enumerate() @@ -888,7 +888,7 @@ where index: i, } })?; - Ok((OrchardDomain::for_nullifier(action.nullifier()), action)) + Ok((OrchardDomain::for_compact_action(&action), action)) }) .collect::, _>>()?, batch_runners diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index e90c8c0777..1490aeb224 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -35,6 +35,12 @@ and this library adheres to Rust's notion of - `init::WalletMigrationError` has added variants: - `WalletMigrationError::AddressGeneration` - `WalletMigrationError::CannotRevert` +- The `v_transactions` and `v_tx_outputs` views now include Orchard notes. + +## [0.9.1] - 2024-03-09 + +### Fixed +- Documentation now correctly builds with all feature flags. ## [0.9.0] - 2024-03-01 diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index a7dff88eef..2c247b393b 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zcash_client_sqlite" description = "An SQLite-based Zcash light client" -version = "0.9.0" +version = "0.9.1" authors = [ "Jack Grigg ", "Kris Nuttycombe " @@ -15,7 +15,14 @@ rust-version.workspace = true categories.workspace = true [package.metadata.docs.rs] -all-features = true +# Manually specify features while `orchard` is not in the public API. +#all-features = true +features = [ + "multicore", + "test-dependencies", + "transparent-inputs", + "unstable", +] rustdoc-args = ["--cfg", "docsrs"] [dependencies] @@ -71,11 +78,14 @@ maybe-rayon.workspace = true [dev-dependencies] assert_matches.workspace = true +bls12_381.workspace = true incrementalmerkletree = { workspace = true, features = ["test-dependencies"] } pasta_curves.workspace = true shardtree = { workspace = true, features = ["legacy-api", "test-dependencies"] } nonempty.workspace = true +orchard = { workspace = true, features = ["test-dependencies"] } proptest.workspace = true +rand_chacha.workspace = true rand_core.workspace = true regex = "1.4" tempfile = "3.5.0" @@ -83,6 +93,7 @@ zcash_keys = { workspace = true, features = ["test-dependencies"] } zcash_note_encryption.workspace = true zcash_proofs = { workspace = true, features = ["bundled-prover"] } zcash_primitives = { workspace = true, features = ["test-dependencies"] } +zcash_protocol = { workspace = true, features = ["local-consensus"] } zcash_client_backend = { workspace = true, features = ["test-dependencies", "unstable-serialization", "unstable-spanning-tree"] } zcash_address = { workspace = true, features = ["test-dependencies"] } diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 7585720383..028b2980f3 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -322,331 +322,87 @@ where #[cfg(test)] #[allow(deprecated)] mod tests { - use std::num::NonZeroU32; - - use sapling::zip32::ExtendedSpendingKey; - use zcash_primitives::{ - block::BlockHash, - transaction::{components::amount::NonNegativeAmount, fees::zip317::FeeRule}, - }; - - use zcash_client_backend::{ - address::Address, - data_api::{ - chain::error::Error, wallet::input_selection::GreedyInputSelector, AccountBirthday, - WalletRead, - }, - fees::{zip317::SingleOutputChangeStrategy, DustOutputPolicy}, - scanning::ScanError, - wallet::OvkPolicy, - zip321::{Payment, TransactionRequest}, - ShieldedProtocol, - }; - - use crate::{ - testing::{AddressType, TestBuilder}, - wallet::truncate_to_height, - }; + use crate::{testing, wallet::sapling::tests::SaplingPoolTester}; + + #[cfg(feature = "orchard")] + use crate::wallet::orchard::tests::OrchardPoolTester; #[test] - fn valid_chain_states() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let dfvk = st.test_account_sapling().unwrap(); - - // Empty chain should return None - assert_matches!(st.wallet().chain_height(), Ok(None)); - - // Create a fake CompactBlock sending value to the address - let (h1, _, _) = st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(5), - ); - - // Scan the cache - st.scan_cached_blocks(h1, 1); - - // Create a second fake CompactBlock sending more value to the address - let (h2, _, _) = st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(7), - ); - - // Scanning should detect no inconsistencies - st.scan_cached_blocks(h2, 1); + fn valid_chain_states_sapling() { + testing::pool::valid_chain_states::() } #[test] - fn invalid_chain_cache_disconnected() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let dfvk = st.test_account_sapling().unwrap(); - - // Create some fake CompactBlocks - let (h, _, _) = st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(5), - ); - let (last_contiguous_height, _, _) = st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(7), - ); - - // Scanning the cache should find no inconsistencies - st.scan_cached_blocks(h, 2); - - // Create more fake CompactBlocks that don't connect to the scanned ones - let disconnect_height = last_contiguous_height + 1; - st.generate_block_at( - disconnect_height, - BlockHash([1; 32]), - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(8), - 2, - ); - st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(3), - ); - - // Data+cache chain should be invalid at the data/cache boundary - assert_matches!( - st.try_scan_cached_blocks( - disconnect_height, - 2 - ), - Err(Error::Scan(ScanError::PrevHashMismatch { at_height })) - if at_height == disconnect_height - ); + #[cfg(feature = "orchard")] + fn valid_chain_states_orchard() { + testing::pool::valid_chain_states::() } + // FIXME: This requires test framework fixes to pass. #[test] - fn data_db_truncation() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - let account = st.test_account().unwrap(); - - let dfvk = st.test_account_sapling().unwrap(); - - // Wallet summary is not yet available - assert_eq!(st.get_wallet_summary(0), None); - - // Create fake CompactBlocks sending value to the address - let value = NonNegativeAmount::const_from_u64(5); - let value2 = NonNegativeAmount::const_from_u64(7); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); - - // Scan the cache - st.scan_cached_blocks(h, 2); - - // Account balance should reflect both received notes - assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); - - // "Rewind" to height of last scanned block - st.wallet_mut() - .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h + 1)) - .unwrap(); - - // Account balance should be unaltered - assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); - - // Rewind so that one block is dropped - st.wallet_mut() - .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h)) - .unwrap(); - - // Account balance should only contain the first received note - assert_eq!(st.get_total_balance(account.0), value); - - // Scan the cache again - st.scan_cached_blocks(h, 2); - - // Account balance should again reflect both received notes - assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); + #[cfg(feature = "orchard")] + fn invalid_chain_cache_disconnected_sapling() { + testing::pool::invalid_chain_cache_disconnected::() } #[test] - fn scan_cached_blocks_allows_blocks_out_of_order() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - let account = st.test_account().unwrap(); - - let (_, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Create a block with height SAPLING_ACTIVATION_HEIGHT - let value = NonNegativeAmount::const_from_u64(50000); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h1, 1); - assert_eq!(st.get_total_balance(account.0), value); - - // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - - // Scan the later block first - st.scan_cached_blocks(h3, 1); - - // Now scan the block of height SAPLING_ACTIVATION_HEIGHT + 1 - st.scan_cached_blocks(h2, 1); - assert_eq!( - st.get_total_balance(account.0), - NonNegativeAmount::const_from_u64(150_000) - ); - - // We can spend the received notes - let req = TransactionRequest::new(vec![Payment { - recipient_address: Address::Sapling(dfvk.default_address().1), - amount: NonNegativeAmount::const_from_u64(110_000), - memo: None, - label: None, - message: None, - other_params: vec![], - }]) - .unwrap(); - let input_selector = GreedyInputSelector::new( - SingleOutputChangeStrategy::new(FeeRule::standard(), None, ShieldedProtocol::Sapling), - DustOutputPolicy::default(), - ); - assert_matches!( - st.spend( - &input_selector, - &usk, - req, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - ), - Ok(_) - ); + #[cfg(feature = "orchard")] + fn invalid_chain_cache_disconnected_orchard() { + testing::pool::invalid_chain_cache_disconnected::() } #[test] - fn scan_cached_blocks_finds_received_notes() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - let account = st.test_account().unwrap(); - - let dfvk = st.test_account_sapling().unwrap(); - - // Wallet summary is not yet available - assert_eq!(st.get_wallet_summary(0), None); - - // Create a fake CompactBlock sending value to the address - let value = NonNegativeAmount::const_from_u64(5); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - - // Scan the cache - let summary = st.scan_cached_blocks(h1, 1); - assert_eq!(summary.scanned_range().start, h1); - assert_eq!(summary.scanned_range().end, h1 + 1); - assert_eq!(summary.received_sapling_note_count(), 1); - - // Account balance should reflect the received note - assert_eq!(st.get_total_balance(account.0), value); - - // Create a second fake CompactBlock sending more value to the address - let value2 = NonNegativeAmount::const_from_u64(7); - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); - - // Scan the cache again - let summary = st.scan_cached_blocks(h2, 1); - assert_eq!(summary.scanned_range().start, h2); - assert_eq!(summary.scanned_range().end, h2 + 1); - assert_eq!(summary.received_sapling_note_count(), 1); - - // Account balance should reflect both received notes - assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); + fn data_db_truncation_sapling() { + testing::pool::data_db_truncation::() } #[test] - fn scan_cached_blocks_finds_change_notes() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - let account = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Wallet summary is not yet available - assert_eq!(st.get_wallet_summary(0), None); - - // Create a fake CompactBlock sending value to the address - let value = NonNegativeAmount::const_from_u64(5); - let (received_height, _, nf) = - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - - // Scan the cache - st.scan_cached_blocks(received_height, 1); - - // Account balance should reflect the received note - assert_eq!(st.get_total_balance(account.0), value); - - // Create a second fake CompactBlock spending value from the address - let extsk2 = ExtendedSpendingKey::master(&[0]); - let to2 = extsk2.default_address().1; - let value2 = NonNegativeAmount::const_from_u64(2); - let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); - - // Scan the cache again - st.scan_cached_blocks(spent_height, 1); - - // Account balance should equal the change - assert_eq!(st.get_total_balance(account.0), (value - value2).unwrap()); + #[cfg(feature = "orchard")] + fn data_db_truncation_orchard() { + testing::pool::data_db_truncation::() } #[test] - fn scan_cached_blocks_detects_spends_out_of_order() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - let account = st.test_account().unwrap(); - - let dfvk = st.test_account_sapling().unwrap(); + fn scan_cached_blocks_allows_blocks_out_of_order_sapling() { + testing::pool::scan_cached_blocks_allows_blocks_out_of_order::() + } - // Wallet summary is not yet available - assert_eq!(st.get_wallet_summary(0), None); + #[test] + #[cfg(feature = "orchard")] + fn scan_cached_blocks_allows_blocks_out_of_order_orchard() { + testing::pool::scan_cached_blocks_allows_blocks_out_of_order::() + } - // Create a fake CompactBlock sending value to the address - let value = NonNegativeAmount::const_from_u64(5); - let (received_height, _, nf) = - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + #[test] + fn scan_cached_blocks_finds_received_notes_sapling() { + testing::pool::scan_cached_blocks_finds_received_notes::() + } - // Create a second fake CompactBlock spending value from the address - let extsk2 = ExtendedSpendingKey::master(&[0]); - let to2 = extsk2.default_address().1; - let value2 = NonNegativeAmount::const_from_u64(2); - let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + #[test] + #[cfg(feature = "orchard")] + fn scan_cached_blocks_finds_received_notes_orchard() { + testing::pool::scan_cached_blocks_finds_received_notes::() + } - // Scan the spending block first. - st.scan_cached_blocks(spent_height, 1); + #[test] + fn scan_cached_blocks_finds_change_notes_sapling() { + testing::pool::scan_cached_blocks_finds_change_notes::() + } - // Account balance should equal the change - assert_eq!(st.get_total_balance(account.0), (value - value2).unwrap()); + #[test] + #[cfg(feature = "orchard")] + fn scan_cached_blocks_finds_change_notes_orchard() { + testing::pool::scan_cached_blocks_finds_change_notes::() + } - // Now scan the block in which we received the note that was spent. - st.scan_cached_blocks(received_height, 1); + #[test] + fn scan_cached_blocks_detects_spends_out_of_order_sapling() { + testing::pool::scan_cached_blocks_detects_spends_out_of_order::() + } - // Account balance should be the same. - assert_eq!(st.get_total_balance(account.0), (value - value2).unwrap()); + #[test] + #[cfg(feature = "orchard")] + fn scan_cached_blocks_detects_spends_out_of_order_orchard() { + testing::pool::scan_cached_blocks_detects_spends_out_of_order::() } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index b04ea3e5a7..12c08f054a 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -32,7 +32,7 @@ // Catch documentation errors caused by code changes. #![deny(rustdoc::broken_intra_doc_links)] -use incrementalmerkletree::Position; +use incrementalmerkletree::{Position, Retention}; use maybe_rayon::{ prelude::{IndexedParallelIterator, ParallelIterator}, slice::ParallelSliceMut, @@ -58,7 +58,7 @@ use zcash_client_backend::{ address::UnifiedAddress, data_api::{ self, - chain::{BlockSource, CommitmentTreeRoot}, + chain::{BlockSource, ChainState, CommitmentTreeRoot}, scanning::{ScanPriority, ScanRange}, AccountBirthday, BlockMetadata, DecryptedTransaction, InputSource, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletSummary, @@ -69,13 +69,18 @@ use zcash_client_backend::{ }, proto::compact_formats::CompactBlock, wallet::{Note, NoteId, ReceivedNote, Recipient, WalletTransparentOutput}, - DecryptedOutput, ShieldedProtocol, TransferType, + DecryptedOutput, PoolType, ShieldedProtocol, TransferType, }; use crate::{error::SqliteClientError, wallet::commitment_tree::SqliteShardStore}; #[cfg(feature = "orchard")] -use zcash_client_backend::{data_api::ORCHARD_SHARD_HEIGHT, PoolType}; +use { + incrementalmerkletree::frontier::Frontier, + shardtree::store::{Checkpoint, ShardStore}, + std::collections::BTreeMap, + zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT, +}; #[cfg(feature = "transparent-inputs")] use { @@ -92,7 +97,6 @@ use { pub mod chain; pub mod error; - pub mod wallet; use wallet::{ commitment_tree::{self, put_shard_roots}, @@ -112,6 +116,9 @@ pub(crate) const VERIFY_LOOKAHEAD: u32 = 10; pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; +#[cfg(feature = "orchard")] +pub(crate) const ORCHARD_TABLES_PREFIX: &str = "orchard"; + #[cfg(not(feature = "transparent-inputs"))] pub(crate) const UA_TRANSPARENT: bool = false; #[cfg(feature = "transparent-inputs")] @@ -206,7 +213,20 @@ impl, P: consensus::Parameters> InputSource for txid, index, ), - ShieldedProtocol::Orchard => Ok(None), + ShieldedProtocol::Orchard => { + #[cfg(feature = "orchard")] + return wallet::orchard::get_spendable_orchard_note( + self.conn.borrow(), + &self.params, + txid, + index, + ); + + #[cfg(not(feature = "orchard"))] + return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded( + ShieldedProtocol::Orchard, + ))); + } } } @@ -218,14 +238,25 @@ impl, P: consensus::Parameters> InputSource for anchor_height: BlockHeight, exclude: &[Self::NoteRef], ) -> Result>, Self::Error> { - wallet::sapling::select_spendable_sapling_notes( + let received_iter = std::iter::empty(); + let received_iter = received_iter.chain(wallet::sapling::select_spendable_sapling_notes( self.conn.borrow(), &self.params, account, target_value, anchor_height, exclude, - ) + )?); + #[cfg(feature = "orchard")] + let received_iter = received_iter.chain(wallet::orchard::select_spendable_orchard_notes( + self.conn.borrow(), + &self.params, + account, + target_value, + anchor_height, + exclude, + )?); + Ok(received_iter.collect()) } #[cfg(feature = "transparent-inputs")] @@ -256,6 +287,7 @@ impl, P: consensus::Parameters> InputSource for impl, P: consensus::Parameters> WalletRead for WalletDb { type Error = SqliteClientError; type AccountId = AccountId; + type Account = (AccountId, Option); fn validate_seed( &self, @@ -373,10 +405,18 @@ impl, P: consensus::Parameters> WalletRead for W fn get_account_for_ufvk( &self, ufvk: &UnifiedFullViewingKey, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { wallet::get_account_for_ufvk(self.conn.borrow(), &self.params, ufvk) } + fn get_seed_account( + &self, + seed: &HdSeedFingerprint, + account_id: zip32::AccountId, + ) -> Result, Self::Error> { + wallet::get_seed_account(self.conn.borrow(), &self.params, seed, account_id) + } + fn get_wallet_summary( &self, min_confirmations: u32, @@ -414,10 +454,9 @@ impl, P: consensus::Parameters> WalletRead for W #[cfg(feature = "orchard")] fn get_orchard_nullifiers( &self, - _query: NullifierQuery, + query: NullifierQuery, ) -> Result, Self::Error> { - // FIXME! Orchard. - Ok(vec![]) + wallet::orchard::get_orchard_nullifiers(self.conn.borrow(), query) } #[cfg(feature = "transparent-inputs")] @@ -509,19 +548,32 @@ impl WalletWrite for WalletDb #[allow(clippy::type_complexity)] fn put_blocks( &mut self, + from_state: &ChainState, blocks: Vec>, ) -> Result<(), Self::Error> { + struct BlockPositions { + height: BlockHeight, + sapling_start_position: Position, + #[cfg(feature = "orchard")] + orchard_start_position: Position, + } + self.transactionally(|wdb| { - let start_positions = blocks.first().map(|block| { - ( - block.height(), - Position::from( - u64::from(block.sapling().final_tree_size()) - - u64::try_from(block.sapling().commitments().len()).unwrap(), - ), - ) + let start_positions = blocks.first().map(|block| BlockPositions { + height: block.height(), + sapling_start_position: Position::from( + u64::from(block.sapling().final_tree_size()) + - u64::try_from(block.sapling().commitments().len()).unwrap(), + ), + #[cfg(feature = "orchard")] + orchard_start_position: Position::from( + u64::from(block.orchard().final_tree_size()) + - u64::try_from(block.orchard().commitments().len()).unwrap(), + ), }); let mut sapling_commitments = vec![]; + #[cfg(feature = "orchard")] + let mut orchard_commitments = vec![]; let mut last_scanned_height = None; let mut note_positions = vec![]; for block in blocks.into_iter() { @@ -540,6 +592,10 @@ impl WalletWrite for WalletDb block.block_time(), block.sapling().final_tree_size(), block.sapling().commitments().len().try_into().unwrap(), + #[cfg(feature = "orchard")] + block.orchard().final_tree_size(), + #[cfg(feature = "orchard")] + block.orchard().commitments().len().try_into().unwrap(), )?; for tx in block.transactions() { @@ -549,6 +605,10 @@ impl WalletWrite for WalletDb for spend in tx.sapling_spends() { wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; } + #[cfg(feature = "orchard")] + for spend in tx.orchard_spends() { + wallet::orchard::mark_orchard_note_spent(wdb.conn.0, tx_row, spend.nf())?; + } for output in tx.sapling_outputs() { // Check whether this note was spent in a later block range that @@ -567,6 +627,24 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; } + #[cfg(feature = "orchard")] + for output in tx.orchard_outputs() { + // Check whether this note was spent in a later block range that + // we previously scanned. + let spent_in = output + .nf() + .map(|nf| { + wallet::query_nullifier_map::<_, Scope>( + wdb.conn.0, + ShieldedProtocol::Orchard, + &nf.to_bytes(), + ) + }) + .transpose()? + .flatten(); + + wallet::orchard::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + } } // Insert the new nullifiers from this block into the nullifier map. @@ -576,19 +654,44 @@ impl WalletWrite for WalletDb ShieldedProtocol::Sapling, block.sapling().nullifier_map(), )?; + #[cfg(feature = "orchard")] + wallet::insert_nullifier_map( + wdb.conn.0, + block.height(), + ShieldedProtocol::Orchard, + &block + .orchard() + .nullifier_map() + .iter() + .map(|(txid, idx, nfs)| { + (*txid, *idx, nfs.iter().map(|nf| nf.to_bytes()).collect()) + }) + .collect::>(), + )?; note_positions.extend(block.transactions().iter().flat_map(|wtx| { - wtx.sapling_outputs().iter().map(|out| { + let iter = wtx.sapling_outputs().iter().map(|out| { ( ShieldedProtocol::Sapling, out.note_commitment_tree_position(), ) - }) + }); + #[cfg(feature = "orchard")] + let iter = iter.chain(wtx.orchard_outputs().iter().map(|out| { + ( + ShieldedProtocol::Orchard, + out.note_commitment_tree_position(), + ) + })); + + iter })); last_scanned_height = Some(block.height()); let block_commitments = block.into_commitments(); sapling_commitments.extend(block_commitments.sapling.into_iter().map(Some)); + #[cfg(feature = "orchard")] + orchard_commitments.extend(block_commitments.orchard.into_iter().map(Some)); } // Prune the nullifier map of entries we no longer need. @@ -601,16 +704,17 @@ impl WalletWrite for WalletDb // We will have a start position and a last scanned height in all cases where // `blocks` is non-empty. - if let Some(((start_height, start_position), last_scanned_height)) = + if let Some((start_positions, last_scanned_height)) = start_positions.zip(last_scanned_height) { // Create subtrees from the note commitments in parallel. const CHUNK_SIZE: usize = 1024; - let subtrees = sapling_commitments + let sapling_subtrees = sapling_commitments .par_chunks_mut(CHUNK_SIZE) .enumerate() .filter_map(|(i, chunk)| { - let start = start_position + (i * CHUNK_SIZE) as u64; + let start = + start_positions.sapling_start_position + (i * CHUNK_SIZE) as u64; let end = start + chunk.len() as u64; shardtree::LocatedTree::from_iter( @@ -622,15 +726,161 @@ impl WalletWrite for WalletDb .map(|res| (res.subtree, res.checkpoints)) .collect::>(); + #[cfg(feature = "orchard")] + let orchard_subtrees = orchard_commitments + .par_chunks_mut(CHUNK_SIZE) + .enumerate() + .filter_map(|(i, chunk)| { + let start = + start_positions.orchard_start_position + (i * CHUNK_SIZE) as u64; + let end = start + chunk.len() as u64; + + shardtree::LocatedTree::from_iter( + start..end, + ORCHARD_SHARD_HEIGHT.into(), + chunk.iter_mut().map(|n| n.take().expect("always Some")), + ) + }) + .map(|res| (res.subtree, res.checkpoints)) + .collect::>(); + + // Collect the complete set of Sapling checkpoints + #[cfg(feature = "orchard")] + let sapling_checkpoint_positions: BTreeMap = + sapling_subtrees + .iter() + .flat_map(|(_, checkpoints)| checkpoints.iter()) + .map(|(k, v)| (*k, *v)) + .collect(); + + #[cfg(feature = "orchard")] + let orchard_checkpoint_positions: BTreeMap = + orchard_subtrees + .iter() + .flat_map(|(_, checkpoints)| checkpoints.iter()) + .map(|(k, v)| (*k, *v)) + .collect(); + + #[cfg(feature = "orchard")] + fn ensure_checkpoints< + 'a, + H, + I: Iterator, + const DEPTH: u8, + >( + // An iterator of checkpoints heights for which we wish to ensure that + // checkpoints exists. + checkpoint_heights: I, + // The map of checkpoint positions from which we will draw note commitment tree + // position information for the newly created checkpoints. + existing_checkpoint_positions: &BTreeMap, + // The frontier whose position will be used for an inserted checkpoint when + // there is no preceding checkpoint in existing_checkpoint_positions. + state_final_tree: &Frontier, + ) -> Vec<(BlockHeight, Checkpoint)> { + checkpoint_heights + .flat_map(|from_checkpoint_height| { + existing_checkpoint_positions + .range::(..=*from_checkpoint_height) + .last() + .map_or_else( + || { + Some(( + *from_checkpoint_height, + state_final_tree + .value() + .map_or_else(Checkpoint::tree_empty, |t| { + Checkpoint::at_position(t.position()) + }), + )) + }, + |(to_prev_height, position)| { + if *to_prev_height < *from_checkpoint_height { + Some(( + *from_checkpoint_height, + Checkpoint::at_position(*position), + )) + } else { + // The checkpoint already exists, so we don't need to + // do anything. + None + } + }, + ) + .into_iter() + }) + .collect::>() + } + + #[cfg(feature = "orchard")] + let missing_sapling_checkpoints = ensure_checkpoints( + orchard_checkpoint_positions.keys(), + &sapling_checkpoint_positions, + from_state.final_sapling_tree(), + ); + #[cfg(feature = "orchard")] + let missing_orchard_checkpoints = ensure_checkpoints( + sapling_checkpoint_positions.keys(), + &orchard_checkpoint_positions, + from_state.final_orchard_tree(), + ); + // Update the Sapling note commitment tree with all newly read note commitments - let mut subtrees = subtrees.into_iter(); - wdb.with_sapling_tree_mut::<_, _, Self::Error>(move |sapling_tree| { - for (tree, checkpoints) in &mut subtrees { - sapling_tree.insert_tree(tree, checkpoints)?; - } + { + let mut sapling_subtrees_iter = sapling_subtrees.into_iter(); + wdb.with_sapling_tree_mut::<_, _, Self::Error>(|sapling_tree| { + sapling_tree.insert_frontier( + from_state.final_sapling_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + is_marked: false, + }, + )?; - Ok(()) - })?; + for (tree, checkpoints) in &mut sapling_subtrees_iter { + sapling_tree.insert_tree(tree, checkpoints)?; + } + + // Ensure we have a Sapling checkpoint for each checkpointed Orchard block height + #[cfg(feature = "orchard")] + for (height, checkpoint) in &missing_sapling_checkpoints { + sapling_tree + .store_mut() + .add_checkpoint(*height, checkpoint.clone()) + .map_err(ShardTreeError::Storage)?; + } + + Ok(()) + })?; + } + + // Update the Orchard note commitment tree with all newly read note commitments + #[cfg(feature = "orchard")] + { + let mut orchard_subtrees = orchard_subtrees.into_iter(); + wdb.with_orchard_tree_mut::<_, _, Self::Error>(|orchard_tree| { + orchard_tree.insert_frontier( + from_state.final_orchard_tree().clone(), + Retention::Checkpoint { + id: from_state.block_height(), + is_marked: false, + }, + )?; + + for (tree, checkpoints) in &mut orchard_subtrees { + orchard_tree.insert_tree(tree, checkpoints)?; + } + + for (height, checkpoint) in &missing_orchard_checkpoints { + orchard_tree + .store_mut() + .add_checkpoint(*height, checkpoint.clone()) + .map_err(ShardTreeError::Storage)?; + } + + Ok(()) + })?; + } // Update now-expired transactions that didn't get mined. wallet::update_expired_notes(wdb.conn.0, last_scanned_height)?; @@ -639,7 +889,7 @@ impl WalletWrite for WalletDb wdb.conn.0, &wdb.params, Range { - start: start_height, + start: start_positions.height, end: last_scanned_height + 1, }, ¬e_positions, @@ -711,7 +961,6 @@ impl WalletWrite for WalletDb } #[cfg(feature = "orchard")] - #[allow(unused_assignments)] // Remove this when the todo!()s below are implemented. for output in d_tx.orchard_outputs() { match output.transfer_type() { TransferType::Outgoing | TransferType::WalletInternal => { @@ -744,8 +993,7 @@ impl WalletWrite for WalletDb )?; if matches!(recipient, Recipient::InternalAccount(_, _)) { - todo!(); - //wallet::orchard::put_received_note(wdb.conn.0, output, tx_ref, None)?; + wallet::orchard::put_received_note(wdb.conn.0, output, tx_ref, None)?; } } TransferType::Incoming => { @@ -760,8 +1008,7 @@ impl WalletWrite for WalletDb } } - todo!() - //wallet::orchard::put_received_note(wdb.conn.0, output, tx_ref, None)?; + wallet::orchard::put_received_note(wdb.conn.0, output, tx_ref, None)?; } } } @@ -774,21 +1021,31 @@ impl WalletWrite for WalletDb // If we have some transparent outputs: if d_tx.tx().transparent_bundle().iter().any(|b| !b.vout.is_empty()) { - let nullifiers = wdb.get_sapling_nullifiers(NullifierQuery::All)?; - // If the transaction contains shielded spends from our wallet, we will store z->t + // If the transaction contains spends from our wallet, we will store z->t // transactions we observe in the same way they would be stored by // create_spend_to_address. - if let Some((account_id, _)) = nullifiers.iter().find( + let sapling_from_account = wdb.get_sapling_nullifiers(NullifierQuery::All)?.into_iter().find( |(_, nf)| - d_tx.tx().sapling_bundle().iter().flat_map(|b| b.shielded_spends().iter()) + d_tx.tx().sapling_bundle().into_iter().flat_map(|b| b.shielded_spends().iter()) .any(|input| nf == input.nullifier()) - ) { + ).map(|(account_id, _)| account_id); + + #[cfg(feature = "orchard")] + let orchard_from_account = wdb.get_orchard_nullifiers(NullifierQuery::All)?.into_iter().find( + |(_, nf)| + d_tx.tx().orchard_bundle().iter().flat_map(|b| b.actions().iter()) + .any(|input| nf == input.nullifier()) + ).map(|(account_id, _)| account_id); + #[cfg(not(feature = "orchard"))] + let orchard_from_account = None; + + if let Some(account_id) = orchard_from_account.or(sapling_from_account) { for (output_index, txout) in d_tx.tx().transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { if let Some(address) = txout.recipient_address() { wallet::put_sent_output( wdb.conn.0, &wdb.params, - *account_id, + account_id, tx_ref, output_index, &Recipient::Transparent(address), @@ -830,6 +1087,19 @@ impl WalletWrite for WalletDb )?; } } + if let Some(_bundle) = sent_tx.tx().orchard_bundle() { + #[cfg(feature = "orchard")] + for action in _bundle.actions() { + wallet::orchard::mark_orchard_note_spent( + wdb.conn.0, + tx_ref, + action.nullifier(), + )?; + } + + #[cfg(not(feature = "orchard"))] + panic!("Sent a transaction with Orchard Actions without `orchard` enabled?"); + } #[cfg(feature = "transparent-inputs")] for utxo_outpoint in sent_tx.utxos_spent() { @@ -863,8 +1133,21 @@ impl WalletWrite for WalletDb )?; } #[cfg(feature = "orchard")] - Recipient::InternalAccount(_account, Note::Orchard(_note)) => { - todo!(); + Recipient::InternalAccount(account, Note::Orchard(note)) => { + wallet::orchard::put_received_note( + wdb.conn.0, + &DecryptedOutput::new( + output.output_index(), + *note, + *account, + output + .memo() + .map_or_else(MemoBytes::empty, |memo| memo.clone()), + TransferType::WalletInternal, + ), + tx_ref, + None, + )?; } _ => (), } @@ -954,7 +1237,7 @@ impl WalletCommitmentTrees for WalletDb; #[cfg(feature = "orchard")] - fn with_orchard_tree_mut(&mut self, _callback: F) -> Result + fn with_orchard_tree_mut(&mut self, mut callback: F) -> Result where for<'a> F: FnMut( &'a mut ShardTree< @@ -965,16 +1248,41 @@ impl WalletCommitmentTrees for WalletDb Result, E: From>, { - todo!() + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + let shard_store = SqliteShardStore::from_connection(&tx, ORCHARD_TABLES_PREFIX) + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + let result = { + let mut shardtree = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); + callback(&mut shardtree)? + }; + + tx.commit() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + Ok(result) } #[cfg(feature = "orchard")] fn put_orchard_subtree_roots( &mut self, - _start_index: u64, - _roots: &[CommitmentTreeRoot], + start_index: u64, + roots: &[CommitmentTreeRoot], ) -> Result<(), ShardTreeError> { - todo!() + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + put_shard_roots::<_, { ORCHARD_SHARD_HEIGHT * 2 }, ORCHARD_SHARD_HEIGHT>( + &tx, + ORCHARD_TABLES_PREFIX, + start_index, + roots, + )?; + tx.commit() + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?; + Ok(()) } } @@ -1025,7 +1333,7 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb; #[cfg(feature = "orchard")] - fn with_orchard_tree_mut(&mut self, _callback: F) -> Result + fn with_orchard_tree_mut(&mut self, mut callback: F) -> Result where for<'a> F: FnMut( &'a mut ShardTree< @@ -1036,16 +1344,28 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb Result, E: From>, { - todo!() + let mut shardtree = ShardTree::new( + SqliteShardStore::from_connection(self.conn.0, ORCHARD_TABLES_PREFIX) + .map_err(|e| ShardTreeError::Storage(commitment_tree::Error::Query(e)))?, + PRUNING_DEPTH.try_into().unwrap(), + ); + let result = callback(&mut shardtree)?; + + Ok(result) } #[cfg(feature = "orchard")] fn put_orchard_subtree_roots( &mut self, - _start_index: u64, - _roots: &[CommitmentTreeRoot], + start_index: u64, + roots: &[CommitmentTreeRoot], ) -> Result<(), ShardTreeError> { - todo!() + put_shard_roots::<_, { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 }, ORCHARD_SHARD_HEIGHT>( + self.conn.0, + ORCHARD_TABLES_PREFIX, + start_index, + roots, + ) } } diff --git a/zcash_client_sqlite/src/testing.rs b/zcash_client_sqlite/src/testing.rs index a37ff5c2ca..250a537fc1 100644 --- a/zcash_client_sqlite/src/testing.rs +++ b/zcash_client_sqlite/src/testing.rs @@ -1,13 +1,15 @@ -use std::convert::Infallible; use std::fmt; use std::num::NonZeroU32; +use std::{collections::BTreeMap, convert::Infallible}; #[cfg(feature = "unstable")] use std::fs::File; +use group::ff::Field; use nonempty::NonEmpty; use prost::Message; -use rand_core::{CryptoRng, OsRng, RngCore}; +use rand_chacha::ChaChaRng; +use rand_core::{CryptoRng, RngCore, SeedableRng}; use rusqlite::{params, Connection}; use secrecy::{Secret, SecretVec}; use tempfile::NamedTempFile; @@ -44,13 +46,14 @@ use zcash_client_backend::{ zip321, }; use zcash_client_backend::{ + data_api::chain::ChainState, fees::{standard, DustOutputPolicy}, ShieldedProtocol, }; use zcash_note_encryption::Domain; use zcash_primitives::{ block::BlockHash, - consensus::{self, BlockHeight, Network, NetworkUpgrade, Parameters}, + consensus::{self, BlockHeight, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, transaction::{ components::{amount::NonNegativeAmount, sapling::zip212_enforcement}, @@ -59,6 +62,7 @@ use zcash_primitives::{ }, zip32::DiversifierIndex, }; +use zcash_protocol::local_consensus::LocalNetwork; use crate::{ chain::init::init_cache_database, @@ -74,9 +78,7 @@ use super::BlockDb; #[cfg(feature = "orchard")] use { - group::ff::{Field, PrimeField}, - orchard::note_encryption::{OrchardDomain, OrchardNoteEncryption}, - pasta_curves::pallas, + group::ff::PrimeField, orchard::tree::MerkleHashOrchard, pasta_curves::pallas, zcash_client_backend::proto::compact_formats::CompactOrchardAction, }; @@ -94,20 +96,37 @@ use crate::{ FsBlockDb, }; +pub(crate) mod pool; + /// A builder for a `zcash_client_sqlite` test. pub(crate) struct TestBuilder { - network: Network, + network: LocalNetwork, cache: Cache, test_account_birthday: Option, + rng: ChaChaRng, } impl TestBuilder<()> { /// Constructs a new test. pub(crate) fn new() -> Self { TestBuilder { - network: Network::TestNetwork, + // Use a fake network where Sapling through NU5 activate at the same height. + // We pick 100,000 to be large enough to handle any hard-coded test offsets. + network: LocalNetwork { + overwinter: Some(BlockHeight::from_u32(1)), + sapling: Some(BlockHeight::from_u32(100_000)), + blossom: Some(BlockHeight::from_u32(100_000)), + heartwood: Some(BlockHeight::from_u32(100_000)), + canopy: Some(BlockHeight::from_u32(100_000)), + nu5: Some(BlockHeight::from_u32(100_000)), + #[cfg(zcash_unstable = "nu6")] + nu6: None, + #[cfg(zcash_unstable = "zfuture")] + z_future: None, + }, cache: (), test_account_birthday: None, + rng: ChaChaRng::seed_from_u64(0), } } @@ -117,6 +136,7 @@ impl TestBuilder<()> { network: self.network, cache: BlockCache::new(), test_account_birthday: self.test_account_birthday, + rng: self.rng, } } @@ -127,12 +147,13 @@ impl TestBuilder<()> { network: self.network, cache: FsBlockCache::new(), test_account_birthday: self.test_account_birthday, + rng: self.rng, } } } impl TestBuilder { - pub(crate) fn with_test_account AccountBirthday>( + pub(crate) fn with_test_account AccountBirthday>( mut self, birthday: F, ) -> Self { @@ -156,26 +177,117 @@ impl TestBuilder { TestState { cache: self.cache, - latest_cached_block: None, + cached_blocks: BTreeMap::new(), + latest_block_height: None, _data_file: data_file, db_data, test_account, + rng: self.rng, } } } +#[derive(Clone, Debug)] +pub(crate) struct CachedBlock { + hash: BlockHash, + chain_state: ChainState, + sapling_end_size: u32, + orchard_end_size: u32, +} + +impl CachedBlock { + fn none(sapling_activation_height: BlockHeight) -> Self { + Self { + hash: BlockHash([0; 32]), + chain_state: ChainState::empty(sapling_activation_height), + sapling_end_size: 0, + orchard_end_size: 0, + } + } + + fn at( + hash: BlockHash, + chain_state: ChainState, + sapling_end_size: u32, + orchard_end_size: u32, + ) -> Self { + assert_eq!( + chain_state.final_sapling_tree().tree_size() as u32, + sapling_end_size + ); + #[cfg(feature = "orchard")] + assert_eq!( + chain_state.final_orchard_tree().tree_size() as u32, + orchard_end_size + ); + + Self { + hash, + chain_state, + sapling_end_size, + orchard_end_size, + } + } + + fn roll_forward(&self, cb: &CompactBlock) -> Self { + assert_eq!(self.chain_state.block_height() + 1, cb.height()); + + let sapling_final_tree = cb.vtx.iter().flat_map(|tx| tx.outputs.iter()).fold( + self.chain_state.final_sapling_tree().clone(), + |mut acc, c_out| { + acc.append(sapling::Node::from_cmu(&c_out.cmu().unwrap())); + acc + }, + ); + let sapling_end_size = sapling_final_tree.tree_size() as u32; + + #[cfg(feature = "orchard")] + let orchard_final_tree = cb.vtx.iter().flat_map(|tx| tx.actions.iter()).fold( + self.chain_state.final_orchard_tree().clone(), + |mut acc, c_act| { + acc.append(MerkleHashOrchard::from_cmx(&c_act.cmx().unwrap())); + acc + }, + ); + #[cfg(feature = "orchard")] + let orchard_end_size = orchard_final_tree.tree_size() as u32; + #[cfg(not(feature = "orchard"))] + let orchard_end_size = cb.vtx.iter().fold(self.orchard_end_size, |sz, tx| { + sz + (tx.actions.len() as u32) + }); + + Self { + hash: cb.hash(), + chain_state: ChainState::new( + cb.height(), + sapling_final_tree, + #[cfg(feature = "orchard")] + orchard_final_tree, + ), + sapling_end_size, + orchard_end_size, + } + } + + fn height(&self) -> BlockHeight { + self.chain_state.block_height() + } +} + /// The state for a `zcash_client_sqlite` test. pub(crate) struct TestState { cache: Cache, - latest_cached_block: Option<(BlockHeight, BlockHash, u32)>, + cached_blocks: BTreeMap, + latest_block_height: Option, _data_file: NamedTempFile, - db_data: WalletDb, + db_data: WalletDb, test_account: Option<( SecretVec, AccountId, UnifiedSpendingKey, AccountBirthday, )>, + rng: ChaChaRng, } impl TestState @@ -188,8 +300,26 @@ where self.cache.block_source() } - pub(crate) fn latest_cached_block(&self) -> &Option<(BlockHeight, BlockHash, u32)> { - &self.latest_cached_block + pub(crate) fn latest_cached_block(&self) -> Option<&CachedBlock> { + self.latest_block_height + .as_ref() + .and_then(|h| self.cached_blocks.get(h)) + } + + fn latest_cached_block_below_height(&self, height: BlockHeight) -> Option<&CachedBlock> { + self.cached_blocks.range(..height).last().map(|(_, b)| b) + } + + fn cache_block( + &mut self, + prev_block: &CachedBlock, + compact_block: CompactBlock, + ) -> Cache::InsertResult { + self.cached_blocks.insert( + compact_block.height(), + prev_block.roll_forward(&compact_block), + ); + self.cache.insert(&compact_block) } /// Creates a fake block at the expected next height containing a single output of the @@ -200,18 +330,18 @@ where req: AddressType, value: NonNegativeAmount, ) -> (BlockHeight, Cache::InsertResult, Fvk::Nullifier) { - let (height, prev_hash, initial_sapling_tree_size) = self - .latest_cached_block - .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); + let pre_activation_block = CachedBlock::none(self.sapling_activation_height() - 1); + let prior_cached_block = self.latest_cached_block().unwrap_or(&pre_activation_block); + let height = prior_cached_block.height() + 1; let (res, nf) = self.generate_block_at( height, - prev_hash, + prior_cached_block.hash, fvk, req, value, - initial_sapling_tree_size, + prior_cached_block.sapling_end_size, + prior_cached_block.orchard_end_size, ); (height, res, nf) @@ -222,6 +352,7 @@ where /// /// This generated block will be treated as the latest block, and subsequent calls to /// [`Self::generate_next_block`] will build on it. + #[allow(clippy::too_many_arguments)] pub(crate) fn generate_block_at( &mut self, height: BlockHeight, @@ -230,7 +361,59 @@ where req: AddressType, value: NonNegativeAmount, initial_sapling_tree_size: u32, + initial_orchard_tree_size: u32, ) -> (Cache::InsertResult, Fvk::Nullifier) { + let mut prior_cached_block = self + .latest_cached_block_below_height(height) + .cloned() + .unwrap_or_else(|| CachedBlock::none(self.sapling_activation_height() - 1)); + assert!(prior_cached_block.chain_state.block_height() < height); + assert!(prior_cached_block.sapling_end_size <= initial_sapling_tree_size); + assert!(prior_cached_block.orchard_end_size <= initial_orchard_tree_size); + + // If the block height has increased or the Sapling and/or Orchard tree sizes have changed, + // we need to generate a new prior cached block that the block to be generated can + // successfully chain from, with the provided tree sizes. + if prior_cached_block.chain_state.block_height() == height - 1 { + assert_eq!(prev_hash, prior_cached_block.hash); + } else { + let final_sapling_tree = + (prior_cached_block.sapling_end_size..initial_sapling_tree_size).fold( + prior_cached_block.chain_state.final_sapling_tree().clone(), + |mut acc, _| { + acc.append(sapling::Node::from_scalar(bls12_381::Scalar::random( + &mut self.rng, + ))); + acc + }, + ); + + #[cfg(feature = "orchard")] + let final_orchard_tree = + (prior_cached_block.orchard_end_size..initial_orchard_tree_size).fold( + prior_cached_block.chain_state.final_orchard_tree().clone(), + |mut acc, _| { + acc.append(MerkleHashOrchard::random(&mut self.rng)); + acc + }, + ); + + prior_cached_block = CachedBlock::at( + prev_hash, + ChainState::new( + height - 1, + final_sapling_tree, + #[cfg(feature = "orchard")] + final_orchard_tree, + ), + initial_sapling_tree_size, + initial_orchard_tree_size, + ); + + self.cached_blocks + .insert(height - 1, prior_cached_block.clone()); + } + let (cb, nf) = fake_compact_block( &self.network(), height, @@ -239,15 +422,13 @@ where req, value, initial_sapling_tree_size, + initial_orchard_tree_size, + &mut self.rng, ); - let res = self.cache.insert(&cb); + assert_eq!(cb.height(), height); - self.latest_cached_block = Some(( - height, - cb.hash(), - initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), - )); + let res = self.cache_block(&prior_cached_block, cb); + self.latest_block_height = Some(height); (res, nf) } @@ -261,29 +442,28 @@ where to: impl Into
, value: NonNegativeAmount, ) -> (BlockHeight, Cache::InsertResult) { - let (height, prev_hash, initial_sapling_tree_size) = self - .latest_cached_block - .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); + let prior_cached_block = self + .latest_cached_block() + .cloned() + .unwrap_or_else(|| CachedBlock::none(self.sapling_activation_height() - 1)); + let height = prior_cached_block.height() + 1; let cb = fake_compact_block_spending( &self.network(), height, - prev_hash, + prior_cached_block.hash, note, fvk, to.into(), value, - initial_sapling_tree_size, + prior_cached_block.sapling_end_size, + prior_cached_block.orchard_end_size, + &mut self.rng, ); - let res = self.cache.insert(&cb); + assert_eq!(cb.height(), height); - self.latest_cached_block = Some(( - height, - cb.hash(), - initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), - )); + let res = self.cache_block(&prior_cached_block, cb); + self.latest_block_height = Some(height); (height, res) } @@ -318,27 +498,25 @@ where tx_index: usize, tx: &Transaction, ) -> (BlockHeight, Cache::InsertResult) { - let (height, prev_hash, initial_sapling_tree_size) = self - .latest_cached_block - .map(|(prev_height, prev_hash, end_size)| (prev_height + 1, prev_hash, end_size)) - .unwrap_or_else(|| (self.sapling_activation_height(), BlockHash([0; 32]), 0)); + let prior_cached_block = self + .latest_cached_block() + .cloned() + .unwrap_or_else(|| CachedBlock::none(self.sapling_activation_height() - 1)); + let height = prior_cached_block.height() + 1; let cb = fake_compact_block_from_tx( height, - prev_hash, + prior_cached_block.hash, tx_index, tx, - initial_sapling_tree_size, - 0, + prior_cached_block.sapling_end_size, + prior_cached_block.orchard_end_size, + &mut self.rng, ); - let res = self.cache.insert(&cb); + assert_eq!(cb.height(), height); - self.latest_cached_block = Some(( - height, - cb.hash(), - initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), - )); + let res = self.cache_block(&prior_cached_block, cb); + self.latest_block_height = Some(height); (height, res) } @@ -366,13 +544,20 @@ where ::Error, >, > { - scan_cached_blocks( + let prior_cached_block = self + .latest_cached_block_below_height(from_height) + .cloned() + .unwrap_or_else(|| CachedBlock::none(from_height - 1)); + + let result = scan_cached_blocks( &self.network(), self.cache.block_source(), &mut self.db_data, from_height, + &prior_cached_block.chain_state, limit, - ) + ); + result } /// Resets the wallet using a new wallet database but with the same cache of blocks, @@ -383,7 +568,7 @@ where /// Before using any `generate_*` method on the reset state, call `reset_latest_cached_block()`. pub(crate) fn reset(&mut self) -> NamedTempFile { let network = self.network(); - self.latest_cached_block = None; + self.latest_block_height = None; let tf = std::mem::replace(&mut self._data_file, NamedTempFile::new().unwrap()); self.db_data = WalletDb::for_path(self._data_file.path(), network).unwrap(); self.test_account = None; @@ -391,36 +576,38 @@ where tf } - /// Reset the latest cached block to the most recent one in the cache database. - #[allow(dead_code)] - pub(crate) fn reset_latest_cached_block(&mut self) { - self.cache - .block_source() - .with_blocks::<_, Infallible>(None, None, |block: CompactBlock| { - self.latest_cached_block = Some(( - BlockHeight::from_u32(block.height.try_into().unwrap()), - BlockHash::from_slice(block.hash.as_slice()), - block.chain_metadata.unwrap().sapling_commitment_tree_size, - )); - Ok(()) - }) - .unwrap(); - } + // /// Reset the latest cached block to the most recent one in the cache database. + // #[allow(dead_code)] + // pub(crate) fn reset_latest_cached_block(&mut self) { + // self.cache + // .block_source() + // .with_blocks::<_, Infallible>(None, None, |block: CompactBlock| { + // let chain_metadata = block.chain_metadata.unwrap(); + // self.latest_cached_block = Some(CachedBlock::at( + // BlockHash::from_slice(block.hash.as_slice()), + // BlockHeight::from_u32(block.height.try_into().unwrap()), + // chain_metadata.sapling_commitment_tree_size, + // chain_metadata.orchard_commitment_tree_size, + // )); + // Ok(()) + // }) + // .unwrap(); + // } } impl TestState { /// Exposes an immutable reference to the test's [`WalletDb`]. - pub(crate) fn wallet(&self) -> &WalletDb { + pub(crate) fn wallet(&self) -> &WalletDb { &self.db_data } /// Exposes a mutable reference to the test's [`WalletDb`]. - pub(crate) fn wallet_mut(&mut self) -> &mut WalletDb { + pub(crate) fn wallet_mut(&mut self) -> &mut WalletDb { &mut self.db_data } /// Exposes the network in use. - pub(crate) fn network(&self) -> Network { + pub(crate) fn network(&self) -> LocalNetwork { self.db_data.params } @@ -451,6 +638,14 @@ impl TestState { .and_then(|(_, _, usk, _)| usk.to_unified_full_viewing_key().sapling().cloned()) } + /// Exposes the test account's Sapling DFVK, if enabled via [`TestBuilder::with_test_account`]. + #[cfg(feature = "orchard")] + pub(crate) fn test_account_orchard(&self) -> Option { + self.test_account + .as_ref() + .and_then(|(_, _, usk, _)| usk.to_unified_full_viewing_key().orchard().cloned()) + } + /// Invokes [`create_spend_to_address`] with the given arguments. #[allow(deprecated)] #[allow(clippy::type_complexity)] @@ -511,7 +706,7 @@ impl TestState { >, > where - InputsT: InputSelector>, + InputsT: InputSelector>, { #![allow(deprecated)] let params = self.network(); @@ -547,7 +742,7 @@ impl TestState { >, > where - InputsT: InputSelector>, + InputsT: InputSelector>, { let params = self.network(); propose_transfer::<_, _, _, Infallible>( @@ -623,7 +818,7 @@ impl TestState { >, > where - InputsT: ShieldingSelector>, + InputsT: ShieldingSelector>, { let params = self.network(); propose_shielding::<_, _, _, Infallible>( @@ -687,7 +882,7 @@ impl TestState { >, > where - InputsT: ShieldingSelector>, + InputsT: ShieldingSelector>, { let params = self.network(); let prover = test_prover(); @@ -724,7 +919,7 @@ impl TestState { min_confirmations: u32, ) -> NonNegativeAmount { self.with_account_balance(account, min_confirmations, |balance| { - balance.sapling_balance().spendable_value() + balance.spendable_value() }) } @@ -734,8 +929,7 @@ impl TestState { min_confirmations: u32, ) -> NonNegativeAmount { self.with_account_balance(account, min_confirmations, |balance| { - balance.sapling_balance().value_pending_spendability() - + balance.sapling_balance().change_pending_confirmation() + balance.value_pending_spendability() + balance.change_pending_confirmation() }) .unwrap() } @@ -747,7 +941,7 @@ impl TestState { min_confirmations: u32, ) -> NonNegativeAmount { self.with_account_balance(account, min_confirmations, |balance| { - balance.sapling_balance().change_pending_confirmation() + balance.change_pending_confirmation() }) } @@ -881,7 +1075,7 @@ impl TestFvk for orchard::keys::FullViewingKey { fn add_spend( &self, ctx: &mut CompactTx, - nf: Self::Nullifier, + revealed_spent_note_nullifier: Self::Nullifier, rng: &mut R, ) { // Generate a dummy recipient. @@ -896,7 +1090,7 @@ impl TestFvk for orchard::keys::FullViewingKey { }; let (cact, _) = compact_orchard_action( - nf, + revealed_spent_note_nullifier, recipient, NonNegativeAmount::ZERO, self.orchard_ovk(zip32::Scope::Internal), @@ -916,7 +1110,7 @@ impl TestFvk for orchard::keys::FullViewingKey { mut rng: &mut R, ) -> Self::Nullifier { // Generate a dummy nullifier - let nullifier = + let revealed_spent_note_nullifier = orchard::note::Nullifier::from_bytes(&pallas::Base::random(&mut rng).to_repr()) .unwrap(); @@ -927,7 +1121,7 @@ impl TestFvk for orchard::keys::FullViewingKey { }; let (cact, note) = compact_orchard_action( - nullifier, + revealed_spent_note_nullifier, self.address_at(j, scope), value, self.orchard_ovk(scope), @@ -944,7 +1138,7 @@ impl TestFvk for orchard::keys::FullViewingKey { ctx: &mut CompactTx, _: &P, _: BlockHeight, - nf: Self::Nullifier, + revealed_spent_note_nullifier: Self::Nullifier, req: AddressType, value: NonNegativeAmount, _: u32, @@ -957,7 +1151,7 @@ impl TestFvk for orchard::keys::FullViewingKey { }; let (cact, note) = compact_orchard_action( - nf, + revealed_spent_note_nullifier, self.address_at(j, scope), value, self.orchard_ovk(scope), @@ -965,6 +1159,7 @@ impl TestFvk for orchard::keys::FullViewingKey { ); ctx.actions.push(cact); + // Return the nullifier of the newly created output note note.nullifier(self) } } @@ -1013,44 +1208,28 @@ fn compact_sapling_output( /// Returns the `CompactOrchardAction` and the new note. #[cfg(feature = "orchard")] fn compact_orchard_action( - nullifier: orchard::note::Nullifier, + nf_old: orchard::note::Nullifier, recipient: orchard::Address, value: NonNegativeAmount, ovk: Option, rng: &mut R, ) -> (CompactOrchardAction, orchard::Note) { - let nf = nullifier.to_bytes().to_vec(); + use zcash_note_encryption::ShieldedOutput; - let rseed = { - loop { - let mut bytes = [0; 32]; - rng.fill_bytes(&mut bytes); - let rseed = orchard::note::RandomSeed::from_bytes(bytes, &nullifier); - if rseed.is_some().into() { - break rseed.unwrap(); - } - } - }; - let note = orchard::Note::from_parts( + let (compact_action, note) = orchard::note_encryption::testing::fake_compact_action( + rng, + nf_old, recipient, orchard::value::NoteValue::from_raw(value.into_u64()), - nullifier, - rseed, - ) - .unwrap(); - let encryptor = OrchardNoteEncryption::new(ovk, note, *MemoBytes::empty().as_array()); - let cmx = orchard::note::ExtractedNoteCommitment::from(note.commitment()) - .to_bytes() - .to_vec(); - let ephemeral_key = OrchardDomain::epk_bytes(encryptor.epk()).0.to_vec(); - let enc_ciphertext = encryptor.encrypt_note_plaintext(); + ovk, + ); ( CompactOrchardAction { - nullifier: nf, - cmx, - ephemeral_key, - ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + nullifier: compact_action.nullifier().to_bytes().to_vec(), + cmx: compact_action.cmx().to_bytes().to_vec(), + ephemeral_key: compact_action.ephemeral_key().0.to_vec(), + ciphertext: compact_action.enc_ciphertext().as_ref()[..52].to_vec(), }, note, ) @@ -1068,6 +1247,7 @@ fn fake_compact_tx(rng: &mut R) -> CompactTx { /// Create a fake CompactBlock at the given height, containing a single output paying /// an address. Returns the CompactBlock and the nullifier for the new note. +#[allow(clippy::too_many_arguments)] fn fake_compact_block( params: &P, height: BlockHeight, @@ -1076,10 +1256,9 @@ fn fake_compact_block( req: AddressType, value: NonNegativeAmount, initial_sapling_tree_size: u32, + initial_orchard_tree_size: u32, + mut rng: impl RngCore + CryptoRng, ) -> (CompactBlock, Fvk::Nullifier) { - // Create a fake Note for the account - let mut rng = OsRng; - // Create a fake CompactBlock containing the note let mut ctx = fake_compact_tx(&mut rng); let nf = fvk.add_output( @@ -1092,8 +1271,14 @@ fn fake_compact_block( &mut rng, ); - let cb = - fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size, 0); + let cb = fake_compact_block_from_compact_tx( + ctx, + height, + prev_hash, + initial_sapling_tree_size, + initial_orchard_tree_size, + rng, + ); (cb, nf) } @@ -1105,6 +1290,7 @@ fn fake_compact_block_from_tx( tx: &Transaction, initial_sapling_tree_size: u32, initial_orchard_tree_size: u32, + rng: impl RngCore, ) -> CompactBlock { // Create a fake CompactTx containing the transaction. let mut ctx = CompactTx { @@ -1135,6 +1321,7 @@ fn fake_compact_block_from_tx( prev_hash, initial_sapling_tree_size, initial_orchard_tree_size, + rng, ) } @@ -1150,8 +1337,9 @@ fn fake_compact_block_spending( to: Address, value: NonNegativeAmount, initial_sapling_tree_size: u32, + initial_orchard_tree_size: u32, + mut rng: impl RngCore + CryptoRng, ) -> CompactBlock { - let mut rng = OsRng; let mut ctx = fake_compact_tx(&mut rng); // Create a fake spend and a fake Note for the change @@ -1227,7 +1415,14 @@ fn fake_compact_block_spending( } } - fake_compact_block_from_compact_tx(ctx, height, prev_hash, initial_sapling_tree_size, 0) + fake_compact_block_from_compact_tx( + ctx, + height, + prev_hash, + initial_sapling_tree_size, + initial_orchard_tree_size, + rng, + ) } fn fake_compact_block_from_compact_tx( @@ -1236,8 +1431,8 @@ fn fake_compact_block_from_compact_tx( prev_hash: BlockHash, initial_sapling_tree_size: u32, initial_orchard_tree_size: u32, + mut rng: impl RngCore, ) -> CompactBlock { - let mut rng = OsRng; let mut cb = CompactBlock { hash: { let mut hash = vec![0; 32]; @@ -1364,7 +1559,7 @@ pub(crate) fn input_selector( change_memo: Option<&str>, fallback_change_pool: ShieldedProtocol, ) -> GreedyInputSelector< - WalletDb, + WalletDb, standard::SingleOutputChangeStrategy, > { let change_memo = change_memo.map(|m| MemoBytes::from(m.parse::().unwrap())); @@ -1376,7 +1571,7 @@ pub(crate) fn input_selector( // Checks that a protobuf proposal serialized from the provided proposal value correctly parses to // the same proposal value. fn check_proposal_serialization_roundtrip( - db_data: &WalletDb, + db_data: &WalletDb, proposal: &Proposal, ) { let proposal_proto = proposal::Proposal::from_standard_proposal(&db_data.params, proposal); diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs new file mode 100644 index 0000000000..637d4d5ba5 --- /dev/null +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -0,0 +1,1795 @@ +//! Test logic involving a single shielded pool. +//! +//! Generalised for sharing across the Sapling and Orchard implementations. + +use std::{convert::Infallible, num::NonZeroU32}; + +use incrementalmerkletree::Level; +use rusqlite::params; +use secrecy::Secret; +use shardtree::error::ShardTreeError; +use zcash_primitives::{ + block::BlockHash, + consensus::BranchId, + legacy::TransparentAddress, + memo::{Memo, MemoBytes}, + transaction::{ + components::amount::NonNegativeAmount, + fees::{ + fixed::FeeRule as FixedFeeRule, zip317::FeeError as Zip317FeeError, StandardFeeRule, + }, + Transaction, + }, + zip32::Scope, +}; + +use zcash_client_backend::{ + address::Address, + data_api::{ + self, + chain::{self, CommitmentTreeRoot, ScanSummary}, + error::Error, + wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, + AccountBirthday, DecryptedTransaction, Ratio, WalletRead, WalletSummary, WalletWrite, + }, + decrypt_transaction, + fees::{fixed, standard, DustOutputPolicy}, + keys::UnifiedSpendingKey, + scanning::ScanError, + wallet::{Note, OvkPolicy, ReceivedNote}, + zip321::{self, Payment, TransactionRequest}, + ShieldedProtocol, +}; +use zcash_protocol::consensus::BlockHeight; + +use super::TestFvk; +use crate::{ + error::SqliteClientError, + testing::{input_selector, AddressType, BlockCache, TestBuilder, TestState}, + wallet::{ + block_max_scanned, commitment_tree, parse_scope, + scanning::tests::test_with_nu5_birthday_offset, truncate_to_height, + }, + AccountId, NoteId, ReceivedNoteId, +}; + +#[cfg(feature = "orchard")] +use zcash_primitives::consensus::NetworkUpgrade; + +#[cfg(feature = "transparent-inputs")] +use { + zcash_client_backend::{ + fees::TransactionBalance, proposal::Step, wallet::WalletTransparentOutput, PoolType, + }, + zcash_primitives::{ + legacy::keys::IncomingViewingKey, + transaction::components::{OutPoint, TxOut}, + }, +}; + +pub(crate) type OutputRecoveryError = Error< + SqliteClientError, + commitment_tree::Error, + GreedyInputSelectorError, + Zip317FeeError, +>; + +/// Trait that exposes the pool-specific types and operations necessary to run the +/// single-shielded-pool tests on a given pool. +pub(crate) trait ShieldedPoolTester { + const SHIELDED_PROTOCOL: ShieldedProtocol; + const TABLES_PREFIX: &'static str; + + type Sk; + type Fvk: TestFvk; + type MerkleTreeHash; + + fn test_account_fvk(st: &TestState) -> Self::Fvk; + fn usk_to_sk(usk: &UnifiedSpendingKey) -> &Self::Sk; + fn sk(seed: &[u8]) -> Self::Sk; + fn sk_to_fvk(sk: &Self::Sk) -> Self::Fvk; + fn sk_default_address(sk: &Self::Sk) -> Address; + fn fvk_default_address(fvk: &Self::Fvk) -> Address; + fn fvks_equal(a: &Self::Fvk, b: &Self::Fvk) -> bool; + + fn empty_tree_leaf() -> Self::MerkleTreeHash; + fn empty_tree_root(level: Level) -> Self::MerkleTreeHash; + + fn put_subtree_roots( + st: &mut TestState, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError>; + + fn next_subtree_index(s: &WalletSummary) -> u64; + + fn select_spendable_notes( + st: &TestState, + account: AccountId, + target_value: NonNegativeAmount, + anchor_height: BlockHeight, + exclude: &[ReceivedNoteId], + ) -> Result>, SqliteClientError>; + + fn decrypted_pool_outputs_count(d_tx: &DecryptedTransaction<'_, AccountId>) -> usize; + + fn with_decrypted_pool_memos( + d_tx: &DecryptedTransaction<'_, AccountId>, + f: impl FnMut(&MemoBytes), + ); + + fn try_output_recovery( + st: &TestState, + height: BlockHeight, + tx: &Transaction, + fvk: &Self::Fvk, + ) -> Result, OutputRecoveryError>; + + fn received_note_count(summary: &ScanSummary) -> usize; +} + +pub(crate) fn send_single_step_proposed_transfer() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::const_from_u64(60000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); + + // Spendable balance matches total balance + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + assert_eq!( + block_max_scanned(&st.wallet().conn, &st.wallet().params) + .unwrap() + .unwrap() + .block_height(), + h + ); + + let to_extsk = T::sk(&[0xf5; 32]); + let to: Address = T::sk_default_address(&to_extsk); + let request = zip321::TransactionRequest::new(vec![Payment { + recipient_address: to, + amount: NonNegativeAmount::const_from_u64(10000), + memo: None, // this should result in the creation of an empty memo + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + // TODO: This test was originally written to use the pre-zip-313 fee rule + // and has not yet been updated. + #[allow(deprecated)] + let fee_rule = StandardFeeRule::PreZip313; + + let change_memo = "Test change memo".parse::().unwrap(); + let change_strategy = standard::SingleOutputChangeStrategy::new( + fee_rule, + Some(change_memo.clone().into()), + T::SHIELDED_PROTOCOL, + ); + let input_selector = &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); + + let proposal = st + .propose_transfer( + account, + input_selector, + request, + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); + + let create_proposed_result = + st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal); + assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 1); + + let sent_tx_id = create_proposed_result.unwrap()[0]; + + // Verify that the sent transaction was stored and that we can decrypt the memos + let tx = st + .wallet() + .get_transaction(sent_tx_id) + .expect("Created transaction was stored."); + let ufvks = [(account, usk.to_unified_full_viewing_key())] + .into_iter() + .collect(); + let d_tx = decrypt_transaction(&st.network(), h + 1, &tx, &ufvks); + assert_eq!(T::decrypted_pool_outputs_count(&d_tx), 2); + + let mut found_tx_change_memo = false; + let mut found_tx_empty_memo = false; + T::with_decrypted_pool_memos(&d_tx, |memo| { + if Memo::try_from(memo).unwrap() == change_memo { + found_tx_change_memo = true + } + if Memo::try_from(memo).unwrap() == Memo::Empty { + found_tx_empty_memo = true + } + }); + assert!(found_tx_change_memo); + assert!(found_tx_empty_memo); + + // Verify that the stored sent notes match what we're expecting + let mut stmt_sent_notes = st + .wallet() + .conn + .prepare( + "SELECT output_index + FROM sent_notes + JOIN transactions ON transactions.id_tx = sent_notes.tx + WHERE transactions.txid = ?", + ) + .unwrap(); + + let sent_note_ids = stmt_sent_notes + .query(rusqlite::params![sent_tx_id.as_ref()]) + .unwrap() + .mapped(|row| Ok(NoteId::new(sent_tx_id, T::SHIELDED_PROTOCOL, row.get(0)?))) + .collect::, _>>() + .unwrap(); + + assert_eq!(sent_note_ids.len(), 2); + + // The sent memo should be the empty memo for the sent output, and the + // change output's memo should be as specified. + let mut found_sent_change_memo = false; + let mut found_sent_empty_memo = false; + for sent_note_id in sent_note_ids { + match st + .wallet() + .get_memo(sent_note_id) + .expect("Note id is valid") + .as_ref() + { + Some(m) if m == &change_memo => { + found_sent_change_memo = true; + } + Some(m) if m == &Memo::Empty => { + found_sent_empty_memo = true; + } + Some(other) => panic!("Unexpected memo value: {:?}", other), + None => panic!("Memo should not be stored as NULL"), + } + } + assert!(found_sent_change_memo); + assert!(found_sent_empty_memo); + + // Check that querying for a nonexistent sent note returns None + assert_matches!( + st.wallet() + .get_memo(NoteId::new(sent_tx_id, T::SHIELDED_PROTOCOL, 12345)), + Ok(None) + ); +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn send_multi_step_proposed_transfer() { + use nonempty::NonEmpty; + use zcash_client_backend::proposal::{Proposal, StepOutput, StepOutputIndex}; + + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::const_from_u64(65000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); + + // Spendable balance matches total balance + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + assert_eq!( + block_max_scanned(&st.wallet().conn, &st.wallet().params) + .unwrap() + .unwrap() + .block_height(), + h + ); + + // Generate a single-step proposal. Then, instead of executing that proposal, + // we will use its only step as the first step in a multi-step proposal that + // spends the first step's output. + + // The first step will deshield to the wallet's default transparent address + let to0 = Address::Transparent(usk.default_transparent_address().0); + let request0 = zip321::TransactionRequest::new(vec![Payment { + recipient_address: to0, + amount: NonNegativeAmount::const_from_u64(50000), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + let fee_rule = StandardFeeRule::Zip317; + let input_selector = GreedyInputSelector::new( + standard::SingleOutputChangeStrategy::new(fee_rule, None, T::SHIELDED_PROTOCOL), + DustOutputPolicy::default(), + ); + let proposal0 = st + .propose_transfer( + account, + &input_selector, + request0, + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); + + let min_target_height = proposal0.min_target_height(); + let step0 = &proposal0.steps().head; + + assert!(step0.balance().proposed_change().is_empty()); + assert_eq!( + step0.balance().fee_required(), + NonNegativeAmount::const_from_u64(15000) + ); + + // We'll use an internal transparent address that hasn't been added to the wallet + // to simulate an external transparent recipient. + let to1 = Address::Transparent( + usk.transparent() + .to_account_pubkey() + .derive_internal_ivk() + .unwrap() + .default_address() + .0, + ); + let request1 = zip321::TransactionRequest::new(vec![Payment { + recipient_address: to1, + amount: NonNegativeAmount::const_from_u64(40000), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + let step1 = Step::from_parts( + &[step0.clone()], + request1, + [(0, PoolType::Transparent)].into_iter().collect(), + vec![], + None, + vec![StepOutput::new(0, StepOutputIndex::Payment(0))], + TransactionBalance::new(vec![], NonNegativeAmount::const_from_u64(10000)).unwrap(), + false, + ) + .unwrap(); + + let proposal = Proposal::multi_step( + fee_rule, + min_target_height, + NonEmpty::from_vec(vec![step0.clone(), step1]).unwrap(), + ) + .unwrap(); + + let create_proposed_result = + st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal); + assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); + let txids = create_proposed_result.unwrap(); + + // Verify that the stored sent outputs match what we're expecting + let mut stmt_sent = st + .wallet() + .conn + .prepare( + "SELECT value + FROM sent_notes + JOIN transactions ON transactions.id_tx = sent_notes.tx + WHERE transactions.txid = ?", + ) + .unwrap(); + + let confirmed_sent = txids + .iter() + .map(|sent_txid| { + // check that there's a sent output with the correct value corresponding to + stmt_sent + .query(rusqlite::params![sent_txid.as_ref()]) + .unwrap() + .mapped(|row| { + let value: u32 = row.get(0)?; + Ok((sent_txid, value)) + }) + .collect::, _>>() + .unwrap() + }) + .collect::>(); + + assert_eq!( + confirmed_sent.get(0).and_then(|v| v.get(0)), + Some(&(&txids[0], 50000)) + ); + assert_eq!( + confirmed_sent.get(1).and_then(|v| v.get(0)), + Some(&(&txids[1], 40000)) + ); +} + +#[allow(deprecated)] +pub(crate) fn create_to_address_fails_on_incorrect_usk() { + let mut st = TestBuilder::new() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + let dfvk = T::test_account_fvk(&st); + let to = T::fvk_default_address(&dfvk); + + // Create a USK that doesn't exist in the wallet + let acct1 = zip32::AccountId::try_from(1).unwrap(); + let usk1 = UnifiedSpendingKey::from_seed(&st.network(), &[1u8; 32], acct1).unwrap(); + + // Attempting to spend with a USK that is not in the wallet results in an error + assert_matches!( + st.create_spend_to_address( + &usk1, + &to, + NonNegativeAmount::const_from_u64(1), + None, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + None, + T::SHIELDED_PROTOCOL, + ), + Err(data_api::error::Error::KeyNotRecognized) + ); +} + +#[allow(deprecated)] +pub(crate) fn proposal_fails_with_no_blocks() { + let mut st = TestBuilder::new() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, _, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + let to = T::fvk_default_address(&dfvk); + + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); + + // We cannot do anything if we aren't synchronised + assert_matches!( + st.propose_standard_transfer::( + account, + StandardFeeRule::PreZip313, + NonZeroU32::new(1).unwrap(), + &to, + NonNegativeAmount::const_from_u64(1), + None, + None, + T::SHIELDED_PROTOCOL, + ), + Err(data_api::error::Error::ScanRequired) + ); +} + +pub(crate) fn spend_fails_on_unverified_notes() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); + + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + // Value is considered pending at 10 confirmations. + assert_eq!(st.get_pending_shielded_balance(account, 10), value); + assert_eq!( + st.get_spendable_balance(account, 10), + NonNegativeAmount::ZERO + ); + + // Wallet is fully scanned + let summary = st.get_wallet_summary(1); + assert_eq!( + summary.and_then(|s| s.scan_progress()), + Some(Ratio::new(1, 1)) + ); + + // Add more funds to the wallet in a second note + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h2, 1); + + // Verified balance does not include the second note + let total = (value + value).unwrap(); + assert_eq!(st.get_spendable_balance(account, 2), value); + assert_eq!(st.get_pending_shielded_balance(account, 2), value); + assert_eq!(st.get_total_balance(account), total); + + // Wallet is still fully scanned + let summary = st.get_wallet_summary(1); + assert_eq!( + summary.and_then(|s| s.scan_progress()), + Some(Ratio::new(2, 2)) + ); + + // Spend fails because there are insufficient verified notes + let extsk2 = T::sk(&[0xf5; 32]); + let to = T::sk_default_address(&extsk2); + assert_matches!( + st.propose_standard_transfer::( + account, + StandardFeeRule::Zip317, + NonZeroU32::new(2).unwrap(), + &to, + NonNegativeAmount::const_from_u64(70000), + None, + None, + T::SHIELDED_PROTOCOL, + ), + Err(data_api::error::Error::InsufficientFunds { + available, + required + }) + if available == NonNegativeAmount::const_from_u64(50000) + && required == NonNegativeAmount::const_from_u64(80000) + ); + + // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second + // note is verified + for _ in 2..10 { + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + } + st.scan_cached_blocks(h2 + 1, 8); + + // Total balance is value * number of blocks scanned (10). + assert_eq!(st.get_total_balance(account), (value * 10).unwrap()); + + // Spend still fails + assert_matches!( + st.propose_standard_transfer::( + account, + StandardFeeRule::Zip317, + NonZeroU32::new(10).unwrap(), + &to, + NonNegativeAmount::const_from_u64(70000), + None, + None, + T::SHIELDED_PROTOCOL, + ), + Err(data_api::error::Error::InsufficientFunds { + available, + required + }) + if available == NonNegativeAmount::const_from_u64(50000) + && required == NonNegativeAmount::const_from_u64(80000) + ); + + // Mine block 11 so that the second note becomes verified + let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h11, 1); + + // Total balance is value * number of blocks scanned (11). + assert_eq!(st.get_total_balance(account), (value * 11).unwrap()); + // Spendable balance at 10 confirmations is value * 2. + assert_eq!(st.get_spendable_balance(account, 10), (value * 2).unwrap()); + assert_eq!( + st.get_pending_shielded_balance(account, 10), + (value * 9).unwrap() + ); + + // Should now be able to generate a proposal + let amount_sent = NonNegativeAmount::from_u64(70000).unwrap(); + let min_confirmations = NonZeroU32::new(10).unwrap(); + let proposal = st + .propose_standard_transfer::( + account, + StandardFeeRule::Zip317, + min_confirmations, + &to, + amount_sent, + None, + None, + T::SHIELDED_PROTOCOL, + ) + .unwrap(); + + // Executing the proposal should succeed + let txid = st + .create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal) + .unwrap()[0]; + + let (h, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(h, 1); + + // TODO: send to an account so that we can check its balance. + assert_eq!( + st.get_total_balance(account), + ((value * 11).unwrap() + - (amount_sent + NonNegativeAmount::from_u64(10000).unwrap()).unwrap()) + .unwrap() + ); +} + +pub(crate) fn spend_fails_on_locked_notes() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // TODO: This test was originally written to use the pre-zip-313 fee rule + // and has not yet been updated. + #[allow(deprecated)] + let fee_rule = StandardFeeRule::PreZip313; + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); + + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + // Send some of the funds to another address, but don't mine the tx. + let extsk2 = T::sk(&[0xf5; 32]); + let to = T::sk_default_address(&extsk2); + let min_confirmations = NonZeroU32::new(1).unwrap(); + let proposal = st + .propose_standard_transfer::( + account, + fee_rule, + min_confirmations, + &to, + NonNegativeAmount::const_from_u64(15000), + None, + None, + T::SHIELDED_PROTOCOL, + ) + .unwrap(); + + // Executing the proposal should succeed + assert_matches!( + st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal,), + Ok(txids) if txids.len() == 1 + ); + + // A second proposal fails because there are no usable notes + assert_matches!( + st.propose_standard_transfer::( + account, + fee_rule, + NonZeroU32::new(1).unwrap(), + &to, + NonNegativeAmount::const_from_u64(2000), + None, + None, + T::SHIELDED_PROTOCOL, + ), + Err(data_api::error::Error::InsufficientFunds { + available, + required + }) + if available == NonNegativeAmount::ZERO && required == NonNegativeAmount::const_from_u64(12000) + ); + + // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) + // until just before the first transaction expires + for i in 1..42 { + st.generate_next_block( + &T::sk_to_fvk(&T::sk(&[i as u8; 32])), + AddressType::DefaultExternal, + value, + ); + } + st.scan_cached_blocks(h1 + 1, 41); + + // Second proposal still fails + assert_matches!( + st.propose_standard_transfer::( + account, + fee_rule, + NonZeroU32::new(1).unwrap(), + &to, + NonNegativeAmount::const_from_u64(2000), + None, + None, + T::SHIELDED_PROTOCOL, + ), + Err(data_api::error::Error::InsufficientFunds { + available, + required + }) + if available == NonNegativeAmount::ZERO && required == NonNegativeAmount::const_from_u64(12000) + ); + + // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires + let (h43, _, _) = st.generate_next_block( + &T::sk_to_fvk(&T::sk(&[42; 32])), + AddressType::DefaultExternal, + value, + ); + st.scan_cached_blocks(h43, 1); + + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + // Second spend should now succeed + let amount_sent2 = NonNegativeAmount::const_from_u64(2000); + let min_confirmations = NonZeroU32::new(1).unwrap(); + let proposal = st + .propose_standard_transfer::( + account, + fee_rule, + min_confirmations, + &to, + amount_sent2, + None, + None, + T::SHIELDED_PROTOCOL, + ) + .unwrap(); + + let txid2 = st + .create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal) + .unwrap()[0]; + + let (h, _) = st.generate_next_block_including(txid2); + st.scan_cached_blocks(h, 1); + + // TODO: send to an account so that we can check its balance. + assert_eq!( + st.get_total_balance(account), + (value - (amount_sent2 + NonNegativeAmount::from_u64(10000).unwrap()).unwrap()).unwrap() + ); +} + +pub(crate) fn ovk_policy_prevents_recovery_from_chain() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); + + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + let extsk2 = T::sk(&[0xf5; 32]); + let addr2 = T::sk_default_address(&extsk2); + + // TODO: This test was originally written to use the pre-zip-313 fee rule + // and has not yet been updated. + #[allow(deprecated)] + let fee_rule = StandardFeeRule::PreZip313; + + #[allow(clippy::type_complexity)] + let send_and_recover_with_policy = |st: &mut TestState, + ovk_policy| + -> Result< + Option<(Note, Address, MemoBytes)>, + Error< + SqliteClientError, + commitment_tree::Error, + GreedyInputSelectorError, + Zip317FeeError, + >, + > { + let min_confirmations = NonZeroU32::new(1).unwrap(); + let proposal = st.propose_standard_transfer( + account, + fee_rule, + min_confirmations, + &addr2, + NonNegativeAmount::const_from_u64(15000), + None, + None, + T::SHIELDED_PROTOCOL, + )?; + + // Executing the proposal should succeed + let txid = st.create_proposed_transactions(&usk, ovk_policy, &proposal)?[0]; + + // Fetch the transaction from the database + let raw_tx: Vec<_> = st + .wallet() + .conn + .query_row( + "SELECT raw FROM transactions + WHERE txid = ?", + [txid.as_ref()], + |row| row.get(0), + ) + .unwrap(); + let tx = Transaction::read(&raw_tx[..], BranchId::Canopy).unwrap(); + + T::try_output_recovery(st, h1, &tx, &dfvk) + }; + + // Send some of the funds to another address, keeping history. + // The recipient output is decryptable by the sender. + assert_matches!( + send_and_recover_with_policy(&mut st, OvkPolicy::Sender), + Ok(Some((_, recovered_to, _))) if recovered_to == addr2 + ); + + // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) + // so that the first transaction expires + for i in 1..=42 { + st.generate_next_block( + &T::sk_to_fvk(&T::sk(&[i as u8; 32])), + AddressType::DefaultExternal, + value, + ); + } + st.scan_cached_blocks(h1 + 1, 42); + + // Send the funds again, discarding history. + // Neither transaction output is decryptable by the sender. + assert_matches!( + send_and_recover_with_policy(&mut st, OvkPolicy::Discard), + Ok(None) + ); +} + +pub(crate) fn spend_succeeds_to_t_addr_zero_change() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::const_from_u64(60000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); + + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + // TODO: This test was originally written to use the pre-zip-313 fee rule + // and has not yet been updated. + #[allow(deprecated)] + let fee_rule = StandardFeeRule::PreZip313; + + // TODO: generate_next_block_from_tx does not currently support transparent outputs. + let to = TransparentAddress::PublicKeyHash([7; 20]).into(); + let min_confirmations = NonZeroU32::new(1).unwrap(); + let proposal = st + .propose_standard_transfer::( + account, + fee_rule, + min_confirmations, + &to, + NonNegativeAmount::const_from_u64(50000), + None, + None, + T::SHIELDED_PROTOCOL, + ) + .unwrap(); + + // Executing the proposal should succeed + assert_matches!( + st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal), + Ok(txids) if txids.len() == 1 + ); +} + +pub(crate) fn change_note_spends_succeed() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet in a single note owned by the internal spending key + let value = NonNegativeAmount::const_from_u64(60000); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value); + st.scan_cached_blocks(h, 1); + + // Spendable balance matches total balance at 1 confirmation. + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + + // Value is considered pending at 10 confirmations. + assert_eq!(st.get_pending_shielded_balance(account, 10), value); + assert_eq!( + st.get_spendable_balance(account, 10), + NonNegativeAmount::ZERO + ); + + let change_note_scope = st.wallet().conn.query_row( + &format!( + "SELECT recipient_key_scope + FROM {}_received_notes + WHERE value = ?", + T::TABLES_PREFIX, + ), + params![u64::from(value)], + |row| Ok(parse_scope(row.get(0)?)), + ); + assert_matches!(change_note_scope, Ok(Some(Scope::Internal))); + + // TODO: This test was originally written to use the pre-zip-313 fee rule + // and has not yet been updated. + #[allow(deprecated)] + let fee_rule = StandardFeeRule::PreZip313; + + // TODO: generate_next_block_from_tx does not currently support transparent outputs. + let to = TransparentAddress::PublicKeyHash([7; 20]).into(); + let min_confirmations = NonZeroU32::new(1).unwrap(); + let proposal = st + .propose_standard_transfer::( + account, + fee_rule, + min_confirmations, + &to, + NonNegativeAmount::const_from_u64(50000), + None, + None, + T::SHIELDED_PROTOCOL, + ) + .unwrap(); + + // Executing the proposal should succeed + assert_matches!( + st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal), + Ok(txids) if txids.len() == 1 + ); +} + +pub(crate) fn external_address_change_spends_detected_in_restore_from_seed< + T: ShieldedPoolTester, +>() { + let mut st = TestBuilder::new().with_block_cache().build(); + + // Add two accounts to the wallet. + let seed = Secret::new([0u8; 32].to_vec()); + let birthday = AccountBirthday::from_sapling_activation(&st.network()); + let (account, usk) = st + .wallet_mut() + .create_account(&seed, birthday.clone()) + .unwrap(); + let dfvk = T::sk_to_fvk(T::usk_to_sk(&usk)); + + let (account2, usk2) = st + .wallet_mut() + .create_account(&seed, birthday.clone()) + .unwrap(); + let dfvk2 = T::sk_to_fvk(T::usk_to_sk(&usk2)); + + // Add funds to the wallet in a single note + let value = NonNegativeAmount::from_u64(100000).unwrap(); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h, 1); + + // Spendable balance matches total balance + assert_eq!(st.get_total_balance(account), value); + assert_eq!(st.get_spendable_balance(account, 1), value); + assert_eq!(st.get_total_balance(account2), NonNegativeAmount::ZERO); + + let amount_sent = NonNegativeAmount::from_u64(20000).unwrap(); + let amount_legacy_change = NonNegativeAmount::from_u64(30000).unwrap(); + let addr = T::fvk_default_address(&dfvk); + let addr2 = T::fvk_default_address(&dfvk2); + let req = TransactionRequest::new(vec![ + // payment to an external recipient + Payment { + recipient_address: addr2, + amount: amount_sent, + memo: None, + label: None, + message: None, + other_params: vec![], + }, + // payment back to the originating wallet, simulating legacy change + Payment { + recipient_address: addr, + amount: amount_legacy_change, + memo: None, + label: None, + message: None, + other_params: vec![], + }, + ]) + .unwrap(); + + #[allow(deprecated)] + let fee_rule = FixedFeeRule::standard(); + let input_selector = GreedyInputSelector::new( + fixed::SingleOutputChangeStrategy::new(fee_rule, None, T::SHIELDED_PROTOCOL), + DustOutputPolicy::default(), + ); + + let txid = st + .spend( + &input_selector, + &usk, + req, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + ) + .unwrap()[0]; + + let amount_left = (value - (amount_sent + fee_rule.fixed_fee()).unwrap()).unwrap(); + let pending_change = (amount_left - amount_legacy_change).unwrap(); + + // The "legacy change" is not counted by get_pending_change(). + assert_eq!(st.get_pending_change(account, 1), pending_change); + // We spent the only note so we only have pending change. + assert_eq!(st.get_total_balance(account), pending_change); + + let (h, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(h, 1); + + assert_eq!(st.get_total_balance(account2), amount_sent,); + assert_eq!(st.get_total_balance(account), amount_left); + + st.reset(); + + // Account creation and DFVK derivation should be deterministic. + let (_, restored_usk) = st + .wallet_mut() + .create_account(&seed, birthday.clone()) + .unwrap(); + assert!(T::fvks_equal( + &T::sk_to_fvk(T::usk_to_sk(&restored_usk)), + &dfvk, + )); + + let (_, restored_usk2) = st.wallet_mut().create_account(&seed, birthday).unwrap(); + assert!(T::fvks_equal( + &T::sk_to_fvk(T::usk_to_sk(&restored_usk2)), + &dfvk2, + )); + + st.scan_cached_blocks(st.sapling_activation_height(), 2); + + assert_eq!(st.get_total_balance(account2), amount_sent,); + assert_eq!(st.get_total_balance(account), amount_left); +} + +pub(crate) fn zip317_spend() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Add funds to the wallet + let (h1, _, _) = st.generate_next_block( + &dfvk, + AddressType::Internal, + NonNegativeAmount::const_from_u64(50000), + ); + + // Add 10 dust notes to the wallet + for _ in 1..=10 { + st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(1000), + ); + } + + st.scan_cached_blocks(h1, 11); + + // Spendable balance matches total balance + let total = NonNegativeAmount::const_from_u64(60000); + assert_eq!(st.get_total_balance(account), total); + assert_eq!(st.get_spendable_balance(account, 1), total); + + let input_selector = input_selector(StandardFeeRule::Zip317, None, T::SHIELDED_PROTOCOL); + + // This first request will fail due to insufficient non-dust funds + let req = TransactionRequest::new(vec![Payment { + recipient_address: T::fvk_default_address(&dfvk), + amount: NonNegativeAmount::const_from_u64(50000), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + assert_matches!( + st.spend( + &input_selector, + &usk, + req, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + ), + Err(Error::InsufficientFunds { available, required }) + if available == NonNegativeAmount::const_from_u64(51000) + && required == NonNegativeAmount::const_from_u64(60000) + ); + + // This request will succeed, spending a single dust input to pay the 10000 + // ZAT fee in addition to the 41000 ZAT output to the recipient + let req = TransactionRequest::new(vec![Payment { + recipient_address: T::fvk_default_address(&dfvk), + amount: NonNegativeAmount::const_from_u64(41000), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + let txid = st + .spend( + &input_selector, + &usk, + req, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + ) + .unwrap()[0]; + + let (h, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(h, 1); + + // TODO: send to an account so that we can check its balance. + // We sent back to the same account so the amount_sent should be included + // in the total balance. + assert_eq!( + st.get_total_balance(account), + (total - NonNegativeAmount::const_from_u64(10000)).unwrap() + ); +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn shield_transparent() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account_id, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + let uaddr = st + .wallet() + .get_current_address(account_id) + .unwrap() + .unwrap(); + let taddr = uaddr.transparent().unwrap(); + + // Ensure that the wallet has at least one block + let (h, _, _) = st.generate_next_block( + &dfvk, + AddressType::Internal, + NonNegativeAmount::const_from_u64(50000), + ); + st.scan_cached_blocks(h, 1); + + let utxo = WalletTransparentOutput::from_parts( + OutPoint::new([1u8; 32], 1), + TxOut { + value: NonNegativeAmount::const_from_u64(10000), + script_pubkey: taddr.script(), + }, + h, + ) + .unwrap(); + + let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); + assert!(matches!(res0, Ok(_))); + + // TODO: This test was originally written to use the pre-zip-313 fee rule + // and has not yet been updated. + #[allow(deprecated)] + let fee_rule = StandardFeeRule::PreZip313; + + let input_selector = GreedyInputSelector::new( + standard::SingleOutputChangeStrategy::new(fee_rule, None, T::SHIELDED_PROTOCOL), + DustOutputPolicy::default(), + ); + + assert_matches!( + st.shield_transparent_funds( + &input_selector, + NonNegativeAmount::from_u64(10000).unwrap(), + &usk, + &[*taddr], + 1 + ), + Ok(_) + ); +} + +// FIXME: This requires fixes to the test framework. +#[allow(dead_code)] +pub(crate) fn birthday_in_anchor_shard() { + // Use a non-zero birthday offset because Sapling and NU5 are activated at the same height. + let (mut st, dfvk, birthday, _) = test_with_nu5_birthday_offset::(76); + + // Set up the following situation: + // + // |<------ 500 ------->|<--- 10 --->|<--- 10 --->| + // last_shard_start wallet_birthday received_tx anchor_height + // + // Set up some shard root history before the wallet birthday. + let prev_shard_start = birthday.height() - 500; + T::put_subtree_roots( + &mut st, + 0, + &[CommitmentTreeRoot::from_parts( + prev_shard_start, + // fake a hash, the value doesn't matter + T::empty_tree_leaf(), + )], + ) + .unwrap(); + + let received_tx_height = birthday.height() + 10; + + let initial_sapling_tree_size = birthday + .sapling_frontier() + .value() + .map(|f| u64::from(f.position() + 1)) + .unwrap_or(0) + .try_into() + .unwrap(); + #[cfg(feature = "orchard")] + let initial_orchard_tree_size = birthday + .orchard_frontier() + .value() + .map(|f| u64::from(f.position() + 1)) + .unwrap_or(0) + .try_into() + .unwrap(); + #[cfg(not(feature = "orchard"))] + let initial_orchard_tree_size = 0; + + // Generate 9 blocks that have no value for us, starting at the birthday height. + let not_our_key = T::sk_to_fvk(&T::sk(&[0xf5; 32])); + let not_our_value = NonNegativeAmount::const_from_u64(10000); + st.generate_block_at( + birthday.height(), + BlockHash([0; 32]), + ¬_our_key, + AddressType::DefaultExternal, + not_our_value, + initial_sapling_tree_size, + initial_orchard_tree_size, + ); + for _ in 1..9 { + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + } + + // Now, generate a block that belongs to our wallet + st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(500000), + ); + + // Generate some more blocks to get above our anchor height + for _ in 0..15 { + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + } + + // Scan a block range that includes our received note, but skips some blocks we need to + // make it spendable. + st.scan_cached_blocks(birthday.height() + 5, 20); + + // Verify that the received note is not considered spendable + let account = st.test_account().unwrap(); + let spendable = T::select_spendable_notes( + &st, + account.0, + NonNegativeAmount::const_from_u64(300000), + received_tx_height + 10, + &[], + ) + .unwrap(); + + assert_eq!(spendable.len(), 0); + + // Scan the blocks we skipped + st.scan_cached_blocks(birthday.height(), 5); + + // Verify that the received note is now considered spendable + let spendable = T::select_spendable_notes( + &st, + account.0, + NonNegativeAmount::const_from_u64(300000), + received_tx_height + 10, + &[], + ) + .unwrap(); + + assert_eq!(spendable.len(), 1); +} + +pub(crate) fn checkpoint_gaps() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let (account, usk, birthday) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Generate a block with funds belonging to our wallet. + st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(500000), + ); + st.scan_cached_blocks(birthday.height(), 1); + + // Create a gap of 10 blocks having no shielded outputs, then add a block that doesn't + // belong to us so that we can get a checkpoint in the tree. + let not_our_key = T::sk_to_fvk(&T::sk(&[0xf5; 32])); + let not_our_value = NonNegativeAmount::const_from_u64(10000); + st.generate_block_at( + birthday.height() + 10, + BlockHash([0; 32]), + ¬_our_key, + AddressType::DefaultExternal, + not_our_value, + st.latest_cached_block().unwrap().sapling_end_size, + st.latest_cached_block().unwrap().orchard_end_size, + ); + + // Scan the block + st.scan_cached_blocks(birthday.height() + 10, 1); + + // Fake that everything has been scanned + st.wallet() + .conn + .execute_batch("UPDATE scan_queue SET priority = 10") + .unwrap(); + + // Verify that our note is considered spendable + let spendable = T::select_spendable_notes( + &st, + account, + NonNegativeAmount::const_from_u64(300000), + birthday.height() + 5, + &[], + ) + .unwrap(); + assert_eq!(spendable.len(), 1); + + // Attempt to spend the note with 5 confirmations + let to = T::fvk_default_address(¬_our_key); + assert_matches!( + st.create_spend_to_address( + &usk, + &to, + NonNegativeAmount::const_from_u64(10000), + None, + OvkPolicy::Sender, + NonZeroU32::new(5).unwrap(), + None, + T::SHIELDED_PROTOCOL, + ), + Ok(_) + ); +} + +#[cfg(feature = "orchard")] +pub(crate) fn cross_pool_exchange() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(|params| AccountBirthday::from_activation(params, NetworkUpgrade::Nu5)) + .build(); + + let (account, usk, birthday) = st.test_account().unwrap(); + + let p0_fvk = P0::test_account_fvk(&st); + + let p1_fvk = P1::test_account_fvk(&st); + let p1_to = P1::fvk_default_address(&p1_fvk); + + let note_value = NonNegativeAmount::const_from_u64(300000); + st.generate_next_block(&p0_fvk, AddressType::DefaultExternal, note_value); + st.generate_next_block(&p1_fvk, AddressType::DefaultExternal, note_value); + st.scan_cached_blocks(birthday.height(), 2); + + let initial_balance = (note_value * 2).unwrap(); + assert_eq!(st.get_total_balance(account), initial_balance); + assert_eq!(st.get_spendable_balance(account, 1), initial_balance); + + let transfer_amount = NonNegativeAmount::const_from_u64(200000); + let p0_to_p1 = zip321::TransactionRequest::new(vec![Payment { + recipient_address: p1_to, + amount: transfer_amount, + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + let fee_rule = StandardFeeRule::Zip317; + let input_selector = GreedyInputSelector::new( + standard::SingleOutputChangeStrategy::new(fee_rule, None, P1::SHIELDED_PROTOCOL), + DustOutputPolicy::default(), + ); + let proposal0 = st + .propose_transfer( + account, + &input_selector, + p0_to_p1, + NonZeroU32::new(1).unwrap(), + ) + .unwrap(); + + let _min_target_height = proposal0.min_target_height(); + assert_eq!(proposal0.steps().len(), 1); + let step0 = &proposal0.steps().head; + + // We expect 4 logical actions, two per pool (due to padding). + let expected_fee = NonNegativeAmount::const_from_u64(20000); + assert_eq!(step0.balance().fee_required(), expected_fee); + + let expected_change = (note_value - transfer_amount - expected_fee).unwrap(); + let proposed_change = step0.balance().proposed_change(); + assert_eq!(proposed_change.len(), 1); + let change_output = proposed_change.get(0).unwrap(); + // Since this is a cross-pool transfer, change will be sent to the preferred pool. + assert_eq!( + change_output.output_pool(), + std::cmp::max(ShieldedProtocol::Sapling, ShieldedProtocol::Orchard) + ); + assert_eq!(change_output.value(), expected_change); + + let create_proposed_result = + st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal0); + assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 1); + + let (h, _) = st.generate_next_block_including(create_proposed_result.unwrap()[0]); + st.scan_cached_blocks(h, 1); + + assert_eq!( + st.get_total_balance(account), + (initial_balance - expected_fee).unwrap() + ); + assert_eq!( + st.get_spendable_balance(account, 1), + (initial_balance - expected_fee).unwrap() + ); +} + +pub(crate) fn valid_chain_states() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let dfvk = T::test_account_fvk(&st); + + // Empty chain should return None + assert_matches!(st.wallet().chain_height(), Ok(None)); + + // Create a fake CompactBlock sending value to the address + let (h1, _, _) = st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(5), + ); + + // Scan the cache + st.scan_cached_blocks(h1, 1); + + // Create a second fake CompactBlock sending more value to the address + let (h2, _, _) = st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(7), + ); + + // Scanning should detect no inconsistencies + st.scan_cached_blocks(h2, 1); +} + +// FIXME: This requires fixes to the test framework. +#[allow(dead_code)] +pub(crate) fn invalid_chain_cache_disconnected() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let dfvk = T::test_account_fvk(&st); + + // Create some fake CompactBlocks + let (h, _, _) = st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(5), + ); + let (last_contiguous_height, _, _) = st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(7), + ); + + // Scanning the cache should find no inconsistencies + st.scan_cached_blocks(h, 2); + + // Create more fake CompactBlocks that don't connect to the scanned ones + let disconnect_height = last_contiguous_height + 1; + st.generate_block_at( + disconnect_height, + BlockHash([1; 32]), + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(8), + 2, + 2, + ); + st.generate_next_block( + &dfvk, + AddressType::DefaultExternal, + NonNegativeAmount::const_from_u64(3), + ); + + // Data+cache chain should be invalid at the data/cache boundary + assert_matches!( + st.try_scan_cached_blocks( + disconnect_height, + 2 + ), + Err(chain::error::Error::Scan(ScanError::PrevHashMismatch { at_height })) + if at_height == disconnect_height + ); +} + +pub(crate) fn data_db_truncation() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let account = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); + + // Create fake CompactBlocks sending value to the address + let value = NonNegativeAmount::const_from_u64(5); + let value2 = NonNegativeAmount::const_from_u64(7); + let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); + + // Scan the cache + st.scan_cached_blocks(h, 2); + + // Account balance should reflect both received notes + assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); + + // "Rewind" to height of last scanned block + st.wallet_mut() + .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h + 1)) + .unwrap(); + + // Account balance should be unaltered + assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); + + // Rewind so that one block is dropped + st.wallet_mut() + .transactionally(|wdb| truncate_to_height(wdb.conn.0, &wdb.params, h)) + .unwrap(); + + // Account balance should only contain the first received note + assert_eq!(st.get_total_balance(account.0), value); + + // Scan the cache again + st.scan_cached_blocks(h, 2); + + // Account balance should again reflect both received notes + assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); +} + +pub(crate) fn scan_cached_blocks_allows_blocks_out_of_order() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let account = st.test_account().unwrap(); + let (_, usk, _) = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + let value = NonNegativeAmount::const_from_u64(50000); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + st.scan_cached_blocks(h1, 1); + assert_eq!(st.get_total_balance(account.0), value); + + // Create blocks to reach height + 2 + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + let (h3, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + + // Scan the later block first + st.scan_cached_blocks(h3, 1); + + // Now scan the block of height height + 1 + st.scan_cached_blocks(h2, 1); + assert_eq!( + st.get_total_balance(account.0), + NonNegativeAmount::const_from_u64(150_000) + ); + + // We can spend the received notes + let req = TransactionRequest::new(vec![Payment { + recipient_address: T::fvk_default_address(&dfvk), + amount: NonNegativeAmount::const_from_u64(110_000), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + + #[allow(deprecated)] + let input_selector = GreedyInputSelector::new( + standard::SingleOutputChangeStrategy::new( + StandardFeeRule::Zip317, + None, + T::SHIELDED_PROTOCOL, + ), + DustOutputPolicy::default(), + ); + assert_matches!( + st.spend( + &input_selector, + &usk, + req, + OvkPolicy::Sender, + NonZeroU32::new(1).unwrap(), + ), + Ok(_) + ); +} + +pub(crate) fn scan_cached_blocks_finds_received_notes() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let account = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); + + // Create a fake CompactBlock sending value to the address + let value = NonNegativeAmount::const_from_u64(5); + let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + + // Scan the cache + let summary = st.scan_cached_blocks(h1, 1); + assert_eq!(summary.scanned_range().start, h1); + assert_eq!(summary.scanned_range().end, h1 + 1); + assert_eq!(T::received_note_count(&summary), 1); + + // Account balance should reflect the received note + assert_eq!(st.get_total_balance(account.0), value); + + // Create a second fake CompactBlock sending more value to the address + let value2 = NonNegativeAmount::const_from_u64(7); + let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value2); + + // Scan the cache again + let summary = st.scan_cached_blocks(h2, 1); + assert_eq!(summary.scanned_range().start, h2); + assert_eq!(summary.scanned_range().end, h2 + 1); + assert_eq!(T::received_note_count(&summary), 1); + + // Account balance should reflect both received notes + assert_eq!(st.get_total_balance(account.0), (value + value2).unwrap()); +} + +// TODO: This test can probably be entirely removed, as the following test duplicates it entirely. +pub(crate) fn scan_cached_blocks_finds_change_notes() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let account = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); + + // Create a fake CompactBlock sending value to the address + let value = NonNegativeAmount::const_from_u64(5); + let (received_height, _, nf) = + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + + // Scan the cache + st.scan_cached_blocks(received_height, 1); + + // Account balance should reflect the received note + assert_eq!(st.get_total_balance(account.0), value); + + // Create a second fake CompactBlock spending value from the address + let not_our_key = T::sk_to_fvk(&T::sk(&[0xf5; 32])); + let to2 = T::fvk_default_address(¬_our_key); + let value2 = NonNegativeAmount::const_from_u64(2); + let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + + // Scan the cache again + st.scan_cached_blocks(spent_height, 1); + + // Account balance should equal the change + assert_eq!(st.get_total_balance(account.0), (value - value2).unwrap()); +} + +pub(crate) fn scan_cached_blocks_detects_spends_out_of_order() { + let mut st = TestBuilder::new() + .with_block_cache() + .with_test_account(AccountBirthday::from_sapling_activation) + .build(); + + let account = st.test_account().unwrap(); + let dfvk = T::test_account_fvk(&st); + + // Wallet summary is not yet available + assert_eq!(st.get_wallet_summary(0), None); + + // Create a fake CompactBlock sending value to the address + let value = NonNegativeAmount::const_from_u64(5); + let (received_height, _, nf) = + st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + + // Create a second fake CompactBlock spending value from the address + let not_our_key = T::sk_to_fvk(&T::sk(&[0xf5; 32])); + let to2 = T::fvk_default_address(¬_our_key); + let value2 = NonNegativeAmount::const_from_u64(2); + let (spent_height, _) = st.generate_next_block_spending(&dfvk, (nf, value), to2, value2); + + // Scan the spending block first. + st.scan_cached_blocks(spent_height, 1); + + // Account balance should equal the change + assert_eq!(st.get_total_balance(account.0), (value - value2).unwrap()); + + // Now scan the block in which we received the note that was spent. + st.scan_cached_blocks(received_height, 1); + + // Account balance should be the same. + assert_eq!(st.get_total_balance(account.0), (value - value2).unwrap()); +} diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 7d2f8b793e..c52a47b39d 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -112,6 +112,9 @@ use crate::{ use self::scanning::{parse_priority_code, priority_code, replace_queue_entries}; +#[cfg(feature = "orchard")] +use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; + #[cfg(feature = "transparent-inputs")] use { crate::UtxoId, @@ -128,7 +131,10 @@ use { }; pub mod commitment_tree; +pub(crate) mod common; pub mod init; +#[cfg(feature = "orchard")] +pub(crate) mod orchard; pub(crate) mod sapling; pub(crate) mod scanning; @@ -218,7 +224,7 @@ impl Account { /// Returns the default Unified Address for the account, /// along with the diversifier index that generated it. /// - /// The diversifier index may be non-zero if the Unified Address includes a Sapling + /// The diversifier index may be non-zero if the Unified Address includes a Sapling /// receiver, and there was no valid Sapling receiver at diversifier index zero. pub fn default_address( &self, @@ -291,11 +297,11 @@ struct AccountSqlValues<'a> { account_type: u32, hd_seed_fingerprint: Option<&'a [u8]>, hd_account_index: Option, - ufvk: Option, + ufvk: Option<&'a UnifiedFullViewingKey>, uivk: String, } -/// Returns (account_type, hd_seed_fingerprint, hd_account_index, ufvk, uivk) for a given account. +/// Returns (account_type, hd_seed_fingerprint, hd_account_index, ufvk, uivk) for a given account. fn get_sql_values_for_account_parameters<'a, P: consensus::Parameters>( account: &'a Account, params: &P, @@ -305,7 +311,7 @@ fn get_sql_values_for_account_parameters<'a, P: consensus::Parameters>( account_type: AccountType::Zip32.into(), hd_seed_fingerprint: Some(hdaccount.hd_seed_fingerprint().as_bytes()), hd_account_index: Some(hdaccount.account_index().into()), - ufvk: Some(hdaccount.ufvk().encode(params)), + ufvk: Some(hdaccount.ufvk()), uivk: hdaccount .ufvk() .to_unified_incoming_viewing_key() @@ -317,7 +323,7 @@ fn get_sql_values_for_account_parameters<'a, P: consensus::Parameters>( account_type: AccountType::Imported.into(), hd_seed_fingerprint: None, hd_account_index: None, - ufvk: Some(ufvk.encode(params)), + ufvk: Some(ufvk), uivk: ufvk .to_unified_incoming_viewing_key() .map_err(|e| SqliteClientError::CorruptedData(e.to_string()))? @@ -341,27 +347,55 @@ pub(crate) fn add_account( birthday: AccountBirthday, ) -> Result { let args = get_sql_values_for_account_parameters(&account, params)?; - let account_id: AccountId = conn.query_row(r#" - INSERT INTO accounts (account_type, hd_seed_fingerprint, hd_account_index, ufvk, uivk, birthday_height, recover_until_height) - VALUES (:account_type, :hd_seed_fingerprint, :hd_account_index, :ufvk, :uivk, :birthday_height, :recover_until_height) + + let orchard_item = args + .ufvk + .and_then(|ufvk| ufvk.orchard().map(|k| k.to_bytes())); + let sapling_item = args + .ufvk + .and_then(|ufvk| ufvk.sapling().map(|k| k.to_bytes())); + #[cfg(feature = "transparent-inputs")] + let transparent_item = args + .ufvk + .and_then(|ufvk| ufvk.transparent().map(|k| k.serialize())); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_item: Option> = None; + + let account_id: AccountId = conn.query_row( + r#" + INSERT INTO accounts ( + account_type, hd_seed_fingerprint, hd_account_index, + ufvk, uivk, + orchard_fvk_item_cache, sapling_fvk_item_cache, p2pkh_fvk_item_cache, + birthday_height, recover_until_height + ) + VALUES ( + :account_type, :hd_seed_fingerprint, :hd_account_index, + :ufvk, :uivk, + :orchard_fvk_item_cache, :sapling_fvk_item_cache, :p2pkh_fvk_item_cache, + :birthday_height, :recover_until_height + ) RETURNING id; "#, named_params![ ":account_type": args.account_type, ":hd_seed_fingerprint": args.hd_seed_fingerprint, ":hd_account_index": args.hd_account_index, - ":ufvk": args.ufvk, + ":ufvk": args.ufvk.map(|ufvk| ufvk.encode(params)), ":uivk": args.uivk, + ":orchard_fvk_item_cache": orchard_item, + ":sapling_fvk_item_cache": sapling_item, + ":p2pkh_fvk_item_cache": transparent_item, ":birthday_height": u32::from(birthday.height()), ":recover_until_height": birthday.recover_until().map(u32::from) ], - |row| Ok(AccountId(row.get(0)?)) + |row| Ok(AccountId(row.get(0)?)), )?; // If a birthday frontier is available, insert it into the note commitment tree. If the // birthday frontier is the empty frontier, we don't need to do anything. if let Some(frontier) = birthday.sapling_frontier().value() { - debug!("Inserting frontier into ShardTree: {:?}", frontier); + debug!("Inserting Sapling frontier into ShardTree: {:?}", frontier); let shard_store = SqliteShardStore::<_, ::sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( conn, @@ -386,6 +420,34 @@ pub(crate) fn add_account( )?; } + #[cfg(feature = "orchard")] + if let Some(frontier) = birthday.orchard_frontier().value() { + debug!("Inserting Orchard frontier into ShardTree: {:?}", frontier); + let shard_store = SqliteShardStore::< + _, + ::orchard::tree::MerkleHashOrchard, + ORCHARD_SHARD_HEIGHT, + >::from_connection(conn, ORCHARD_TABLES_PREFIX)?; + let mut shard_tree: ShardTree< + _, + { ::orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 }, + ORCHARD_SHARD_HEIGHT, + > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); + shard_tree.insert_frontier_nodes( + frontier.clone(), + Retention::Checkpoint { + // This subtraction is safe, because all leaves in the tree appear in blocks, and + // the invariant that birthday.height() always corresponds to the block for which + // `frontier` is the tree state at the start of the block. Together, this means + // there exists a prior block for which frontier is the tree state at the end of + // the block. + id: birthday.height() - 1, + is_marked: false, + }, + )?; + } + + // The ignored range always starts at Sapling activation let sapling_activation_height = params .activation_height(NetworkUpgrade::Sapling) .expect("Sapling activation height must be available."); @@ -655,17 +717,92 @@ pub(crate) fn get_account_for_ufvk( conn: &rusqlite::Connection, params: &P, ufvk: &UnifiedFullViewingKey, -) -> Result, SqliteClientError> { - conn.query_row( - "SELECT id FROM accounts WHERE ufvk = ?", - [&ufvk.encode(params)], +) -> Result)>, SqliteClientError> { + #[cfg(feature = "transparent-inputs")] + let transparent_item = ufvk.transparent().map(|k| k.serialize()); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_item: Option> = None; + + let mut stmt = conn.prepare( + "SELECT id, ufvk + FROM accounts + WHERE orchard_fvk_item_cache = :orchard_fvk_item_cache + OR sapling_fvk_item_cache = :sapling_fvk_item_cache + OR p2pkh_fvk_item_cache = :p2pkh_fvk_item_cache", + )?; + + let accounts = stmt + .query_and_then::<_, SqliteClientError, _, _>( + named_params![ + ":orchard_fvk_item_cache": ufvk.orchard().map(|k| k.to_bytes()), + ":sapling_fvk_item_cache": ufvk.sapling().map(|k| k.to_bytes()), + ":p2pkh_fvk_item_cache": transparent_item, + ], + |row| { + let account_id = row.get::<_, u32>(0).map(AccountId)?; + Ok(( + account_id, + row.get::<_, Option>(1)? + .map(|ufvk_str| UnifiedFullViewingKey::decode(params, &ufvk_str)) + .transpose() + .map_err(|e| { + SqliteClientError::CorruptedData(format!( + "Could not decode unified full viewing key for account {:?}: {}", + account_id, e + )) + })?, + )) + }, + )? + .collect::, _>>()?; + + if accounts.len() > 1 { + Err(SqliteClientError::CorruptedData( + "Mutiple account records matched the provided UFVK".to_owned(), + )) + } else { + Ok(accounts.into_iter().next()) + } +} + +/// Returns the account id corresponding to a given [`HdSeedFingerprint`] +/// and [`zip32::AccountId`], if any. +pub(crate) fn get_seed_account( + conn: &rusqlite::Connection, + params: &P, + seed: &HdSeedFingerprint, + account_id: zip32::AccountId, +) -> Result)>, SqliteClientError> { + let mut stmt = conn.prepare( + "SELECT id, ufvk + FROM accounts + WHERE hd_seed_fingerprint = :hd_seed_fingerprint + AND hd_account_index = :account_id", + )?; + + let mut accounts = stmt.query_and_then::<_, SqliteClientError, _, _>( + named_params![ + ":hd_seed_fingerprint": seed.as_bytes(), + ":hd_account_index": u32::from(account_id), + ], |row| { - let acct = row.get(0)?; - Ok(AccountId(acct)) + let account_id = row.get::<_, u32>(0).map(AccountId)?; + Ok(( + account_id, + row.get::<_, Option>(1)? + .map(|ufvk_str| UnifiedFullViewingKey::decode(params, &ufvk_str)) + .transpose() + .map_err(|e| { + SqliteClientError::CorruptedData(format!( + "Could not decode unified full viewing key for account {:?}: {}", + account_id, e + )) + })?, + )) }, - ) - .optional() - .map_err(SqliteClientError::from) + )?; + + accounts.next().transpose() } pub(crate) trait ScanProgress { @@ -676,6 +813,15 @@ pub(crate) trait ScanProgress { fully_scanned_height: BlockHeight, chain_tip_height: BlockHeight, ) -> Result>, SqliteClientError>; + + #[cfg(feature = "orchard")] + fn orchard_scan_progress( + &self, + conn: &rusqlite::Connection, + birthday_height: BlockHeight, + fully_scanned_height: BlockHeight, + chain_tip_height: BlockHeight, + ) -> Result>, SqliteClientError>; } #[derive(Debug)] @@ -757,6 +903,83 @@ impl ScanProgress for SubtreeScanProgress { .flatten()) } } + + #[cfg(feature = "orchard")] + #[tracing::instrument(skip(conn))] + fn orchard_scan_progress( + &self, + conn: &rusqlite::Connection, + birthday_height: BlockHeight, + fully_scanned_height: BlockHeight, + chain_tip_height: BlockHeight, + ) -> Result>, SqliteClientError> { + if fully_scanned_height == chain_tip_height { + // Compute the total blocks scanned since the wallet birthday + conn.query_row( + "SELECT SUM(orchard_action_count) + FROM blocks + WHERE height >= :birthday_height", + named_params![":birthday_height": u32::from(birthday_height)], + |row| { + let scanned = row.get::<_, Option>(0)?; + Ok(scanned.map(|n| Ratio::new(n, n))) + }, + ) + .map_err(SqliteClientError::from) + } else { + let start_height = birthday_height; + // Compute the starting number of notes directly from the blocks table + let start_size = conn.query_row( + "SELECT MAX(orchard_commitment_tree_size) + FROM blocks + WHERE height <= :start_height", + named_params![":start_height": u32::from(start_height)], + |row| row.get::<_, Option>(0), + )?; + + // Compute the total blocks scanned so far above the starting height + let scanned_count = conn.query_row( + "SELECT SUM(orchard_action_count) + FROM blocks + WHERE height > :start_height", + named_params![":start_height": u32::from(start_height)], + |row| row.get::<_, Option>(0), + )?; + + // We don't have complete information on how many actions will exist in the shard at + // the chain tip without having scanned the chain tip block, so we overestimate by + // computing the maximum possible number of notes directly from the shard indices. + // + // TODO: it would be nice to be able to reliably have the size of the commitment tree + // at the chain tip without having to have scanned that block. + Ok(conn + .query_row( + "SELECT MIN(shard_index), MAX(shard_index) + FROM orchard_tree_shards + WHERE subtree_end_height > :start_height + OR subtree_end_height IS NULL", + named_params![":start_height": u32::from(start_height)], + |row| { + let min_tree_size = row + .get::<_, Option>(0)? + .map(|min| min << ORCHARD_SHARD_HEIGHT); + let max_idx = row.get::<_, Option>(1)?; + Ok(start_size + .or(min_tree_size) + .zip(max_idx) + .map(|(min_tree_size, max)| { + let max_tree_size = (max + 1) << ORCHARD_SHARD_HEIGHT; + Ratio::new( + scanned_count.unwrap_or(0), + max_tree_size - min_tree_size, + ) + })) + }, + ) + .optional()? + .flatten()) + } + } } /// Returns the spendable balance for the account at the specified height. @@ -794,27 +1017,27 @@ pub(crate) fn get_wallet_summary( chain_tip_height, )?; - // If the shard containing the summary height contains any unscanned ranges that start below or - // including that height, none of our balance is currently spendable. - #[tracing::instrument(skip_all)] - fn is_any_spendable( - conn: &rusqlite::Connection, - summary_height: BlockHeight, - ) -> Result { - conn.query_row( - "SELECT NOT EXISTS( - SELECT 1 FROM v_sapling_shard_unscanned_ranges - WHERE :summary_height - BETWEEN subtree_start_height - AND IFNULL(subtree_end_height, :summary_height) - AND block_range_start <= :summary_height - )", - named_params![":summary_height": u32::from(summary_height)], - |row| row.get::<_, bool>(0), - ) - .map_err(|e| e.into()) - } - let any_spendable = is_any_spendable(tx, summary_height)?; + #[cfg(feature = "orchard")] + let orchard_scan_progress = progress.orchard_scan_progress( + tx, + birthday_height, + fully_scanned_height, + chain_tip_height, + )?; + #[cfg(not(feature = "orchard"))] + let orchard_scan_progress: Option> = None; + + // Treat Sapling and Orchard outputs as having the same cost to scan. + let scan_progress = sapling_scan_progress + .zip(orchard_scan_progress) + .map(|(s, o)| { + Ratio::new( + s.numerator() + o.numerator(), + s.denominator() + o.denominator(), + ) + }) + .or(sapling_scan_progress) + .or(orchard_scan_progress); let mut stmt_accounts = tx.prepare_cached("SELECT id FROM accounts")?; let mut account_balances = stmt_accounts @@ -824,78 +1047,159 @@ pub(crate) fn get_wallet_summary( }) .collect::, _>>()?; - let sapling_trace = tracing::info_span!("stmt_select_notes").entered(); - let mut stmt_select_notes = tx.prepare_cached( - "SELECT n.account_id, n.value, n.is_change, scan_state.max_priority, t.block - FROM sapling_received_notes n - JOIN transactions t ON t.id_tx = n.tx - LEFT OUTER JOIN v_sapling_shards_scan_state scan_state - ON n.commitment_tree_position >= scan_state.start_position - AND n.commitment_tree_position < scan_state.end_position_exclusive - WHERE n.spent IS NULL - AND ( - t.expiry_height IS NULL - OR t.block IS NOT NULL - OR t.expiry_height >= :summary_height - )", - )?; + fn count_notes( + tx: &rusqlite::Transaction, + summary_height: BlockHeight, + account_balances: &mut HashMap, + table_prefix: &'static str, + with_pool_balance: F, + ) -> Result<(), SqliteClientError> + where + F: Fn( + &mut AccountBalance, + NonNegativeAmount, + NonNegativeAmount, + NonNegativeAmount, + ) -> Result<(), SqliteClientError>, + { + // If the shard containing the summary height contains any unscanned ranges that start below or + // including that height, none of our balance is currently spendable. + #[tracing::instrument(skip_all)] + fn is_any_spendable( + conn: &rusqlite::Connection, + summary_height: BlockHeight, + table_prefix: &'static str, + ) -> Result { + conn.query_row( + &format!( + "SELECT NOT EXISTS( + SELECT 1 FROM v_{table_prefix}_shard_unscanned_ranges + WHERE :summary_height + BETWEEN subtree_start_height + AND IFNULL(subtree_end_height, :summary_height) + AND block_range_start <= :summary_height + )" + ), + named_params![":summary_height": u32::from(summary_height)], + |row| row.get::<_, bool>(0), + ) + .map_err(|e| e.into()) + } - let mut rows = - stmt_select_notes.query(named_params![":summary_height": u32::from(summary_height)])?; - while let Some(row) = rows.next()? { - let account = AccountId(row.get::<_, u32>(0)?); + let any_spendable = is_any_spendable(tx, summary_height, table_prefix)?; + let mut stmt_select_notes = tx.prepare_cached(&format!( + "SELECT n.account_id, n.value, n.is_change, scan_state.max_priority, t.block + FROM {table_prefix}_received_notes n + JOIN transactions t ON t.id_tx = n.tx + LEFT OUTER JOIN v_{table_prefix}_shards_scan_state scan_state + ON n.commitment_tree_position >= scan_state.start_position + AND n.commitment_tree_position < scan_state.end_position_exclusive + WHERE n.spent IS NULL + AND ( + t.expiry_height IS NULL + OR t.block IS NOT NULL + OR t.expiry_height >= :summary_height + )", + ))?; - let value_raw = row.get::<_, i64>(1)?; - let value = NonNegativeAmount::from_nonnegative_i64(value_raw).map_err(|_| { - SqliteClientError::CorruptedData(format!("Negative received note value: {}", value_raw)) - })?; + let mut rows = + stmt_select_notes.query(named_params![":summary_height": u32::from(summary_height)])?; + while let Some(row) = rows.next()? { + let account = AccountId(row.get::<_, u32>(0)?); - let is_change = row.get::<_, bool>(2)?; - - // If `max_priority` is null, this means that the note is not positioned; the note - // will not be spendable, so we assign the scan priority to `ChainTip` as a priority - // that is greater than `Scanned` - let max_priority_raw = row.get::<_, Option>(3)?; - let max_priority = max_priority_raw.map_or_else( - || Ok(ScanPriority::ChainTip), - |raw| { - parse_priority_code(raw).ok_or_else(|| { - SqliteClientError::CorruptedData(format!( - "Priority code {} not recognized.", - raw - )) - }) - }, - )?; + let value_raw = row.get::<_, i64>(1)?; + let value = NonNegativeAmount::from_nonnegative_i64(value_raw).map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Negative received note value: {}", + value_raw + )) + })?; - let received_height = row.get::<_, Option>(4)?.map(BlockHeight::from); + let is_change = row.get::<_, bool>(2)?; + + // If `max_priority` is null, this means that the note is not positioned; the note + // will not be spendable, so we assign the scan priority to `ChainTip` as a priority + // that is greater than `Scanned` + let max_priority_raw = row.get::<_, Option>(3)?; + let max_priority = max_priority_raw.map_or_else( + || Ok(ScanPriority::ChainTip), + |raw| { + parse_priority_code(raw).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "Priority code {} not recognized.", + raw + )) + }) + }, + )?; - let is_spendable = any_spendable - && received_height.iter().any(|h| h <= &summary_height) - && max_priority <= ScanPriority::Scanned; + let received_height = row.get::<_, Option>(4)?.map(BlockHeight::from); - let is_pending_change = is_change && received_height.iter().all(|h| h > &summary_height); + let is_spendable = any_spendable + && received_height.iter().any(|h| h <= &summary_height) + && max_priority <= ScanPriority::Scanned; - let (spendable_value, change_pending_confirmation, value_pending_spendability) = { - let zero = NonNegativeAmount::ZERO; - if is_spendable { - (value, zero, zero) - } else if is_pending_change { - (zero, value, zero) - } else { - (zero, zero, value) + let is_pending_change = + is_change && received_height.iter().all(|h| h > &summary_height); + + let (spendable_value, change_pending_confirmation, value_pending_spendability) = { + let zero = NonNegativeAmount::ZERO; + if is_spendable { + (value, zero, zero) + } else if is_pending_change { + (zero, value, zero) + } else { + (zero, zero, value) + } + }; + + if let Some(balances) = account_balances.get_mut(&account) { + with_pool_balance( + balances, + spendable_value, + change_pending_confirmation, + value_pending_spendability, + )?; } - }; + } + Ok(()) + } - if let Some(balances) = account_balances.get_mut(&account) { + #[cfg(feature = "orchard")] + { + let orchard_trace = tracing::info_span!("orchard_balances").entered(); + count_notes( + tx, + summary_height, + &mut account_balances, + ORCHARD_TABLES_PREFIX, + |balances, spendable_value, change_pending_confirmation, value_pending_spendability| { + balances.with_orchard_balance_mut::<_, SqliteClientError>(|bal| { + bal.add_spendable_value(spendable_value)?; + bal.add_pending_change_value(change_pending_confirmation)?; + bal.add_pending_spendable_value(value_pending_spendability)?; + Ok(()) + }) + }, + )?; + drop(orchard_trace); + } + + let sapling_trace = tracing::info_span!("sapling_balances").entered(); + count_notes( + tx, + summary_height, + &mut account_balances, + SAPLING_TABLES_PREFIX, + |balances, spendable_value, change_pending_confirmation, value_pending_spendability| { balances.with_sapling_balance_mut::<_, SqliteClientError>(|bal| { bal.add_spendable_value(spendable_value)?; bal.add_pending_change_value(change_pending_confirmation)?; bal.add_pending_spendable_value(value_pending_spendability)?; Ok(()) - })?; - } - } + }) + }, + )?; drop(sapling_trace); #[cfg(feature = "transparent-inputs")] @@ -932,6 +1236,9 @@ pub(crate) fn get_wallet_summary( drop(transparent_trace); } + // The approach used here for Sapling and Orchard subtree indexing was a quick hack + // that has not yet been replaced. TODO: Make less hacky. + // https://github.com/zcash/librustzcash/issues/1249 let next_sapling_subtree_index = { let shard_store = SqliteShardStore::<_, ::sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( @@ -951,12 +1258,34 @@ pub(crate) fn get_wallet_summary( .unwrap_or(0) }; + #[cfg(feature = "orchard")] + let next_orchard_subtree_index = { + let shard_store = SqliteShardStore::< + _, + ::orchard::tree::MerkleHashOrchard, + ORCHARD_SHARD_HEIGHT, + >::from_connection(tx, ORCHARD_TABLES_PREFIX)?; + + // The last shard will be incomplete, and we want the next range to overlap with + // the last complete shard, so return the index of the second-to-last shard root. + shard_store + .get_shard_roots() + .map_err(ShardTreeError::Storage)? + .iter() + .rev() + .nth(1) + .map(|addr| addr.index()) + .unwrap_or(0) + }; + let summary = WalletSummary::new( account_balances, chain_tip_height, fully_scanned_height, - sapling_scan_progress, + scan_progress, next_sapling_subtree_index, + #[cfg(feature = "orchard")] + next_orchard_subtree_index, ); Ok(Some(summary)) @@ -967,24 +1296,31 @@ pub(crate) fn get_received_memo( conn: &rusqlite::Connection, note_id: NoteId, ) -> Result, SqliteClientError> { - let memo_bytes: Option> = match note_id.protocol() { - ShieldedProtocol::Sapling => conn - .query_row( - "SELECT memo FROM sapling_received_notes - JOIN transactions ON sapling_received_notes.tx = transactions.id_tx + let fetch_memo = |table_prefix: &'static str, output_col: &'static str| { + conn.query_row( + &format!( + "SELECT memo FROM {table_prefix}_received_notes + JOIN transactions ON {table_prefix}_received_notes.tx = transactions.id_tx WHERE transactions.txid = :txid - AND sapling_received_notes.output_index = :output_index", - named_params![ - ":txid": note_id.txid().as_ref(), - ":output_index": note_id.output_index() - ], - |row| row.get(0), - ) - .optional()? - .flatten(), - _ => { + AND {table_prefix}_received_notes.{output_col} = :output_index" + ), + named_params![ + ":txid": note_id.txid().as_ref(), + ":output_index": note_id.output_index() + ], + |row| row.get(0), + ) + .optional() + }; + + let memo_bytes: Option> = match note_id.protocol() { + ShieldedProtocol::Sapling => fetch_memo(SAPLING_TABLES_PREFIX, "output_index")?.flatten(), + #[cfg(feature = "orchard")] + ShieldedProtocol::Orchard => fetch_memo(ORCHARD_TABLES_PREFIX, "action_index")?.flatten(), + #[cfg(not(feature = "orchard"))] + ShieldedProtocol::Orchard => { return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded( - note_id.protocol(), + ShieldedProtocol::Orchard, ))) } }; @@ -1251,7 +1587,24 @@ pub(crate) fn get_target_and_anchor_heights( min_confirmations, )?; - Ok(sapling_anchor_height.map(|h| (chain_tip_height + 1, h))) + #[cfg(feature = "orchard")] + let orchard_anchor_height = get_max_checkpointed_height( + conn, + ORCHARD_TABLES_PREFIX, + chain_tip_height, + min_confirmations, + )?; + + #[cfg(not(feature = "orchard"))] + let orchard_anchor_height: Option = None; + + let anchor_height = sapling_anchor_height + .zip(orchard_anchor_height) + .map(|(s, o)| std::cmp::min(s, o)) + .or(sapling_anchor_height) + .or(orchard_anchor_height); + + Ok(anchor_height.map(|h| (chain_tip_height + 1, h))) } None => Ok(None), } @@ -1471,7 +1824,7 @@ pub(crate) fn get_max_height_hash( pub(crate) fn get_min_unspent_height( conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { - conn.query_row( + let min_sapling: Option = conn.query_row( "SELECT MIN(tx.block) FROM sapling_received_notes n JOIN transactions tx ON tx.id_tx = n.tx @@ -1481,8 +1834,27 @@ pub(crate) fn get_min_unspent_height( row.get(0) .map(|maybe_height: Option| maybe_height.map(|height| height.into())) }, - ) - .map_err(SqliteClientError::from) + )?; + #[cfg(feature = "orchard")] + let min_orchard: Option = conn.query_row( + "SELECT MIN(tx.block) + FROM orchard_received_notes n + JOIN transactions tx ON tx.id_tx = n.tx + WHERE n.spent IS NULL", + [], + |row| { + row.get(0) + .map(|maybe_height: Option| maybe_height.map(|height| height.into())) + }, + )?; + #[cfg(not(feature = "orchard"))] + let min_orchard = None; + + Ok(min_sapling + .zip(min_orchard) + .map(|(s, o)| s.min(o)) + .or(min_sapling) + .or(min_orchard)) } /// Truncates the database to the given height. @@ -1524,6 +1896,10 @@ pub(crate) fn truncate_to_height( wdb.with_sapling_tree_mut(|tree| { tree.truncate_removing_checkpoint(&block_height).map(|_| ()) })?; + #[cfg(feature = "orchard")] + wdb.with_orchard_tree_mut(|tree| { + tree.truncate_removing_checkpoint(&block_height).map(|_| ()) + })?; // Rewind received notes conn.execute( @@ -1537,6 +1913,18 @@ pub(crate) fn truncate_to_height( );", [u32::from(block_height)], )?; + #[cfg(feature = "orchard")] + conn.execute( + "DELETE FROM orchard_received_notes + WHERE id IN ( + SELECT rn.id + FROM orchard_received_notes rn + LEFT OUTER JOIN transactions tx + ON tx.id_tx = rn.tx + WHERE tx.block IS NOT NULL AND tx.block > ? + );", + [u32::from(block_height)], + )?; // Do not delete sent notes; this can contain data that is not recoverable // from the chain. Wallets must continue to operate correctly in the @@ -1764,6 +2152,7 @@ pub(crate) fn get_account_ids( } /// Inserts information about a scanned block into the database. +#[allow(clippy::too_many_arguments)] pub(crate) fn put_block( conn: &rusqlite::Transaction<'_>, block_height: BlockHeight, @@ -1771,6 +2160,8 @@ pub(crate) fn put_block( block_time: u32, sapling_commitment_tree_size: u32, sapling_output_count: u32, + #[cfg(feature = "orchard")] orchard_commitment_tree_size: u32, + #[cfg(feature = "orchard")] orchard_action_count: u32, ) -> Result<(), SqliteClientError> { let block_hash_data = conn .query_row( @@ -1801,7 +2192,9 @@ pub(crate) fn put_block( time, sapling_commitment_tree_size, sapling_output_count, - sapling_tree + sapling_tree, + orchard_commitment_tree_size, + orchard_action_count ) VALUES ( :height, @@ -1809,21 +2202,32 @@ pub(crate) fn put_block( :block_time, :sapling_commitment_tree_size, :sapling_output_count, - x'00' + x'00', + :orchard_commitment_tree_size, + :orchard_action_count ) ON CONFLICT (height) DO UPDATE SET hash = :hash, time = :block_time, sapling_commitment_tree_size = :sapling_commitment_tree_size, - sapling_output_count = :sapling_output_count", + sapling_output_count = :sapling_output_count, + orchard_commitment_tree_size = :orchard_commitment_tree_size, + orchard_action_count = :orchard_action_count", )?; + #[cfg(not(feature = "orchard"))] + let orchard_commitment_tree_size: Option = None; + #[cfg(not(feature = "orchard"))] + let orchard_action_count: Option = None; + stmt_upsert_block.execute(named_params![ ":height": u32::from(block_height), ":hash": &block_hash.0[..], ":block_time": block_time, ":sapling_commitment_tree_size": sapling_commitment_tree_size, ":sapling_output_count": sapling_output_count, + ":orchard_commitment_tree_size": orchard_commitment_tree_size, + ":orchard_action_count": orchard_action_count, ])?; Ok(()) @@ -2008,13 +2412,20 @@ pub(crate) fn update_expired_notes( conn: &rusqlite::Connection, expiry_height: BlockHeight, ) -> Result<(), SqliteClientError> { - let mut stmt_update_expired = conn.prepare_cached( + let mut stmt_update_sapling_expired = conn.prepare_cached( "UPDATE sapling_received_notes SET spent = NULL WHERE EXISTS ( SELECT id_tx FROM transactions WHERE id_tx = sapling_received_notes.spent AND block IS NULL AND expiry_height < ? )", )?; - stmt_update_expired.execute([u32::from(expiry_height)])?; + stmt_update_sapling_expired.execute([u32::from(expiry_height)])?; + let mut stmt_update_orchard_expired = conn.prepare_cached( + "UPDATE orchard_received_notes SET spent = NULL WHERE EXISTS ( + SELECT id_tx FROM transactions + WHERE id_tx = orchard_received_notes.spent AND block IS NULL AND expiry_height < ? + )", + )?; + stmt_update_orchard_expired.execute([u32::from(expiry_height)])?; Ok(()) } @@ -2625,15 +3036,22 @@ mod tests { // Scan a block above the wallet's birthday height. let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); let not_our_value = NonNegativeAmount::const_from_u64(10000); - let end_height = st.sapling_activation_height() + 2; + let start_height = st.sapling_activation_height(); let _ = st.generate_block_at( - end_height, - BlockHash([37; 32]), + start_height, + BlockHash([0; 32]), ¬_our_key, AddressType::DefaultExternal, not_our_value, - 17, + 0, + 0, ); + let (mid_height, _, _) = + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + let (end_height, _, _) = + st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); + + // Scan the last block first st.scan_cached_blocks(end_height, 1); // The wallet should still have no fully-scanned block, as no scanned block range @@ -2641,24 +3059,13 @@ mod tests { assert_eq!(block_fully_scanned(&st), None); // Scan the block at the wallet's birthday height. - let start_height = st.sapling_activation_height(); - let _ = st.generate_block_at( - start_height, - BlockHash([0; 32]), - ¬_our_key, - AddressType::DefaultExternal, - not_our_value, - 0, - ); st.scan_cached_blocks(start_height, 1); // The fully-scanned height should now be that of the scanned block. assert_eq!(block_fully_scanned(&st), Some(start_height)); // Scan the block in between the two previous blocks. - let (h, _, _) = - st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); - st.scan_cached_blocks(h, 1); + st.scan_cached_blocks(mid_height, 1); // The fully-scanned height should now be the latest block, as the two disjoint // ranges have been connected. diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index ec2b22fc3e..8dde47a535 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -1091,57 +1091,113 @@ mod tests { use zcash_primitives::consensus::{BlockHeight, Network}; use super::SqliteShardStore; - use crate::{wallet::init::init_wallet_db, WalletDb, SAPLING_TABLES_PREFIX}; + use crate::{ + testing::pool::ShieldedPoolTester, + wallet::{init::init_wallet_db, sapling::tests::SaplingPoolTester}, + WalletDb, + }; - fn new_tree(m: usize) -> ShardTree, 4, 3> { + fn new_tree( + m: usize, + ) -> ShardTree, 4, 3> { let data_file = NamedTempFile::new().unwrap(); let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); let store = - SqliteShardStore::<_, String, 3>::from_connection(db_data.conn, SAPLING_TABLES_PREFIX) + SqliteShardStore::<_, String, 3>::from_connection(db_data.conn, T::TABLES_PREFIX) .unwrap(); ShardTree::new(store, m) } + #[cfg(feature = "orchard")] + mod orchard { + use super::new_tree; + use crate::wallet::orchard::tests::OrchardPoolTester; + + #[test] + fn append() { + super::check_append(new_tree::); + } + + #[test] + fn root_hashes() { + super::check_root_hashes(new_tree::); + } + + #[test] + fn witnesses() { + super::check_witnesses(new_tree::); + } + + #[test] + fn witness_consistency() { + super::check_witness_consistency(new_tree::); + } + + #[test] + fn checkpoint_rewind() { + super::check_checkpoint_rewind(new_tree::); + } + + #[test] + fn remove_mark() { + super::check_remove_mark(new_tree::); + } + + #[test] + fn rewind_remove_mark() { + super::check_rewind_remove_mark(new_tree::); + } + + #[test] + fn put_shard_roots() { + super::put_shard_roots::() + } + } + #[test] - fn append() { - check_append(new_tree); + fn sapling_append() { + check_append(new_tree::); } #[test] - fn root_hashes() { - check_root_hashes(new_tree); + fn sapling_root_hashes() { + check_root_hashes(new_tree::); } #[test] - fn witnesses() { - check_witnesses(new_tree); + fn sapling_witnesses() { + check_witnesses(new_tree::); } #[test] - fn witness_consistency() { - check_witness_consistency(new_tree); + fn sapling_witness_consistency() { + check_witness_consistency(new_tree::); } #[test] - fn checkpoint_rewind() { - check_checkpoint_rewind(new_tree); + fn sapling_checkpoint_rewind() { + check_checkpoint_rewind(new_tree::); } #[test] - fn remove_mark() { - check_remove_mark(new_tree); + fn sapling_remove_mark() { + check_remove_mark(new_tree::); } #[test] - fn rewind_remove_mark() { - check_rewind_remove_mark(new_tree); + fn sapling_rewind_remove_mark() { + check_rewind_remove_mark(new_tree::); } #[test] - fn put_shard_roots() { + fn sapling_put_shard_roots() { + put_shard_roots::() + } + + fn put_shard_roots() { let data_file = NamedTempFile::new().unwrap(); let mut db_data = WalletDb::for_path(data_file.path(), Network::TestNetwork).unwrap(); data_file.keep().unwrap(); @@ -1149,7 +1205,7 @@ mod tests { init_wallet_db(&mut db_data, None).unwrap(); let tx = db_data.conn.transaction().unwrap(); let store = - SqliteShardStore::<_, String, 3>::from_connection(&tx, SAPLING_TABLES_PREFIX).unwrap(); + SqliteShardStore::<_, String, 3>::from_connection(&tx, T::TABLES_PREFIX).unwrap(); // introduce some roots let roots = (0u32..4) @@ -1165,7 +1221,7 @@ mod tests { ) }) .collect::>(); - super::put_shard_roots::<_, 6, 3>(store.conn, SAPLING_TABLES_PREFIX, 0, &roots).unwrap(); + super::put_shard_roots::<_, 6, 3>(store.conn, T::TABLES_PREFIX, 0, &roots).unwrap(); // simulate discovery of a note let mut tree = ShardTree::<_, 6, 3>::new(store, 10); diff --git a/zcash_client_sqlite/src/wallet/common.rs b/zcash_client_sqlite/src/wallet/common.rs new file mode 100644 index 0000000000..1a47fc404a --- /dev/null +++ b/zcash_client_sqlite/src/wallet/common.rs @@ -0,0 +1,216 @@ +//! Functions common to Sapling and Orchard support in the wallet. + +use rusqlite::{named_params, types::Value, Connection, Row}; +use std::rc::Rc; + +use zcash_client_backend::{ + wallet::{Note, ReceivedNote}, + ShieldedProtocol, +}; +use zcash_primitives::transaction::{components::amount::NonNegativeAmount, TxId}; +use zcash_protocol::consensus::{self, BlockHeight}; + +use super::wallet_birthday; +use crate::{error::SqliteClientError, AccountId, ReceivedNoteId, SAPLING_TABLES_PREFIX}; + +#[cfg(feature = "orchard")] +use crate::ORCHARD_TABLES_PREFIX; + +fn per_protocol_names(protocol: ShieldedProtocol) -> (&'static str, &'static str, &'static str) { + match protocol { + ShieldedProtocol::Sapling => (SAPLING_TABLES_PREFIX, "output_index", "rcm"), + #[cfg(feature = "orchard")] + ShieldedProtocol::Orchard => (ORCHARD_TABLES_PREFIX, "action_index", "rho, rseed"), + #[cfg(not(feature = "orchard"))] + ShieldedProtocol::Orchard => { + unreachable!("Should never be called unless the `orchard` feature is enabled") + } + } +} + +fn unscanned_tip_exists( + conn: &Connection, + anchor_height: BlockHeight, + table_prefix: &'static str, +) -> Result { + // v_sapling_shard_unscanned_ranges only returns ranges ending on or after wallet birthday, so + // we don't need to refer to the birthday in this query. + conn.query_row( + &format!( + "SELECT EXISTS ( + SELECT 1 FROM v_{table_prefix}_shard_unscanned_ranges range + WHERE range.block_range_start <= :anchor_height + AND :anchor_height BETWEEN + range.subtree_start_height + AND IFNULL(range.subtree_end_height, :anchor_height) + )" + ), + named_params![":anchor_height": u32::from(anchor_height),], + |row| row.get::<_, bool>(0), + ) +} + +// The `clippy::let_and_return` lint is explicitly allowed here because a bug in Clippy +// (https://github.com/rust-lang/rust-clippy/issues/11308) means it fails to identify that the `result` temporary +// is required in order to resolve the borrows involved in the `query_and_then` call. +#[allow(clippy::let_and_return)] +pub(crate) fn get_spendable_note( + conn: &Connection, + params: &P, + txid: &TxId, + index: u32, + protocol: ShieldedProtocol, + to_spendable_note: F, +) -> Result>, SqliteClientError> +where + F: Fn(&P, &Row) -> Result>, SqliteClientError>, +{ + let (table_prefix, index_col, note_reconstruction_cols) = per_protocol_names(protocol); + let result = conn.query_row_and_then( + &format!( + "SELECT {table_prefix}_received_notes.id, txid, {index_col}, + diversifier, value, {note_reconstruction_cols}, commitment_tree_position, + accounts.ufvk, recipient_key_scope + FROM {table_prefix}_received_notes + INNER JOIN accounts ON accounts.id = {table_prefix}_received_notes.account_id + INNER JOIN transactions ON transactions.id_tx = {table_prefix}_received_notes.tx + WHERE txid = :txid + AND {index_col} = :output_index + AND accounts.ufvk IS NOT NULL + AND recipient_key_scope IS NOT NULL + AND nf IS NOT NULL + AND commitment_tree_position IS NOT NULL + AND spent IS NULL" + ), + named_params![ + ":txid": txid.as_ref(), + ":output_index": index, + ], + |row| to_spendable_note(params, row), + ); + + // `OptionalExtension` doesn't work here because the error type of `Result` is already + // `SqliteClientError` + match result { + Ok(r) => Ok(r), + Err(SqliteClientError::DbError(rusqlite::Error::QueryReturnedNoRows)) => Ok(None), + Err(e) => Err(e), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn select_spendable_notes( + conn: &Connection, + params: &P, + account: AccountId, + target_value: NonNegativeAmount, + anchor_height: BlockHeight, + exclude: &[ReceivedNoteId], + protocol: ShieldedProtocol, + to_spendable_note: F, +) -> Result>, SqliteClientError> +where + F: Fn(&P, &Row) -> Result>, SqliteClientError>, +{ + let birthday_height = match wallet_birthday(conn)? { + Some(birthday) => birthday, + None => { + // the wallet birthday can only be unknown if there are no accounts in the wallet; in + // such a case, the wallet has no notes to spend. + return Ok(vec![]); + } + }; + + let (table_prefix, index_col, note_reconstruction_cols) = per_protocol_names(protocol); + if unscanned_tip_exists(conn, anchor_height, table_prefix)? { + return Ok(vec![]); + } + + // The goal of this SQL statement is to select the oldest notes until the required + // value has been reached. + // 1) Use a window function to create a view of all notes, ordered from oldest to + // newest, with an additional column containing a running sum: + // - Unspent notes accumulate the values of all unspent notes in that note's + // account, up to itself. + // - Spent notes accumulate the values of all notes in the transaction they were + // spent in, up to itself. + // + // 2) Select all unspent notes in the desired account, along with their running sum. + // + // 3) Select all notes for which the running sum was less than the required value, as + // well as a single note for which the sum was greater than or equal to the + // required value, bringing the sum of all selected notes across the threshold. + let mut stmt_select_notes = conn.prepare_cached( + &format!( + "WITH eligible AS ( + SELECT + {table_prefix}_received_notes.id AS id, txid, {index_col}, + diversifier, value, {note_reconstruction_cols}, commitment_tree_position, + SUM(value) OVER ( + PARTITION BY {table_prefix}_received_notes.account_id, spent + ORDER BY {table_prefix}_received_notes.id + ) AS so_far, + accounts.ufvk as ufvk, recipient_key_scope + FROM {table_prefix}_received_notes + INNER JOIN accounts + ON accounts.id = {table_prefix}_received_notes.account_id + INNER JOIN transactions + ON transactions.id_tx = {table_prefix}_received_notes.tx + WHERE {table_prefix}_received_notes.account_id = :account + AND accounts.ufvk IS NOT NULL + AND recipient_key_scope IS NOT NULL + AND nf IS NOT NULL + AND commitment_tree_position IS NOT NULL + AND spent IS NULL + AND transactions.block <= :anchor_height + AND {table_prefix}_received_notes.id NOT IN rarray(:exclude) + AND NOT EXISTS ( + SELECT 1 FROM v_{table_prefix}_shard_unscanned_ranges unscanned + -- select all the unscanned ranges involving the shard containing this note + WHERE {table_prefix}_received_notes.commitment_tree_position >= unscanned.start_position + AND {table_prefix}_received_notes.commitment_tree_position < unscanned.end_position_exclusive + -- exclude unscanned ranges that start above the anchor height (they don't affect spendability) + AND unscanned.block_range_start <= :anchor_height + -- exclude unscanned ranges that end below the wallet birthday + AND unscanned.block_range_end > :wallet_birthday + ) + ) + SELECT id, txid, {index_col}, + diversifier, value, {note_reconstruction_cols}, commitment_tree_position, + ufvk, recipient_key_scope + FROM eligible WHERE so_far < :target_value + UNION + SELECT id, txid, {index_col}, + diversifier, value, {note_reconstruction_cols}, commitment_tree_position, + ufvk, recipient_key_scope + FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", + ) + )?; + + let excluded: Vec = exclude + .iter() + .filter_map(|ReceivedNoteId(p, n)| { + if *p == protocol { + Some(Value::from(*n)) + } else { + None + } + }) + .collect(); + let excluded_ptr = Rc::new(excluded); + + let notes = stmt_select_notes.query_and_then( + named_params![ + ":account": account.0, + ":anchor_height": &u32::from(anchor_height), + ":target_value": &u64::from(target_value), + ":exclude": &excluded_ptr, + ":wallet_birthday": u32::from(birthday_height) + ], + |r| to_spendable_note(params, r), + )?; + + notes + .filter_map(|r| r.transpose()) + .collect::>() +} diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 364e2f23a2..496cb4e0fa 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -11,9 +11,8 @@ use uuid::Uuid; use zcash_client_backend::keys::AddressGenerationError; use zcash_primitives::{consensus, transaction::components::amount::BalanceError}; -use crate::WalletDb; - use super::commitment_tree; +use crate::WalletDb; mod migrations; @@ -230,6 +229,9 @@ mod tests { hd_account_index INTEGER, ufvk TEXT, uivk TEXT NOT NULL, + orchard_fvk_item_cache BLOB, + sapling_fvk_item_cache BLOB, + p2pkh_fvk_item_cache BLOB, birthday_height INTEGER NOT NULL, recover_until_height INTEGER, CHECK ( (account_type = 0 AND hd_seed_fingerprint IS NOT NULL AND hd_account_index IS NOT NULL AND ufvk IS NOT NULL) OR (account_type = 1 AND hd_seed_fingerprint IS NULL AND hd_account_index IS NULL) ) @@ -263,6 +265,51 @@ mod tests { ON UPDATE RESTRICT, CONSTRAINT nf_uniq UNIQUE (spend_pool, nf) )", + "CREATE TABLE orchard_received_notes ( + id INTEGER PRIMARY KEY, + tx INTEGER NOT NULL, + action_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rho BLOB NOT NULL, + rseed BLOB NOT NULL, + nf BLOB UNIQUE, + is_change INTEGER NOT NULL, + memo BLOB, + spent INTEGER, + commitment_tree_position INTEGER, + recipient_key_scope INTEGER, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (spent) REFERENCES transactions(id_tx), + CONSTRAINT tx_output UNIQUE (tx, action_index) + )", + "CREATE TABLE orchard_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, + cap_data BLOB NOT NULL + )", + "CREATE TABLE orchard_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES orchard_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE, + CONSTRAINT spend_position_unique UNIQUE (checkpoint_id, mark_removed_position) + )", + "CREATE TABLE orchard_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + )", + "CREATE TABLE orchard_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER, + CONSTRAINT root_unique UNIQUE (root_hash) + )", r#"CREATE TABLE "sapling_received_notes" ( id INTEGER PRIMARY KEY, tx INTEGER NOT NULL, @@ -276,7 +323,7 @@ mod tests { memo BLOB, spent INTEGER, commitment_tree_position INTEGER, - recipient_key_scope INTEGER NOT NULL DEFAULT 0, + recipient_key_scope INTEGER, FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (spent) REFERENCES transactions(id_tx), @@ -389,7 +436,152 @@ mod tests { expected_idx += 1; } + let expected_indices = vec![ + r#"CREATE UNIQUE INDEX accounts_ufvk ON "accounts" (ufvk)"#, + r#"CREATE UNIQUE INDEX accounts_uivk ON "accounts" (uivk)"#, + r#"CREATE UNIQUE INDEX hd_account ON "accounts" (hd_seed_fingerprint, hd_account_index)"#, + r#"CREATE INDEX "addresses_accounts" ON "addresses" ( + "account_id" ASC + )"#, + r#"CREATE INDEX nf_map_locator_idx ON nullifier_map(block_height, tx_index)"#, + r#"CREATE INDEX orchard_received_notes_account ON orchard_received_notes ( + account_id ASC + )"#, + r#"CREATE INDEX orchard_received_notes_spent ON orchard_received_notes ( + spent ASC + )"#, + r#"CREATE INDEX orchard_received_notes_tx ON orchard_received_notes ( + tx ASC + )"#, + r#"CREATE INDEX "sapling_received_notes_account" ON "sapling_received_notes" ( + "account_id" ASC + )"#, + r#"CREATE INDEX "sapling_received_notes_spent" ON "sapling_received_notes" ( + "spent" ASC + )"#, + r#"CREATE INDEX "sapling_received_notes_tx" ON "sapling_received_notes" ( + "tx" ASC + )"#, + r#"CREATE INDEX sent_notes_from_account ON "sent_notes" (from_account_id)"#, + r#"CREATE INDEX sent_notes_to_account ON "sent_notes" (to_account_id)"#, + r#"CREATE INDEX sent_notes_tx ON "sent_notes" (tx)"#, + r#"CREATE INDEX utxos_received_by_account ON "utxos" (received_by_account_id)"#, + r#"CREATE INDEX utxos_spent_in_tx ON "utxos" (spent_in_tx)"#, + ]; + let mut indices_query = st + .wallet() + .conn + .prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND sql != '' ORDER BY tbl_name, name") + .unwrap(); + let mut rows = indices_query.query([]).unwrap(); + let mut expected_idx = 0; + while let Some(row) = rows.next().unwrap() { + let sql: String = row.get(0).unwrap(); + assert_eq!( + re.replace_all(&sql, " "), + re.replace_all(expected_indices[expected_idx], " ") + ); + expected_idx += 1; + } + let expected_views = vec![ + // v_orchard_shard_scan_ranges + format!( + "CREATE VIEW v_orchard_shard_scan_ranges AS + SELECT + shard.shard_index, + shard.shard_index << 16 AS start_position, + (shard.shard_index + 1) << 16 AS end_position_exclusive, + IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, + shard.subtree_end_height, + shard.contains_marked, + scan_queue.block_range_start, + scan_queue.block_range_end, + scan_queue.priority + FROM orchard_tree_shards shard + LEFT OUTER JOIN orchard_tree_shards prev_shard + ON shard.shard_index = prev_shard.shard_index + 1 + -- Join with scan ranges that overlap with the subtree's involved blocks. + INNER JOIN scan_queue ON ( + subtree_start_height < scan_queue.block_range_end AND + ( + scan_queue.block_range_start <= shard.subtree_end_height OR + shard.subtree_end_height IS NULL + ) + )", + u32::from(st.network().activation_height(NetworkUpgrade::Nu5).unwrap()), + ), + //v_orchard_shard_unscanned_ranges + format!( + "CREATE VIEW v_orchard_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + block_range_start, + block_range_end, + priority + FROM v_orchard_shard_scan_ranges + INNER JOIN wallet_birthday + WHERE priority > {} + AND block_range_end > wallet_birthday.height", + priority_code(&ScanPriority::Scanned), + ), + // v_orchard_shards_scan_state + "CREATE VIEW v_orchard_shards_scan_state AS + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + MAX(priority) AS max_priority + FROM v_orchard_shard_scan_ranges + GROUP BY + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked".to_owned(), + // v_received_notes + "CREATE VIEW v_received_notes AS + SELECT + id, + tx, + 2 AS pool, + sapling_received_notes.output_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + id, + tx, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index)".to_owned(), // v_sapling_shard_scan_ranges format!( "CREATE VIEW v_sapling_shard_scan_ranges AS @@ -456,162 +648,159 @@ mod tests { contains_marked".to_owned(), // v_transactions "CREATE VIEW v_transactions AS - WITH - notes AS ( - SELECT sapling_received_notes.id AS id, - sapling_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 2 AS pool, - sapling_received_notes.value AS value, - CASE - WHEN sapling_received_notes.is_change THEN 1 - ELSE 0 - END AS is_change, - CASE - WHEN sapling_received_notes.is_change THEN 0 - ELSE 1 - END AS received_count, - CASE - WHEN (sapling_received_notes.memo IS NULL OR sapling_received_notes.memo = X'F6') - THEN 0 - ELSE 1 - END AS memo_present - FROM sapling_received_notes + WITH + notes AS ( + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + v_received_notes.value AS value, + CASE + WHEN v_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN v_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (v_received_notes.memo IS NULL OR v_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + 0 AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + -v_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.spent + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + 0 AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR v_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid".to_owned(), + // v_tx_outputs + "CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + v_received_notes.pool AS output_pool, + v_received_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + NULL AS to_address, + v_received_notes.value AS value, + v_received_notes.is_change AS is_change, + v_received_notes.memo AS memo + FROM v_received_notes JOIN transactions - ON transactions.id_tx = sapling_received_notes.tx + ON transactions.id_tx = v_received_notes.tx + LEFT JOIN sent_notes + ON sent_notes.id = v_received_notes.sent_note_id UNION - SELECT utxos.id AS id, - utxos.received_by_account_id AS account_id, - utxos.height AS block, - utxos.prevout_txid AS txid, - 0 AS pool, - utxos.value_zat AS value, - 0 AS is_change, - 1 AS received_count, - 0 AS memo_present + SELECT utxos.prevout_txid AS txid, + 0 AS output_pool, + utxos.prevout_idx AS output_index, + NULL AS from_account_id, + utxos.received_by_account_id AS to_account_id, + utxos.address AS to_address, + utxos.value_zat AS value, + 0 AS is_change, + NULL AS memo FROM utxos UNION - SELECT sapling_received_notes.id AS id, - sapling_received_notes.account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 2 AS pool, - -sapling_received_notes.value AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM sapling_received_notes - JOIN transactions - ON transactions.id_tx = sapling_received_notes.spent - UNION - SELECT utxos.id AS id, - utxos.received_by_account_id AS account_id, - transactions.block AS block, - transactions.txid AS txid, - 0 AS pool, - -utxos.value_zat AS value, - 0 AS is_change, - 0 AS received_count, - 0 AS memo_present - FROM utxos - JOIN transactions - ON transactions.id_tx = utxos.spent_in_tx - ), - sent_note_counts AS ( - SELECT sent_notes.from_account_id AS account_id, - transactions.txid AS txid, - COUNT(DISTINCT sent_notes.id) as sent_notes, - SUM( - CASE - WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR sapling_received_notes.tx IS NOT NULL) - THEN 0 - ELSE 1 - END - ) AS memo_count + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo FROM sent_notes JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN sapling_received_notes - ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE COALESCE(sapling_received_notes.is_change, 0) = 0 - GROUP BY account_id, txid - ), - blocks_max_height AS ( - SELECT MAX(blocks.height) as max_height FROM blocks - ) - SELECT notes.account_id AS account_id, - notes.block AS mined_height, - notes.txid AS txid, - transactions.tx_index AS tx_index, - transactions.expiry_height AS expiry_height, - transactions.raw AS raw, - SUM(notes.value) AS account_balance_delta, - transactions.fee AS fee_paid, - SUM(notes.is_change) > 0 AS has_change, - MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, - SUM(notes.received_count) AS received_note_count, - SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, - blocks.time AS block_time, - ( - blocks.height IS NULL - AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height - ) AS expired_unmined - FROM notes - LEFT JOIN transactions - ON notes.txid = transactions.txid - JOIN blocks_max_height - LEFT JOIN blocks ON blocks.height = notes.block - LEFT JOIN sent_note_counts - ON sent_note_counts.account_id = notes.account_id - AND sent_note_counts.txid = notes.txid - GROUP BY notes.account_id, notes.txid".to_owned(), - // v_tx_outputs - "CREATE VIEW v_tx_outputs AS - SELECT transactions.txid AS txid, - 2 AS output_pool, - sapling_received_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - sapling_received_notes.account_id AS to_account_id, - NULL AS to_address, - sapling_received_notes.value AS value, - sapling_received_notes.is_change AS is_change, - sapling_received_notes.memo AS memo - FROM sapling_received_notes - JOIN transactions - ON transactions.id_tx = sapling_received_notes.tx - LEFT JOIN sent_notes - ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sent_notes.output_index) - UNION - SELECT utxos.prevout_txid AS txid, - 0 AS output_pool, - utxos.prevout_idx AS output_index, - NULL AS from_account_id, - utxos.received_by_account_id AS to_account_id, - utxos.address AS to_address, - utxos.value_zat AS value, - 0 AS is_change, - NULL AS memo - FROM utxos - UNION - SELECT transactions.txid AS txid, - sent_notes.output_pool AS output_pool, - sent_notes.output_index AS output_index, - sent_notes.from_account_id AS from_account_id, - sapling_received_notes.account_id AS to_account_id, - sent_notes.to_address AS to_address, - sent_notes.value AS value, - 0 AS is_change, - sent_notes.memo AS memo - FROM sent_notes - JOIN transactions - ON transactions.id_tx = sent_notes.tx - LEFT JOIN sapling_received_notes - ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = - (sapling_received_notes.tx, 2, sapling_received_notes.output_index) - WHERE COALESCE(sapling_received_notes.is_change, 0) = 0".to_owned(), + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0".to_owned(), ]; let mut views_query = st diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index b9ad7561f5..044065082a 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -5,6 +5,8 @@ mod addresses_table; mod full_account_ids; mod initial_setup; mod nullifier_map; +mod orchard_received_notes; +mod orchard_shardtree; mod received_notes_nullable_nf; mod receiving_key_scopes; mod sapling_memo_consistency; @@ -24,7 +26,7 @@ use std::rc::Rc; use schemer_rusqlite::RusqliteMigration; use secrecy::SecretVec; -use zcash_primitives::consensus; +use zcash_protocol::consensus; use super::WalletMigrationError; @@ -45,20 +47,22 @@ pub(super) fn all_migrations( // | // v_transactions_net // | - // received_notes_nullable_nf - // / | \ - // / | \ - // shardtree_support sapling_memo_consistency nullifier_map - // / \ \ - // add_account_birthdays receiving_key_scopes v_transactions_transparent_history - // | \ | | - // v_sapling_shard_unscanned_ranges \ | v_tx_outputs_use_legacy_false - // | \ | | - // wallet_summaries \ | v_transactions_shielding_balance - // \ | | - // \ | v_transactions_note_uniqueness - // \ | / - // full_account_ids + // received_notes_nullable_nf------ + // / | \ + // / | \ + // --------------- shardtree_support sapling_memo_consistency nullifier_map + // / / \ \ + // orchard_shardtree add_account_birthdays receiving_key_scopes v_transactions_transparent_history + // | \ | | + // v_sapling_shard_unscanned_ranges \ | v_tx_outputs_use_legacy_false + // | \ | | + // wallet_summaries \ | v_transactions_shielding_balance + // \ | | + // \ | v_transactions_note_uniqueness + // \ | / + // full_account_ids + // | + // orchard_received_notes vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -101,5 +105,9 @@ pub(super) fn all_migrations( seed, params: params.clone(), }), + Box::new(orchard_shardtree::Migration { + params: params.clone(), + }), + Box::new(orchard_received_notes::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/full_account_ids.rs b/zcash_client_sqlite/src/wallet/init/migrations/full_account_ids.rs index ec1913b23c..db47ecfd79 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/full_account_ids.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/full_account_ids.rs @@ -59,6 +59,9 @@ impl RusqliteMigration for Migration

{ hd_account_index INTEGER, ufvk TEXT, uivk TEXT NOT NULL, + orchard_fvk_item_cache BLOB, + sapling_fvk_item_cache BLOB, + p2pkh_fvk_item_cache BLOB, birthday_height INTEGER NOT NULL, recover_until_height INTEGER, CHECK ( @@ -67,8 +70,9 @@ impl RusqliteMigration for Migration

{ (account_type = {account_type_imported} AND hd_seed_fingerprint IS NULL AND hd_account_index IS NULL) ) ); - CREATE UNIQUE INDEX accounts_uivk ON accounts_new ("uivk"); - CREATE UNIQUE INDEX accounts_ufvk ON accounts_new ("ufvk"); + CREATE UNIQUE INDEX hd_account ON accounts_new (hd_seed_fingerprint, hd_account_index); + CREATE UNIQUE INDEX accounts_uivk ON accounts_new (uivk); + CREATE UNIQUE INDEX accounts_ufvk ON accounts_new (ufvk); "#), )?; @@ -120,19 +124,40 @@ impl RusqliteMigration for Migration

{ .to_uivk() .encode(&self.params.network_type()); - transaction.execute(r#" - INSERT INTO accounts_new (id, account_type, hd_seed_fingerprint, hd_account_index, ufvk, uivk, birthday_height, recover_until_height) - VALUES (:account_id, :account_type, :seed_id, :account_index, :ufvk, :uivk, :birthday_height, :recover_until_height); - "#, named_params![ - ":account_id": account_id, - ":account_type": account_type, - ":seed_id": seed_id.as_bytes(), - ":account_index": account_index, - ":ufvk": ufvk, - ":uivk": uivk, - ":birthday_height": birthday_height, - ":recover_until_height": recover_until_height, - ])?; + #[cfg(feature = "transparent-inputs")] + let transparent_item = ufvk_parsed.transparent().map(|k| k.serialize()); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_item: Option> = None; + + transaction.execute( + r#" + INSERT INTO accounts_new ( + id, account_type, hd_seed_fingerprint, hd_account_index, + ufvk, uivk, + orchard_fvk_item_cache, sapling_fvk_item_cache, p2pkh_fvk_item_cache, + birthday_height, recover_until_height + ) + VALUES ( + :account_id, :account_type, :seed_id, :account_index, + :ufvk, :uivk, + :orchard_fvk_item_cache, :sapling_fvk_item_cache, :p2pkh_fvk_item_cache, + :birthday_height, :recover_until_height + ); + "#, + named_params![ + ":account_id": account_id, + ":account_type": account_type, + ":seed_id": seed_id.as_bytes(), + ":account_index": account_index, + ":ufvk": ufvk, + ":uivk": uivk, + ":orchard_fvk_item_cache": ufvk_parsed.orchard().map(|k| k.to_bytes()), + ":sapling_fvk_item_cache": ufvk_parsed.sapling().map(|k| k.to_bytes()), + ":p2pkh_fvk_item_cache": transparent_item, + ":birthday_height": birthday_height, + ":recover_until_height": recover_until_height, + ], + )?; } } else { return Err(WalletMigrationError::SeedRequired); @@ -177,7 +202,7 @@ impl RusqliteMigration for Migration

{ memo BLOB, spent INTEGER, commitment_tree_position INTEGER, - recipient_key_scope INTEGER NOT NULL DEFAULT 0, + recipient_key_scope INTEGER, FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), FOREIGN KEY (spent) REFERENCES transactions(id_tx), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs new file mode 100644 index 0000000000..014f80dc49 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_received_notes.rs @@ -0,0 +1,273 @@ +//! This migration adds tables to the wallet database that are needed to persist Orchard received +//! notes. + +use std::collections::HashSet; + +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; +use zcash_client_backend::{PoolType, ShieldedProtocol}; + +use super::full_account_ids; +use crate::wallet::{init::WalletMigrationError, pool_code}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x51d7a273_aa19_4109_9325_80e4a5545048); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [full_account_ids::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Add support for storage of Orchard received notes." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction<'_>) -> Result<(), Self::Error> { + transaction.execute_batch( + "CREATE TABLE orchard_received_notes ( + id INTEGER PRIMARY KEY, + tx INTEGER NOT NULL, + action_index INTEGER NOT NULL, + account_id INTEGER NOT NULL, + diversifier BLOB NOT NULL, + value INTEGER NOT NULL, + rho BLOB NOT NULL, + rseed BLOB NOT NULL, + nf BLOB UNIQUE, + is_change INTEGER NOT NULL, + memo BLOB, + spent INTEGER, + commitment_tree_position INTEGER, + recipient_key_scope INTEGER, + FOREIGN KEY (tx) REFERENCES transactions(id_tx), + FOREIGN KEY (account_id) REFERENCES accounts(id), + FOREIGN KEY (spent) REFERENCES transactions(id_tx), + CONSTRAINT tx_output UNIQUE (tx, action_index) + ); + CREATE INDEX orchard_received_notes_account ON orchard_received_notes ( + account_id ASC + ); + CREATE INDEX orchard_received_notes_tx ON orchard_received_notes ( + tx ASC + ); + CREATE INDEX orchard_received_notes_spent ON orchard_received_notes ( + spent ASC + );", + )?; + + transaction.execute_batch({ + let sapling_pool_code = pool_code(PoolType::Shielded(ShieldedProtocol::Sapling)); + let orchard_pool_code = pool_code(PoolType::Shielded(ShieldedProtocol::Orchard)); + &format!( + "CREATE VIEW v_received_notes AS + SELECT + id, + tx, + {sapling_pool_code} AS pool, + sapling_received_notes.output_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, {sapling_pool_code}, sapling_received_notes.output_index) + UNION + SELECT + id, + tx, + {orchard_pool_code} AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + value, + is_change, + memo, + spent, + sent_notes.id AS sent_note_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, {orchard_pool_code}, orchard_received_notes.action_index);" + ) + })?; + + transaction.execute_batch({ + let transparent_pool_code = pool_code(PoolType::Transparent); + &format!( + "DROP VIEW v_transactions; + CREATE VIEW v_transactions AS + WITH + notes AS ( + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + v_received_notes.value AS value, + CASE + WHEN v_received_notes.is_change THEN 1 + ELSE 0 + END AS is_change, + CASE + WHEN v_received_notes.is_change THEN 0 + ELSE 1 + END AS received_count, + CASE + WHEN (v_received_notes.memo IS NULL OR v_received_notes.memo = X'F6') + THEN 0 + ELSE 1 + END AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + utxos.height AS block, + utxos.prevout_txid AS txid, + {transparent_pool_code} AS pool, + utxos.value_zat AS value, + 0 AS is_change, + 1 AS received_count, + 0 AS memo_present + FROM utxos + UNION + SELECT v_received_notes.id AS id, + v_received_notes.account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + v_received_notes.pool AS pool, + -v_received_notes.value AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.spent + UNION + SELECT utxos.id AS id, + utxos.received_by_account_id AS account_id, + transactions.block AS block, + transactions.txid AS txid, + {transparent_pool_code} AS pool, + -utxos.value_zat AS value, + 0 AS is_change, + 0 AS received_count, + 0 AS memo_present + FROM utxos + JOIN transactions + ON transactions.id_tx = utxos.spent_in_tx + ), + sent_note_counts AS ( + SELECT sent_notes.from_account_id AS account_id, + transactions.txid AS txid, + COUNT(DISTINCT sent_notes.id) as sent_notes, + SUM( + CASE + WHEN (sent_notes.memo IS NULL OR sent_notes.memo = X'F6' OR v_received_notes.tx IS NOT NULL) + THEN 0 + ELSE 1 + END + ) AS memo_count + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0 + GROUP BY account_id, txid + ), + blocks_max_height AS ( + SELECT MAX(blocks.height) as max_height FROM blocks + ) + SELECT notes.account_id AS account_id, + notes.block AS mined_height, + notes.txid AS txid, + transactions.tx_index AS tx_index, + transactions.expiry_height AS expiry_height, + transactions.raw AS raw, + SUM(notes.value) AS account_balance_delta, + transactions.fee AS fee_paid, + SUM(notes.is_change) > 0 AS has_change, + MAX(COALESCE(sent_note_counts.sent_notes, 0)) AS sent_note_count, + SUM(notes.received_count) AS received_note_count, + SUM(notes.memo_present) + MAX(COALESCE(sent_note_counts.memo_count, 0)) AS memo_count, + blocks.time AS block_time, + ( + blocks.height IS NULL + AND transactions.expiry_height BETWEEN 1 AND blocks_max_height.max_height + ) AS expired_unmined + FROM notes + LEFT JOIN transactions + ON notes.txid = transactions.txid + JOIN blocks_max_height + LEFT JOIN blocks ON blocks.height = notes.block + LEFT JOIN sent_note_counts + ON sent_note_counts.account_id = notes.account_id + AND sent_note_counts.txid = notes.txid + GROUP BY notes.account_id, notes.txid; + + DROP VIEW v_tx_outputs; + CREATE VIEW v_tx_outputs AS + SELECT transactions.txid AS txid, + v_received_notes.pool AS output_pool, + v_received_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + NULL AS to_address, + v_received_notes.value AS value, + v_received_notes.is_change AS is_change, + v_received_notes.memo AS memo + FROM v_received_notes + JOIN transactions + ON transactions.id_tx = v_received_notes.tx + LEFT JOIN sent_notes + ON sent_notes.id = v_received_notes.sent_note_id + UNION + SELECT utxos.prevout_txid AS txid, + {transparent_pool_code} AS output_pool, + utxos.prevout_idx AS output_index, + NULL AS from_account_id, + utxos.received_by_account_id AS to_account_id, + utxos.address AS to_address, + utxos.value_zat AS value, + 0 AS is_change, + NULL AS memo + FROM utxos + UNION + SELECT transactions.txid AS txid, + sent_notes.output_pool AS output_pool, + sent_notes.output_index AS output_index, + sent_notes.from_account_id AS from_account_id, + v_received_notes.account_id AS to_account_id, + sent_notes.to_address AS to_address, + sent_notes.value AS value, + 0 AS is_change, + sent_notes.memo AS memo + FROM sent_notes + JOIN transactions + ON transactions.id_tx = sent_notes.tx + LEFT JOIN v_received_notes + ON sent_notes.id = v_received_notes.sent_note_id + WHERE COALESCE(v_received_notes.is_change, 0) = 0;") + })?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction<'_>) -> Result<(), Self::Error> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs new file mode 100644 index 0000000000..10e2796b8f --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/orchard_shardtree.rs @@ -0,0 +1,216 @@ +//! This migration adds tables to the wallet database that are needed to persist Orchard note +//! commitment tree data using the `shardtree` crate. + +use std::collections::HashSet; + +use rusqlite::{named_params, OptionalExtension}; +use schemer_rusqlite::RusqliteMigration; +use tracing::debug; +use uuid::Uuid; +use zcash_client_backend::data_api::scanning::ScanPriority; +use zcash_protocol::consensus::{self, BlockHeight, NetworkUpgrade}; + +use super::shardtree_support; +use crate::wallet::{init::WalletMigrationError, scan_queue_extrema, scanning::priority_code}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x3a6487f7_e068_42bb_9d12_6bb8dbe6da00); + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemer::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [shardtree_support::MIGRATION_ID].into_iter().collect() + } + + fn description(&self) -> &'static str { + "Add support for storage of Orchard note commitment tree data using the `shardtree` crate." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // Add shard persistence + debug!("Creating tables for Orchard shard persistence"); + transaction.execute_batch( + "CREATE TABLE orchard_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER, + CONSTRAINT root_unique UNIQUE (root_hash) + ); + CREATE TABLE orchard_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, + cap_data BLOB NOT NULL + );", + )?; + + // Add checkpoint persistence + debug!("Creating tables for checkpoint persistence"); + transaction.execute_batch( + "CREATE TABLE orchard_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + ); + CREATE TABLE orchard_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES orchard_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE, + CONSTRAINT spend_position_unique UNIQUE (checkpoint_id, mark_removed_position) + );", + )?; + + transaction.execute_batch(&format!( + "CREATE VIEW v_orchard_shard_scan_ranges AS + SELECT + shard.shard_index, + shard.shard_index << {} AS start_position, + (shard.shard_index + 1) << {} AS end_position_exclusive, + IFNULL(prev_shard.subtree_end_height, {}) AS subtree_start_height, + shard.subtree_end_height, + shard.contains_marked, + scan_queue.block_range_start, + scan_queue.block_range_end, + scan_queue.priority + FROM orchard_tree_shards shard + LEFT OUTER JOIN orchard_tree_shards prev_shard + ON shard.shard_index = prev_shard.shard_index + 1 + -- Join with scan ranges that overlap with the subtree's involved blocks. + INNER JOIN scan_queue ON ( + subtree_start_height < scan_queue.block_range_end AND + ( + scan_queue.block_range_start <= shard.subtree_end_height OR + shard.subtree_end_height IS NULL + ) + )", + 16, // ORCHARD_SHARD_HEIGHT is only available when `feature = "orchard"` is enabled. + 16, // ORCHARD_SHARD_HEIGHT is only available when `feature = "orchard"` is enabled. + u32::from(self.params.activation_height(NetworkUpgrade::Nu5).unwrap()), + ))?; + + transaction.execute_batch(&format!( + "CREATE VIEW v_orchard_shard_unscanned_ranges AS + WITH wallet_birthday AS (SELECT MIN(birthday_height) AS height FROM accounts) + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + block_range_start, + block_range_end, + priority + FROM v_orchard_shard_scan_ranges + INNER JOIN wallet_birthday + WHERE priority > {} + AND block_range_end > wallet_birthday.height;", + priority_code(&ScanPriority::Scanned), + ))?; + + transaction.execute_batch( + "CREATE VIEW v_orchard_shards_scan_state AS + SELECT + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked, + MAX(priority) AS max_priority + FROM v_orchard_shard_scan_ranges + GROUP BY + shard_index, + start_position, + end_position_exclusive, + subtree_start_height, + subtree_end_height, + contains_marked;", + )?; + + // Treat the current best-known chain tip height as the height to use for Orchard + // initialization, bounded below by NU5 activation. + if let Some(orchard_init_height) = scan_queue_extrema(transaction)?.and_then(|r| { + self.params + .activation_height(NetworkUpgrade::Nu5) + .map(|orchard_activation| std::cmp::max(orchard_activation, *r.end())) + }) { + // If a scan range exists that contains the Orchard init height, split it in two at the + // init height. + if let Some((start, end, range_priority)) = transaction + .query_row_and_then( + "SELECT block_range_start, block_range_end, priority + FROM scan_queue + WHERE block_range_start <= :orchard_init_height + AND block_range_end > :orchard_init_height", + named_params![":orchard_init_height": u32::from(orchard_init_height)], + |row| { + let start = BlockHeight::from(row.get::<_, u32>(0)?); + let end = BlockHeight::from(row.get::<_, u32>(1)?); + let range_priority: i64 = row.get(2)?; + Ok((start, end, range_priority)) + }, + ) + .optional()? + { + transaction.execute( + "DELETE from scan_queue WHERE block_range_start = :start", + named_params![":start": u32::from(start)], + )?; + if start < orchard_init_height { + // Rewrite the start of the scan range to be exactly what it was prior to the + // change. + transaction.execute( + "INSERT INTO scan_queue (block_range_start, block_range_end, priority) + VALUES (:block_range_start, :block_range_end, :priority)", + named_params![ + ":block_range_start": u32::from(start), + ":block_range_end": u32::from(orchard_init_height), + ":priority": range_priority, + ], + )?; + } + // Rewrite the remainder of the range to have at least priority `Historic` + transaction.execute( + "INSERT INTO scan_queue (block_range_start, block_range_end, priority) + VALUES (:block_range_start, :block_range_end, :priority)", + named_params![ + ":block_range_start": u32::from(orchard_init_height), + ":block_range_end": u32::from(end), + ":priority": + std::cmp::max(range_priority, priority_code(&ScanPriority::Historic)), + ], + )?; + // Rewrite any scanned ranges above the end of the first Orchard + // range to have at least priority `Historic` + transaction.execute( + "UPDATE scan_queue SET priority = :historic + WHERE :block_range_start >= :orchard_initial_range_end + AND priority < :historic", + named_params![ + ":historic": priority_code(&ScanPriority::Historic), + ":orchard_initial_range_end": u32::from(end), + ], + )?; + } + } + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index c3540bf2e4..2009587754 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -553,7 +553,7 @@ mod tests { row_count += 1; let value: u64 = row.get(0).unwrap(); let scope = parse_scope(row.get(1).unwrap()); - match dbg!(value) { + match value { EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), _ => { @@ -652,6 +652,10 @@ mod tests { block.block_time(), block.sapling().final_tree_size(), block.sapling().commitments().len().try_into().unwrap(), + #[cfg(feature = "orchard")] + block.orchard().final_tree_size(), + #[cfg(feature = "orchard")] + block.orchard().commitments().len().try_into().unwrap(), )?; for tx in block.transactions() { @@ -726,7 +730,7 @@ mod tests { row_count += 1; let value: u64 = row.get(0).unwrap(); let scope = parse_scope(row.get(1).unwrap()); - match dbg!(value) { + match value { EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), _ => { diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs new file mode 100644 index 0000000000..3b3859077c --- /dev/null +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -0,0 +1,612 @@ +use incrementalmerkletree::Position; +use orchard::{ + keys::Diversifier, + note::{Note, Nullifier, RandomSeed, Rho}, +}; +use rusqlite::{named_params, params, Connection, Row}; + +use zcash_client_backend::{ + data_api::NullifierQuery, + wallet::{ReceivedNote, WalletOrchardOutput}, + DecryptedOutput, ShieldedProtocol, TransferType, +}; +use zcash_keys::keys::UnifiedFullViewingKey; +use zcash_primitives::transaction::TxId; +use zcash_protocol::{ + consensus::{self, BlockHeight}, + memo::MemoBytes, + value::Zatoshis, +}; +use zip32::Scope; + +use crate::{error::SqliteClientError, AccountId, ReceivedNoteId}; + +use super::{memo_repr, parse_scope, scope_code}; + +/// This trait provides a generalization over shielded output representations. +pub(crate) trait ReceivedOrchardOutput { + fn index(&self) -> usize; + fn account_id(&self) -> AccountId; + fn note(&self) -> &Note; + fn memo(&self) -> Option<&MemoBytes>; + fn is_change(&self) -> bool; + fn nullifier(&self) -> Option<&Nullifier>; + fn note_commitment_tree_position(&self) -> Option; + fn recipient_key_scope(&self) -> Option; +} + +impl ReceivedOrchardOutput for WalletOrchardOutput { + fn index(&self) -> usize { + self.index() + } + fn account_id(&self) -> AccountId { + *WalletOrchardOutput::account_id(self) + } + fn note(&self) -> &Note { + WalletOrchardOutput::note(self) + } + fn memo(&self) -> Option<&MemoBytes> { + None + } + fn is_change(&self) -> bool { + WalletOrchardOutput::is_change(self) + } + fn nullifier(&self) -> Option<&Nullifier> { + self.nf() + } + fn note_commitment_tree_position(&self) -> Option { + Some(WalletOrchardOutput::note_commitment_tree_position(self)) + } + fn recipient_key_scope(&self) -> Option { + self.recipient_key_scope() + } +} + +impl ReceivedOrchardOutput for DecryptedOutput { + fn index(&self) -> usize { + self.index() + } + fn account_id(&self) -> AccountId { + *self.account() + } + fn note(&self) -> &orchard::note::Note { + self.note() + } + fn memo(&self) -> Option<&MemoBytes> { + Some(self.memo()) + } + fn is_change(&self) -> bool { + self.transfer_type() == TransferType::WalletInternal + } + fn nullifier(&self) -> Option<&Nullifier> { + None + } + fn note_commitment_tree_position(&self) -> Option { + None + } + fn recipient_key_scope(&self) -> Option { + if self.transfer_type() == TransferType::WalletInternal { + Some(Scope::Internal) + } else { + Some(Scope::External) + } + } +} + +fn to_spendable_note( + params: &P, + row: &Row, +) -> Result< + Option>, + SqliteClientError, +> { + let note_id = ReceivedNoteId(ShieldedProtocol::Orchard, row.get(0)?); + let txid = row.get::<_, [u8; 32]>(1).map(TxId::from_bytes)?; + let action_index = row.get(2)?; + let diversifier = { + let d: Vec<_> = row.get(3)?; + if d.len() != 11 { + return Err(SqliteClientError::CorruptedData( + "Invalid diversifier length".to_string(), + )); + } + let mut tmp = [0; 11]; + tmp.copy_from_slice(&d); + Diversifier::from_bytes(tmp) + }; + + let note_value: u64 = row.get::<_, i64>(4)?.try_into().map_err(|_e| { + SqliteClientError::CorruptedData("Note values must be nonnegative".to_string()) + })?; + + let rho = { + let rho_bytes: [u8; 32] = row.get(5)?; + Option::from(Rho::from_bytes(&rho_bytes)) + .ok_or_else(|| SqliteClientError::CorruptedData("Invalid rho.".to_string())) + }?; + + let rseed = { + let rseed_bytes: [u8; 32] = row.get(6)?; + Option::from(RandomSeed::from_bytes(rseed_bytes, &rho)).ok_or_else(|| { + SqliteClientError::CorruptedData("Invalid Orchard random seed.".to_string()) + }) + }?; + + let note_commitment_tree_position = + Position::from(u64::try_from(row.get::<_, i64>(7)?).map_err(|_| { + SqliteClientError::CorruptedData("Note commitment tree position invalid.".to_string()) + })?); + + let ufvk_str: Option = row.get(8)?; + let scope_code: Option = row.get(9)?; + + // If we don't have information about the recipient key scope or the ufvk we can't determine + // which spending key to use. This may be because the received note was associated with an + // imported viewing key, so we treat such notes as not spendable. Although this method is + // presently only called using the results of queries where both the ufvk and + // recipient_key_scope columns are checked to be non-null, this is method is written + // defensively to account for the fact that both of these are nullable columns in case it + // is used elsewhere in the future. + ufvk_str + .zip(scope_code) + .map(|(ufvk_str, scope_code)| { + let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) + .map_err(SqliteClientError::CorruptedData)?; + + let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { + SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) + })?; + let recipient = ufvk + .orchard() + .map(|fvk| fvk.to_ivk(spending_key_scope).address(diversifier)) + .ok_or_else(|| { + SqliteClientError::CorruptedData("Diversifier invalid.".to_owned()) + })?; + + let note = Option::from(Note::from_parts( + recipient, + orchard::value::NoteValue::from_raw(note_value), + rho, + rseed, + )) + .ok_or_else(|| SqliteClientError::CorruptedData("Invalid Orchard note.".to_string()))?; + + Ok(ReceivedNote::from_parts( + note_id, + txid, + action_index, + zcash_client_backend::wallet::Note::Orchard(note), + spending_key_scope, + note_commitment_tree_position, + )) + }) + .transpose() +} + +pub(crate) fn get_spendable_orchard_note( + conn: &Connection, + params: &P, + txid: &TxId, + index: u32, +) -> Result< + Option>, + SqliteClientError, +> { + super::common::get_spendable_note( + conn, + params, + txid, + index, + ShieldedProtocol::Orchard, + to_spendable_note, + ) +} + +pub(crate) fn select_spendable_orchard_notes( + conn: &Connection, + params: &P, + account: AccountId, + target_value: Zatoshis, + anchor_height: BlockHeight, + exclude: &[ReceivedNoteId], +) -> Result>, SqliteClientError> +{ + super::common::select_spendable_notes( + conn, + params, + account, + target_value, + anchor_height, + exclude, + ShieldedProtocol::Orchard, + to_spendable_note, + ) +} + +/// Records the specified shielded output as having been received. +/// +/// This implementation relies on the facts that: +/// - A transaction will not contain more than 2^63 shielded outputs. +/// - A note value will never exceed 2^63 zatoshis. +pub(crate) fn put_received_note( + conn: &Connection, + output: &T, + tx_ref: i64, + spent_in: Option, +) -> Result<(), SqliteClientError> { + let mut stmt_upsert_received_note = conn.prepare_cached( + "INSERT INTO orchard_received_notes + ( + tx, action_index, account_id, + diversifier, value, rho, rseed, memo, nf, + is_change, spent, commitment_tree_position, + recipient_key_scope + ) + VALUES ( + :tx, :action_index, :account_id, + :diversifier, :value, :rho, :rseed, :memo, :nf, + :is_change, :spent, :commitment_tree_position, + :recipient_key_scope + ) + ON CONFLICT (tx, action_index) DO UPDATE + SET account_id = :account_id, + diversifier = :diversifier, + value = :value, + rho = :rho, + rseed = :rseed, + nf = IFNULL(:nf, nf), + memo = IFNULL(:memo, memo), + is_change = IFNULL(:is_change, is_change), + spent = IFNULL(:spent, spent), + commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position), + recipient_key_scope = :recipient_key_scope", + )?; + + let rseed = output.note().rseed(); + let to = output.note().recipient(); + let diversifier = to.diversifier(); + + let sql_args = named_params![ + ":tx": &tx_ref, + ":action_index": i64::try_from(output.index()).expect("output indices are representable as i64"), + ":account_id": output.account_id().0, + ":diversifier": diversifier.as_array(), + ":value": output.note().value().inner(), + ":rho": output.note().rho().to_bytes(), + ":rseed": &rseed.as_bytes(), + ":nf": output.nullifier().map(|nf| nf.to_bytes()), + ":memo": memo_repr(output.memo()), + ":is_change": output.is_change(), + ":spent": spent_in, + ":commitment_tree_position": output.note_commitment_tree_position().map(u64::from), + ":recipient_key_scope": output.recipient_key_scope().map(scope_code), + ]; + + stmt_upsert_received_note + .execute(sql_args) + .map_err(SqliteClientError::from)?; + + Ok(()) +} + +/// Retrieves the set of nullifiers for "potentially spendable" Orchard notes that the +/// wallet is tracking. +/// +/// "Potentially spendable" means: +/// - The transaction in which the note was created has been observed as mined. +/// - No transaction in which the note's nullifier appears has been observed as mined. +pub(crate) fn get_orchard_nullifiers( + conn: &Connection, + query: NullifierQuery, +) -> Result, SqliteClientError> { + // Get the nullifiers for the notes we are tracking + let mut stmt_fetch_nullifiers = match query { + NullifierQuery::Unspent => conn.prepare( + "SELECT rn.account_id, rn.nf + FROM orchard_received_notes rn + LEFT OUTER JOIN transactions tx + ON tx.id_tx = rn.spent + WHERE tx.block IS NULL + AND nf IS NOT NULL", + )?, + NullifierQuery::All => conn.prepare( + "SELECT rn.account_id, rn.nf + FROM orchard_received_notes rn + WHERE nf IS NOT NULL", + )?, + }; + + let nullifiers = stmt_fetch_nullifiers.query_and_then([], |row| { + let account = AccountId(row.get(0)?); + let nf_bytes: [u8; 32] = row.get(1)?; + Ok::<_, rusqlite::Error>((account, Nullifier::from_bytes(&nf_bytes).unwrap())) + })?; + + let res: Vec<_> = nullifiers.collect::>()?; + Ok(res) +} + +/// Marks a given nullifier as having been revealed in the construction +/// of the specified transaction. +/// +/// Marking a note spent in this fashion does NOT imply that the +/// spending transaction has been mined. +pub(crate) fn mark_orchard_note_spent( + conn: &Connection, + tx_ref: i64, + nf: &Nullifier, +) -> Result { + let mut stmt_mark_orchard_note_spent = + conn.prepare_cached("UPDATE orchard_received_notes SET spent = ? WHERE nf = ?")?; + + match stmt_mark_orchard_note_spent.execute(params![tx_ref, nf.to_bytes()])? { + 0 => Ok(false), + 1 => Ok(true), + _ => unreachable!("nf column is marked as UNIQUE"), + } +} + +#[cfg(test)] +pub(crate) mod tests { + use incrementalmerkletree::{Hashable, Level}; + use orchard::{ + keys::{FullViewingKey, SpendingKey}, + note_encryption::OrchardDomain, + tree::MerkleHashOrchard, + }; + use shardtree::error::ShardTreeError; + use zcash_client_backend::{ + data_api::{ + chain::CommitmentTreeRoot, DecryptedTransaction, WalletCommitmentTrees, WalletSummary, + }, + wallet::{Note, ReceivedNote}, + }; + use zcash_keys::{ + address::{Address, UnifiedAddress}, + keys::UnifiedSpendingKey, + }; + use zcash_note_encryption::try_output_recovery_with_ovk; + use zcash_primitives::transaction::Transaction; + use zcash_protocol::{consensus::BlockHeight, memo::MemoBytes, ShieldedProtocol}; + + use super::select_spendable_orchard_notes; + use crate::{ + error::SqliteClientError, + testing::{ + self, + pool::{OutputRecoveryError, ShieldedPoolTester}, + TestState, + }, + wallet::commitment_tree, + ORCHARD_TABLES_PREFIX, + }; + + pub(crate) struct OrchardPoolTester; + impl ShieldedPoolTester for OrchardPoolTester { + const SHIELDED_PROTOCOL: ShieldedProtocol = ShieldedProtocol::Orchard; + const TABLES_PREFIX: &'static str = ORCHARD_TABLES_PREFIX; + + type Sk = SpendingKey; + type Fvk = FullViewingKey; + type MerkleTreeHash = MerkleHashOrchard; + + fn test_account_fvk(st: &TestState) -> Self::Fvk { + st.test_account_orchard().unwrap() + } + + fn usk_to_sk(usk: &UnifiedSpendingKey) -> &Self::Sk { + usk.orchard() + } + + fn sk(seed: &[u8]) -> Self::Sk { + let mut account = zip32::AccountId::ZERO; + loop { + if let Ok(sk) = SpendingKey::from_zip32_seed(seed, 1, account) { + break sk; + } + account = account.next().unwrap(); + } + } + + fn sk_to_fvk(sk: &Self::Sk) -> Self::Fvk { + sk.into() + } + + fn sk_default_address(sk: &Self::Sk) -> Address { + Self::fvk_default_address(&Self::sk_to_fvk(sk)) + } + + fn fvk_default_address(fvk: &Self::Fvk) -> Address { + UnifiedAddress::from_receivers( + Some(fvk.address_at(0u32, zip32::Scope::External)), + None, + None, + ) + .unwrap() + .into() + } + + fn fvks_equal(a: &Self::Fvk, b: &Self::Fvk) -> bool { + a == b + } + + fn empty_tree_leaf() -> Self::MerkleTreeHash { + MerkleHashOrchard::empty_leaf() + } + + fn empty_tree_root(level: Level) -> Self::MerkleTreeHash { + MerkleHashOrchard::empty_root(level) + } + + fn put_subtree_roots( + st: &mut TestState, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError> { + st.wallet_mut() + .put_orchard_subtree_roots(start_index, roots) + } + + fn next_subtree_index(s: &WalletSummary) -> u64 { + s.next_orchard_subtree_index() + } + + fn select_spendable_notes( + st: &TestState, + account: crate::AccountId, + target_value: zcash_protocol::value::Zatoshis, + anchor_height: BlockHeight, + exclude: &[crate::ReceivedNoteId], + ) -> Result>, SqliteClientError> { + select_spendable_orchard_notes( + &st.wallet().conn, + &st.wallet().params, + account, + target_value, + anchor_height, + exclude, + ) + } + + fn decrypted_pool_outputs_count( + d_tx: &DecryptedTransaction<'_, crate::AccountId>, + ) -> usize { + d_tx.orchard_outputs().len() + } + + fn with_decrypted_pool_memos( + d_tx: &DecryptedTransaction<'_, crate::AccountId>, + mut f: impl FnMut(&MemoBytes), + ) { + for output in d_tx.orchard_outputs() { + f(output.memo()); + } + } + + fn try_output_recovery( + _: &TestState, + _: BlockHeight, + tx: &Transaction, + fvk: &Self::Fvk, + ) -> Result, OutputRecoveryError> { + for action in tx.orchard_bundle().unwrap().actions() { + // Find the output that decrypts with the external OVK + let result = try_output_recovery_with_ovk( + &OrchardDomain::for_action(action), + &fvk.to_ovk(zip32::Scope::External), + action, + action.cv_net(), + &action.encrypted_note().out_ciphertext, + ); + + if result.is_some() { + return Ok(result.map(|(note, addr, memo)| { + ( + Note::Orchard(note), + UnifiedAddress::from_receivers(Some(addr), None, None) + .unwrap() + .into(), + MemoBytes::from_bytes(&memo).expect("correct length"), + ) + })); + } + } + + Ok(None) + } + + fn received_note_count( + summary: &zcash_client_backend::data_api::chain::ScanSummary, + ) -> usize { + summary.received_orchard_note_count() + } + } + + #[test] + fn send_single_step_proposed_transfer() { + testing::pool::send_single_step_proposed_transfer::() + } + + #[test] + #[cfg(feature = "transparent-inputs")] + fn send_multi_step_proposed_transfer() { + testing::pool::send_multi_step_proposed_transfer::() + } + + #[test] + #[allow(deprecated)] + fn create_to_address_fails_on_incorrect_usk() { + testing::pool::create_to_address_fails_on_incorrect_usk::() + } + + #[test] + #[allow(deprecated)] + fn proposal_fails_with_no_blocks() { + testing::pool::proposal_fails_with_no_blocks::() + } + + #[test] + fn spend_fails_on_unverified_notes() { + testing::pool::spend_fails_on_unverified_notes::() + } + + #[test] + fn spend_fails_on_locked_notes() { + testing::pool::spend_fails_on_locked_notes::() + } + + #[test] + fn ovk_policy_prevents_recovery_from_chain() { + testing::pool::ovk_policy_prevents_recovery_from_chain::() + } + + #[test] + fn spend_succeeds_to_t_addr_zero_change() { + testing::pool::spend_succeeds_to_t_addr_zero_change::() + } + + #[test] + fn change_note_spends_succeed() { + testing::pool::change_note_spends_succeed::() + } + + #[test] + fn external_address_change_spends_detected_in_restore_from_seed() { + testing::pool::external_address_change_spends_detected_in_restore_from_seed::< + OrchardPoolTester, + >() + } + + #[test] + fn zip317_spend() { + testing::pool::zip317_spend::() + } + + #[test] + #[cfg(feature = "transparent-inputs")] + fn shield_transparent() { + testing::pool::shield_transparent::() + } + + #[test] + fn birthday_in_anchor_shard() { + testing::pool::birthday_in_anchor_shard::() + } + + #[test] + fn checkpoint_gaps() { + testing::pool::checkpoint_gaps::() + } + + #[test] + fn scan_cached_blocks_detects_spends_out_of_order() { + testing::pool::scan_cached_blocks_detects_spends_out_of_order::() + } + + #[test] + fn cross_pool_exchange() { + use crate::wallet::sapling::tests::SaplingPoolTester; + + testing::pool::cross_pool_exchange::() + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 17300422f4..38f076619f 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -2,16 +2,15 @@ use group::ff::PrimeField; use incrementalmerkletree::Position; -use rusqlite::{named_params, params, types::Value, Connection, Row}; -use std::rc::Rc; +use rusqlite::{named_params, params, Connection, Row}; use sapling::{self, Diversifier, Nullifier, Rseed}; use zcash_client_backend::{ data_api::NullifierQuery, - keys::UnifiedFullViewingKey, wallet::{Note, ReceivedNote, WalletSaplingOutput}, DecryptedOutput, ShieldedProtocol, TransferType, }; +use zcash_keys::keys::UnifiedFullViewingKey; use zcash_primitives::transaction::{components::amount::NonNegativeAmount, TxId}; use zcash_protocol::{ consensus::{self, BlockHeight}, @@ -21,7 +20,7 @@ use zip32::Scope; use crate::{error::SqliteClientError, AccountId, ReceivedNoteId}; -use super::{memo_repr, parse_scope, scope_code, wallet_birthday}; +use super::{memo_repr, parse_scope, scope_code}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { @@ -96,7 +95,7 @@ impl ReceivedSaplingOutput for DecryptedOutput { fn to_spendable_note( params: &P, row: &Row, -) -> Result, SqliteClientError> { +) -> Result>, SqliteClientError> { let note_id = ReceivedNoteId(ShieldedProtocol::Sapling, row.get(0)?); let txid = row.get::<_, [u8; 32]>(1).map(TxId::from_bytes)?; let output_index = row.get(2)?; @@ -136,37 +135,50 @@ fn to_spendable_note( SqliteClientError::CorruptedData("Note commitment tree position invalid.".to_string()) })?); - let ufvk_str: String = row.get(7)?; - let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) - .map_err(SqliteClientError::CorruptedData)?; - - let scope_code: i64 = row.get(8)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; - - let recipient = match spending_key_scope { - Scope::Internal => ufvk - .sapling() - .and_then(|dfvk| dfvk.diversified_change_address(diversifier)), - Scope::External => ufvk - .sapling() - .and_then(|dfvk| dfvk.diversified_address(diversifier)), - } - .ok_or_else(|| SqliteClientError::CorruptedData("Diversifier invalid.".to_owned()))?; - - Ok(ReceivedNote::from_parts( - note_id, - txid, - output_index, - Note::Sapling(sapling::Note::from_parts( - recipient, - sapling::value::NoteValue::from_raw(note_value), - rseed, - )), - spending_key_scope, - note_commitment_tree_position, - )) + let ufvk_str: Option = row.get(7)?; + let scope_code: Option = row.get(8)?; + + // If we don't have information about the recipient key scope or the ufvk we can't determine + // which spending key to use. This may be because the received note was associated with an + // imported viewing key, so we treat such notes as not spendable. Although this method is + // presently only called using the results of queries where both the ufvk and + // recipient_key_scope columns are checked to be non-null, this is method is written + // defensively to account for the fact that both of these are nullable columns in case it + // is used elsewhere in the future. + ufvk_str + .zip(scope_code) + .map(|(ufvk_str, scope_code)| { + let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) + .map_err(SqliteClientError::CorruptedData)?; + + let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { + SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) + })?; + + let recipient = match spending_key_scope { + Scope::Internal => ufvk + .sapling() + .and_then(|dfvk| dfvk.diversified_change_address(diversifier)), + Scope::External => ufvk + .sapling() + .and_then(|dfvk| dfvk.diversified_address(diversifier)), + } + .ok_or_else(|| SqliteClientError::CorruptedData("Diversifier invalid.".to_owned()))?; + + Ok(ReceivedNote::from_parts( + note_id, + txid, + output_index, + Note::Sapling(sapling::Note::from_parts( + recipient, + sapling::value::NoteValue::from_raw(note_value), + rseed, + )), + spending_key_scope, + note_commitment_tree_position, + )) + }) + .transpose() } // The `clippy::let_and_return` lint is explicitly allowed here because a bug in Clippy @@ -179,29 +191,14 @@ pub(crate) fn get_spendable_sapling_note( txid: &TxId, index: u32, ) -> Result>, SqliteClientError> { - let mut stmt_select_note = conn.prepare_cached( - "SELECT sapling_received_notes.id, txid, output_index, diversifier, value, rcm, commitment_tree_position, - accounts.ufvk, recipient_key_scope - FROM sapling_received_notes - INNER JOIN accounts on accounts.id = sapling_received_notes.account_id - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE txid = :txid AND accounts.ufvk IS NOT NULL - AND output_index = :output_index - AND spent IS NULL", - )?; - - let result = stmt_select_note - .query_and_then( - named_params![ - ":txid": txid.as_ref(), - ":output_index": index, - ], - |r| to_spendable_note(params, r), - )? - .next() - .transpose(); - - result + super::common::get_spendable_note( + conn, + params, + txid, + index, + ShieldedProtocol::Sapling, + to_spendable_note, + ) } /// Utility method for determining whether we have any spendable notes @@ -209,25 +206,6 @@ pub(crate) fn get_spendable_sapling_note( /// If the tip shard has unscanned ranges below the anchor height and greater than or equal to /// the wallet birthday, none of our notes can be spent because we cannot construct witnesses at /// the provided anchor height. -fn unscanned_tip_exists( - conn: &Connection, - anchor_height: BlockHeight, -) -> Result { - // v_sapling_shard_unscanned_ranges only returns ranges ending on or after wallet birthday, so - // we don't need to refer to the birthday in this query. - conn.query_row( - "SELECT EXISTS ( - SELECT 1 FROM v_sapling_shard_unscanned_ranges range - WHERE range.block_range_start <= :anchor_height - AND :anchor_height BETWEEN - range.subtree_start_height - AND IFNULL(range.subtree_end_height, :anchor_height) - )", - named_params![":anchor_height": u32::from(anchor_height),], - |row| row.get::<_, bool>(0), - ) -} - pub(crate) fn select_spendable_sapling_notes( conn: &Connection, params: &P, @@ -236,84 +214,16 @@ pub(crate) fn select_spendable_sapling_notes( anchor_height: BlockHeight, exclude: &[ReceivedNoteId], ) -> Result>, SqliteClientError> { - let birthday_height = match wallet_birthday(conn)? { - Some(birthday) => birthday, - None => { - // the wallet birthday can only be unknown if there are no accounts in the wallet; in - // such a case, the wallet has no notes to spend. - return Ok(vec![]); - } - }; - - if unscanned_tip_exists(conn, anchor_height)? { - return Ok(vec![]); - } - - // The goal of this SQL statement is to select the oldest notes until the required - // value has been reached. - // 1) Use a window function to create a view of all notes, ordered from oldest to - // newest, with an additional column containing a running sum: - // - Unspent notes accumulate the values of all unspent notes in that note's - // account, up to itself. - // - Spent notes accumulate the values of all notes in the transaction they were - // spent in, up to itself. - // - // 2) Select all unspent notes in the desired account, along with their running sum. - // - // 3) Select all notes for which the running sum was less than the required value, as - // well as a single note for which the sum was greater than or equal to the - // required value, bringing the sum of all selected notes across the threshold. - // - // 4) Match the selected notes against the witnesses at the desired height. - let mut stmt_select_notes = conn.prepare_cached( - "WITH eligible AS ( - SELECT - sapling_received_notes.id AS id, txid, output_index, diversifier, value, rcm, commitment_tree_position, - SUM(value) - OVER (PARTITION BY sapling_received_notes.account_id, spent ORDER BY sapling_received_notes.id) AS so_far, - accounts.ufvk as ufvk, recipient_key_scope - FROM sapling_received_notes - INNER JOIN accounts on accounts.id = sapling_received_notes.account_id - INNER JOIN transactions - ON transactions.id_tx = sapling_received_notes.tx - WHERE sapling_received_notes.account_id = :account AND ufvk IS NOT NULL - AND commitment_tree_position IS NOT NULL - AND spent IS NULL - AND transactions.block <= :anchor_height - AND sapling_received_notes.id NOT IN rarray(:exclude) - AND NOT EXISTS ( - SELECT 1 FROM v_sapling_shard_unscanned_ranges unscanned - -- select all the unscanned ranges involving the shard containing this note - WHERE sapling_received_notes.commitment_tree_position >= unscanned.start_position - AND sapling_received_notes.commitment_tree_position < unscanned.end_position_exclusive - -- exclude unscanned ranges that start above the anchor height (they don't affect spendability) - AND unscanned.block_range_start <= :anchor_height - -- exclude unscanned ranges that end below the wallet birthday - AND unscanned.block_range_end > :wallet_birthday - ) - ) - SELECT id, txid, output_index, diversifier, value, rcm, commitment_tree_position, ufvk, recipient_key_scope - FROM eligible WHERE so_far < :target_value - UNION - SELECT id, txid, output_index, diversifier, value, rcm, commitment_tree_position, ufvk, recipient_key_scope - FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", - )?; - - let excluded: Vec = exclude.iter().map(|n| Value::from(n.1)).collect(); - let excluded_ptr = Rc::new(excluded); - - let notes = stmt_select_notes.query_and_then( - named_params![ - ":account": account.0, - ":anchor_height": &u32::from(anchor_height), - ":target_value": &u64::from(target_value), - ":exclude": &excluded_ptr, - ":wallet_birthday": u32::from(birthday_height) - ], - |r| to_spendable_note(params, r), - )?; - - notes.collect::>() + super::common::select_spendable_notes( + conn, + params, + account, + target_value, + anchor_height, + exclude, + ShieldedProtocol::Sapling, + to_spendable_note, + ) } /// Retrieves the set of nullifiers for "potentially spendable" Sapling notes that the @@ -420,12 +330,6 @@ pub(crate) fn put_received_note( let to = output.note().recipient(); let diversifier = to.diversifier(); - // FIXME: recipient key scope will always be available until IVK import is supported. - // Remove this expectation after #1175 merges. - let scope = output - .recipient_key_scope() - .expect("Key import is not yet supported."); - let sql_args = named_params![ ":tx": &tx_ref, ":output_index": i64::try_from(output.index()).expect("output indices are representable as i64"), @@ -438,7 +342,7 @@ pub(crate) fn put_received_note( ":is_change": output.is_change(), ":spent": spent_in, ":commitment_tree_position": output.note_commitment_tree_position().map(u64::from), - ":recipient_key_scope": scope_code(scope), + ":recipient_key_scope": output.recipient_key_scope().map(scope_code) ]; stmt_upsert_received_note @@ -450,30 +354,21 @@ pub(crate) fn put_received_note( #[cfg(test)] pub(crate) mod tests { - use std::{convert::Infallible, num::NonZeroU32}; - - use incrementalmerkletree::Hashable; - use rusqlite::params; - use secrecy::Secret; + use incrementalmerkletree::{Hashable, Level}; + use shardtree::error::ShardTreeError; use zcash_proofs::prover::LocalTxProver; use sapling::{ self, note_encryption::try_sapling_output_recovery, prover::{OutputProver, SpendProver}, - zip32::ExtendedSpendingKey, - Node, PaymentAddress, + zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey}, }; use zcash_primitives::{ - block::BlockHash, - consensus::BranchId, - legacy::TransparentAddress, - memo::{Memo, MemoBytes}, + consensus::BlockHeight, + memo::MemoBytes, transaction::{ components::{amount::NonNegativeAmount, sapling::zip212_enforcement}, - fees::{ - fixed::FeeRule as FixedFeeRule, zip317::FeeError as Zip317FeeError, StandardFeeRule, - }, Transaction, }, zip32::Scope, @@ -482,775 +377,131 @@ pub(crate) mod tests { use zcash_client_backend::{ address::Address, data_api::{ - self, - chain::CommitmentTreeRoot, - error::Error, - wallet::input_selection::{GreedyInputSelector, GreedyInputSelectorError}, - AccountBirthday, Ratio, WalletCommitmentTrees, WalletRead, WalletWrite, + chain::CommitmentTreeRoot, DecryptedTransaction, WalletCommitmentTrees, WalletSummary, }, - decrypt_transaction, - fees::{fixed, standard, DustOutputPolicy}, keys::UnifiedSpendingKey, - wallet::OvkPolicy, - zip321::{self, Payment, TransactionRequest}, + wallet::{Note, ReceivedNote}, ShieldedProtocol, }; use crate::{ error::SqliteClientError, - testing::{input_selector, AddressType, BlockCache, TestBuilder, TestState}, - wallet::{ - block_max_scanned, commitment_tree, parse_scope, - sapling::select_spendable_sapling_notes, scanning::tests::test_with_canopy_birthday, + testing::{ + self, + pool::{OutputRecoveryError, ShieldedPoolTester}, + TestState, }, - NoteId, ReceivedNoteId, + wallet::{commitment_tree, sapling::select_spendable_sapling_notes}, + AccountId, ReceivedNoteId, SAPLING_TABLES_PREFIX, }; - #[cfg(feature = "transparent-inputs")] - use { - zcash_client_backend::{ - fees::TransactionBalance, proposal::Step, wallet::WalletTransparentOutput, PoolType, - }, - zcash_primitives::{ - legacy::keys::IncomingViewingKey, - transaction::components::{OutPoint, TxOut}, - }, - }; + pub(crate) struct SaplingPoolTester; + impl ShieldedPoolTester for SaplingPoolTester { + const SHIELDED_PROTOCOL: ShieldedProtocol = ShieldedProtocol::Sapling; + const TABLES_PREFIX: &'static str = SAPLING_TABLES_PREFIX; - pub(crate) fn test_prover() -> impl SpendProver + OutputProver { - LocalTxProver::bundled() - } + type Sk = ExtendedSpendingKey; + type Fvk = DiversifiableFullViewingKey; + type MerkleTreeHash = sapling::Node; - #[test] - fn send_single_step_proposed_transfer() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::const_from_u64(60000); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h, 1); - - // Spendable balance matches total balance - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - assert_eq!( - block_max_scanned(&st.wallet().conn, &st.wallet().params) - .unwrap() - .unwrap() - .block_height(), - h - ); - - let to_extsk = ExtendedSpendingKey::master(&[]); - let to: Address = to_extsk.default_address().1.into(); - let request = zip321::TransactionRequest::new(vec![Payment { - recipient_address: to, - amount: NonNegativeAmount::const_from_u64(10000), - memo: None, // this should result in the creation of an empty memo - label: None, - message: None, - other_params: vec![], - }]) - .unwrap(); - - // TODO: This test was originally written to use the pre-zip-313 fee rule - // and has not yet been updated. - #[allow(deprecated)] - let fee_rule = StandardFeeRule::PreZip313; - - let change_memo = "Test change memo".parse::().unwrap(); - let change_strategy = standard::SingleOutputChangeStrategy::new( - fee_rule, - Some(change_memo.clone().into()), - ShieldedProtocol::Sapling, - ); - let input_selector = - &GreedyInputSelector::new(change_strategy, DustOutputPolicy::default()); - - let proposal = st - .propose_transfer( - account, - input_selector, - request, - NonZeroU32::new(1).unwrap(), - ) - .unwrap(); - - let create_proposed_result = - st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal); - assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 1); - - let sent_tx_id = create_proposed_result.unwrap()[0]; - - // Verify that the sent transaction was stored and that we can decrypt the memos - let tx = st - .wallet() - .get_transaction(sent_tx_id) - .expect("Created transaction was stored."); - let ufvks = [(account, usk.to_unified_full_viewing_key())] - .into_iter() - .collect(); - let d_tx = decrypt_transaction(&st.network(), h + 1, &tx, &ufvks); - assert_eq!(d_tx.sapling_outputs().len(), 2); - - let mut found_tx_change_memo = false; - let mut found_tx_empty_memo = false; - for output in d_tx.sapling_outputs() { - if Memo::try_from(output.memo()).unwrap() == change_memo { - found_tx_change_memo = true - } - if Memo::try_from(output.memo()).unwrap() == Memo::Empty { - found_tx_empty_memo = true - } + fn test_account_fvk(st: &TestState) -> Self::Fvk { + st.test_account_sapling().unwrap() } - assert!(found_tx_change_memo); - assert!(found_tx_empty_memo); - - // Verify that the stored sent notes match what we're expecting - let mut stmt_sent_notes = st - .wallet() - .conn - .prepare( - "SELECT output_index - FROM sent_notes - JOIN transactions ON transactions.id_tx = sent_notes.tx - WHERE transactions.txid = ?", - ) - .unwrap(); - - let sent_note_ids = stmt_sent_notes - .query(rusqlite::params![sent_tx_id.as_ref()]) - .unwrap() - .mapped(|row| { - Ok(NoteId::new( - sent_tx_id, - ShieldedProtocol::Sapling, - row.get(0)?, - )) - }) - .collect::, _>>() - .unwrap(); - - assert_eq!(sent_note_ids.len(), 2); - - // The sent memo should be the empty memo for the sent output, and the - // change output's memo should be as specified. - let mut found_sent_change_memo = false; - let mut found_sent_empty_memo = false; - for sent_note_id in sent_note_ids { - match st - .wallet() - .get_memo(sent_note_id) - .expect("Note id is valid") - .as_ref() - { - Some(m) if m == &change_memo => { - found_sent_change_memo = true; - } - Some(m) if m == &Memo::Empty => { - found_sent_empty_memo = true; - } - Some(other) => panic!("Unexpected memo value: {:?}", other), - None => panic!("Memo should not be stored as NULL"), - } - } - assert!(found_sent_change_memo); - assert!(found_sent_empty_memo); - - // Check that querying for a nonexistent sent note returns None - assert_matches!( - st.wallet() - .get_memo(NoteId::new(sent_tx_id, ShieldedProtocol::Sapling, 12345)), - Ok(None) - ); - } - - #[test] - #[cfg(feature = "transparent-inputs")] - fn send_multi_step_proposed_transfer() { - use nonempty::NonEmpty; - use zcash_client_backend::proposal::{Proposal, StepOutput, StepOutputIndex}; - - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::const_from_u64(65000); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h, 1); - - // Spendable balance matches total balance - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - assert_eq!( - block_max_scanned(&st.wallet().conn, &st.wallet().params) - .unwrap() - .unwrap() - .block_height(), - h - ); - - // Generate a single-step proposal. Then, instead of executing that proposal, - // we will use its only step as the first step in a multi-step proposal that - // spends the first step's output. - - // The first step will deshield to the wallet's default transparent address - let to0 = Address::Transparent(usk.default_transparent_address().0); - let request0 = zip321::TransactionRequest::new(vec![Payment { - recipient_address: to0, - amount: NonNegativeAmount::const_from_u64(50000), - memo: None, - label: None, - message: None, - other_params: vec![], - }]) - .unwrap(); - - let fee_rule = StandardFeeRule::Zip317; - let input_selector = GreedyInputSelector::new( - standard::SingleOutputChangeStrategy::new(fee_rule, None, ShieldedProtocol::Sapling), - DustOutputPolicy::default(), - ); - let proposal0 = st - .propose_transfer( - account, - &input_selector, - request0, - NonZeroU32::new(1).unwrap(), - ) - .unwrap(); - - let min_target_height = proposal0.min_target_height(); - let step0 = &proposal0.steps().head; - - assert!(step0.balance().proposed_change().is_empty()); - assert_eq!( - step0.balance().fee_required(), - NonNegativeAmount::const_from_u64(15000) - ); - - // We'll use an internal transparent address that hasn't been added to the wallet - // to simulate an external transparent recipient. - let to1 = Address::Transparent( - usk.transparent() - .to_account_pubkey() - .derive_internal_ivk() - .unwrap() - .default_address() - .0, - ); - let request1 = zip321::TransactionRequest::new(vec![Payment { - recipient_address: to1, - amount: NonNegativeAmount::const_from_u64(40000), - memo: None, - label: None, - message: None, - other_params: vec![], - }]) - .unwrap(); - - let step1 = Step::from_parts( - &[step0.clone()], - request1, - [(0, PoolType::Transparent)].into_iter().collect(), - vec![], - None, - vec![StepOutput::new(0, StepOutputIndex::Payment(0))], - TransactionBalance::new(vec![], NonNegativeAmount::const_from_u64(10000)).unwrap(), - false, - ) - .unwrap(); - let proposal = Proposal::multi_step( - fee_rule, - min_target_height, - NonEmpty::from_vec(vec![step0.clone(), step1]).unwrap(), - ) - .unwrap(); - - let create_proposed_result = - st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal); - assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); - let txids = create_proposed_result.unwrap(); - - // Verify that the stored sent outputs match what we're expecting - let mut stmt_sent = st - .wallet() - .conn - .prepare( - "SELECT value - FROM sent_notes - JOIN transactions ON transactions.id_tx = sent_notes.tx - WHERE transactions.txid = ?", - ) - .unwrap(); - - let confirmed_sent = txids - .iter() - .map(|sent_txid| { - // check that there's a sent output with the correct value corresponding to - stmt_sent - .query(rusqlite::params![sent_txid.as_ref()]) - .unwrap() - .mapped(|row| { - let value: u32 = row.get(0)?; - Ok((sent_txid, value)) - }) - .collect::, _>>() - .unwrap() - }) - .collect::>(); - - assert_eq!( - confirmed_sent.get(0).and_then(|v| v.get(0)), - Some(&(&txids[0], 50000)) - ); - assert_eq!( - confirmed_sent.get(1).and_then(|v| v.get(0)), - Some(&(&txids[1], 40000)) - ); - } - - #[test] - #[allow(deprecated)] - fn create_to_address_fails_on_incorrect_usk() { - let mut st = TestBuilder::new() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - let dfvk = st.test_account_sapling().unwrap(); - let to = dfvk.default_address().1.into(); - - // Create a USK that doesn't exist in the wallet - let acct1 = zip32::AccountId::try_from(1).unwrap(); - let usk1 = UnifiedSpendingKey::from_seed(&st.network(), &[1u8; 32], acct1).unwrap(); - - // Attempting to spend with a USK that is not in the wallet results in an error - assert_matches!( - st.create_spend_to_address( - &usk1, - &to, - NonNegativeAmount::const_from_u64(1), - None, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - None, - ShieldedProtocol::Sapling - ), - Err(data_api::error::Error::KeyNotRecognized) - ); - } - - #[test] - #[allow(deprecated)] - fn proposal_fails_with_no_blocks() { - let mut st = TestBuilder::new() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); + fn usk_to_sk(usk: &UnifiedSpendingKey) -> &Self::Sk { + usk.sapling() + } - let (account, _, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - let to = dfvk.default_address().1.into(); + fn sk(seed: &[u8]) -> Self::Sk { + ExtendedSpendingKey::master(seed) + } - // Wallet summary is not yet available - assert_eq!(st.get_wallet_summary(0), None); + fn sk_to_fvk(sk: &Self::Sk) -> Self::Fvk { + sk.to_diversifiable_full_viewing_key() + } - // We cannot do anything if we aren't synchronised - assert_matches!( - st.propose_standard_transfer::( - account, - StandardFeeRule::PreZip313, - NonZeroU32::new(1).unwrap(), - &to, - NonNegativeAmount::const_from_u64(1), - None, - None, - ShieldedProtocol::Sapling - ), - Err(data_api::error::Error::ScanRequired) - ); - } + fn sk_default_address(sk: &Self::Sk) -> Address { + sk.default_address().1.into() + } - #[test] - fn spend_fails_on_unverified_notes() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::const_from_u64(50000); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h1, 1); - - // Spendable balance matches total balance at 1 confirmation. - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - // Value is considered pending at 10 confirmations. - assert_eq!(st.get_pending_shielded_balance(account, 10), value); - assert_eq!( - st.get_spendable_balance(account, 10), - NonNegativeAmount::ZERO - ); - - // Wallet is fully scanned - let summary = st.get_wallet_summary(1); - assert_eq!( - summary.and_then(|s| s.scan_progress()), - Some(Ratio::new(1, 1)) - ); - - // Add more funds to the wallet in a second note - let (h2, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h2, 1); - - // Verified balance does not include the second note - let total = (value + value).unwrap(); - assert_eq!(st.get_spendable_balance(account, 2), value); - assert_eq!(st.get_pending_shielded_balance(account, 2), value); - assert_eq!(st.get_total_balance(account), total); - - // Wallet is still fully scanned - let summary = st.get_wallet_summary(1); - assert_eq!( - summary.and_then(|s| s.scan_progress()), - Some(Ratio::new(2, 2)) - ); - - // Spend fails because there are insufficient verified notes - let extsk2 = ExtendedSpendingKey::master(&[]); - let to = extsk2.default_address().1.into(); - assert_matches!( - st.propose_standard_transfer::( - account, - StandardFeeRule::Zip317, - NonZeroU32::new(2).unwrap(), - &to, - NonNegativeAmount::const_from_u64(70000), - None, - None, - ShieldedProtocol::Sapling - ), - Err(data_api::error::Error::InsufficientFunds { - available, - required - }) - if available == NonNegativeAmount::const_from_u64(50000) - && required == NonNegativeAmount::const_from_u64(80000) - ); - - // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second - // note is verified - for _ in 2..10 { - st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); + fn fvk_default_address(fvk: &Self::Fvk) -> Address { + fvk.default_address().1.into() } - st.scan_cached_blocks(h2 + 1, 8); - // Total balance is value * number of blocks scanned (10). - assert_eq!(st.get_total_balance(account), (value * 10).unwrap()); + fn fvks_equal(a: &Self::Fvk, b: &Self::Fvk) -> bool { + a.to_bytes() == b.to_bytes() + } - // Spend still fails - assert_matches!( - st.propose_standard_transfer::( - account, - StandardFeeRule::Zip317, - NonZeroU32::new(10).unwrap(), - &to, - NonNegativeAmount::const_from_u64(70000), - None, - None, - ShieldedProtocol::Sapling - ), - Err(data_api::error::Error::InsufficientFunds { - available, - required - }) - if available == NonNegativeAmount::const_from_u64(50000) - && required == NonNegativeAmount::const_from_u64(80000) - ); - - // Mine block 11 so that the second note becomes verified - let (h11, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h11, 1); - - // Total balance is value * number of blocks scanned (11). - assert_eq!(st.get_total_balance(account), (value * 11).unwrap()); - // Spendable balance at 10 confirmations is value * 2. - assert_eq!(st.get_spendable_balance(account, 10), (value * 2).unwrap()); - assert_eq!( - st.get_pending_shielded_balance(account, 10), - (value * 9).unwrap() - ); - - // Should now be able to generate a proposal - let amount_sent = NonNegativeAmount::from_u64(70000).unwrap(); - let min_confirmations = NonZeroU32::new(10).unwrap(); - let proposal = st - .propose_standard_transfer::( - account, - StandardFeeRule::Zip317, - min_confirmations, - &to, - amount_sent, - None, - None, - ShieldedProtocol::Sapling, - ) - .unwrap(); - - // Executing the proposal should succeed - let txid = st - .create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal) - .unwrap()[0]; - - let (h, _) = st.generate_next_block_including(txid); - st.scan_cached_blocks(h, 1); - - // TODO: send to an account so that we can check its balance. - assert_eq!( - st.get_total_balance(account), - ((value * 11).unwrap() - - (amount_sent + NonNegativeAmount::from_u64(10000).unwrap()).unwrap()) - .unwrap() - ); - } + fn empty_tree_leaf() -> Self::MerkleTreeHash { + sapling::Node::empty_leaf() + } - #[test] - fn spend_fails_on_locked_notes() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // TODO: This test was originally written to use the pre-zip-313 fee rule - // and has not yet been updated. - #[allow(deprecated)] - let fee_rule = StandardFeeRule::PreZip313; - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::const_from_u64(50000); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h1, 1); - - // Spendable balance matches total balance at 1 confirmation. - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - // Send some of the funds to another address, but don't mine the tx. - let extsk2 = ExtendedSpendingKey::master(&[]); - let to = extsk2.default_address().1.into(); - let min_confirmations = NonZeroU32::new(1).unwrap(); - let proposal = st - .propose_standard_transfer::( - account, - fee_rule, - min_confirmations, - &to, - NonNegativeAmount::const_from_u64(15000), - None, - None, - ShieldedProtocol::Sapling, - ) - .unwrap(); + fn empty_tree_root(level: Level) -> Self::MerkleTreeHash { + sapling::Node::empty_root(level) + } - // Executing the proposal should succeed - assert_matches!( - st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal,), - Ok(txids) if txids.len() == 1 - ); + fn put_subtree_roots( + st: &mut TestState, + start_index: u64, + roots: &[CommitmentTreeRoot], + ) -> Result<(), ShardTreeError> { + st.wallet_mut() + .put_sapling_subtree_roots(start_index, roots) + } - // A second proposal fails because there are no usable notes - assert_matches!( - st.propose_standard_transfer::( - account, - fee_rule, - NonZeroU32::new(1).unwrap(), - &to, - NonNegativeAmount::const_from_u64(2000), - None, - None, - ShieldedProtocol::Sapling - ), - Err(data_api::error::Error::InsufficientFunds { - available, - required - }) - if available == NonNegativeAmount::ZERO && required == NonNegativeAmount::const_from_u64(12000) - ); - - // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) - // until just before the first transaction expires - for i in 1..42 { - st.generate_next_block( - &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), - AddressType::DefaultExternal, - value, - ); + fn next_subtree_index(s: &WalletSummary) -> u64 { + s.next_sapling_subtree_index() } - st.scan_cached_blocks(h1 + 1, 41); - // Second proposal still fails - assert_matches!( - st.propose_standard_transfer::( + fn select_spendable_notes( + st: &TestState, + account: AccountId, + target_value: NonNegativeAmount, + anchor_height: BlockHeight, + exclude: &[ReceivedNoteId], + ) -> Result>, SqliteClientError> { + select_spendable_sapling_notes( + &st.wallet().conn, + &st.wallet().params, account, - fee_rule, - NonZeroU32::new(1).unwrap(), - &to, - NonNegativeAmount::const_from_u64(2000), - None, - None, - ShieldedProtocol::Sapling - ), - Err(data_api::error::Error::InsufficientFunds { - available, - required - }) - if available == NonNegativeAmount::ZERO && required == NonNegativeAmount::const_from_u64(12000) - ); - - // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires - let (h43, _, _) = st.generate_next_block( - &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), - AddressType::DefaultExternal, - value, - ); - st.scan_cached_blocks(h43, 1); - - // Spendable balance matches total balance at 1 confirmation. - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - // Second spend should now succeed - let amount_sent2 = NonNegativeAmount::const_from_u64(2000); - let min_confirmations = NonZeroU32::new(1).unwrap(); - let proposal = st - .propose_standard_transfer::( - account, - fee_rule, - min_confirmations, - &to, - amount_sent2, - None, - None, - ShieldedProtocol::Sapling, + target_value, + anchor_height, + exclude, ) - .unwrap(); - - let txid2 = st - .create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal) - .unwrap()[0]; - - let (h, _) = st.generate_next_block_including(txid2); - st.scan_cached_blocks(h, 1); + } - // TODO: send to an account so that we can check its balance. - assert_eq!( - st.get_total_balance(account), - (value - (amount_sent2 + NonNegativeAmount::from_u64(10000).unwrap()).unwrap()) - .unwrap() - ); - } + fn decrypted_pool_outputs_count(d_tx: &DecryptedTransaction<'_, AccountId>) -> usize { + d_tx.sapling_outputs().len() + } - #[test] - fn ovk_policy_prevents_recovery_from_chain() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::const_from_u64(50000); - let (h1, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h1, 1); - - // Spendable balance matches total balance at 1 confirmation. - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - let extsk2 = ExtendedSpendingKey::master(&[]); - let addr2 = extsk2.default_address().1; - let to = addr2.into(); - - // TODO: This test was originally written to use the pre-zip-313 fee rule - // and has not yet been updated. - #[allow(deprecated)] - let fee_rule = StandardFeeRule::PreZip313; - - #[allow(clippy::type_complexity)] - let send_and_recover_with_policy = |st: &mut TestState, - ovk_policy| - -> Result< - Option<(sapling::Note, PaymentAddress, MemoBytes)>, - Error< - SqliteClientError, - commitment_tree::Error, - GreedyInputSelectorError, - Zip317FeeError, - >, - > { - let min_confirmations = NonZeroU32::new(1).unwrap(); - let proposal = st.propose_standard_transfer( - account, - fee_rule, - min_confirmations, - &to, - NonNegativeAmount::const_from_u64(15000), - None, - None, - ShieldedProtocol::Sapling, - )?; - - // Executing the proposal should succeed - let txid = st.create_proposed_transactions(&usk, ovk_policy, &proposal)?[0]; - - // Fetch the transaction from the database - let raw_tx: Vec<_> = st - .wallet() - .conn - .query_row( - "SELECT raw FROM transactions - WHERE txid = ?", - [txid.as_ref()], - |row| row.get(0), - ) - .unwrap(); - let tx = Transaction::read(&raw_tx[..], BranchId::Canopy).unwrap(); + fn with_decrypted_pool_memos( + d_tx: &DecryptedTransaction<'_, AccountId>, + mut f: impl FnMut(&MemoBytes), + ) { + for output in d_tx.sapling_outputs() { + f(output.memo()); + } + } + fn try_output_recovery( + st: &TestState, + height: BlockHeight, + tx: &Transaction, + fvk: &Self::Fvk, + ) -> Result, OutputRecoveryError> { for output in tx.sapling_bundle().unwrap().shielded_outputs() { // Find the output that decrypts with the external OVK let result = try_sapling_output_recovery( - &dfvk.to_ovk(Scope::External), + &fvk.to_ovk(Scope::External), output, - zip212_enforcement(&st.network(), h1), + zip212_enforcement(&st.network(), height), ); if result.is_some() { return Ok(result.map(|(note, addr, memo)| { ( - note, - addr, + Note::Sapling(note), + addr.into(), MemoBytes::from_bytes(&memo).expect("correct length"), ) })); @@ -1258,572 +509,107 @@ pub(crate) mod tests { } Ok(None) - }; - - // Send some of the funds to another address, keeping history. - // The recipient output is decryptable by the sender. - assert_matches!( - send_and_recover_with_policy(&mut st, OvkPolicy::Sender), - Ok(Some((_, recovered_to, _))) if recovered_to == addr2 - ); - - // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) - // so that the first transaction expires - for i in 1..=42 { - st.generate_next_block( - &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), - AddressType::DefaultExternal, - value, - ); } - st.scan_cached_blocks(h1 + 1, 42); - // Send the funds again, discarding history. - // Neither transaction output is decryptable by the sender. - assert_matches!( - send_and_recover_with_policy(&mut st, OvkPolicy::Discard), - Ok(None) - ); + fn received_note_count( + summary: &zcash_client_backend::data_api::chain::ScanSummary, + ) -> usize { + summary.received_sapling_note_count() + } } - #[test] - fn spend_succeeds_to_t_addr_zero_change() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::const_from_u64(60000); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h, 1); - - // Spendable balance matches total balance at 1 confirmation. - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - // TODO: This test was originally written to use the pre-zip-313 fee rule - // and has not yet been updated. - #[allow(deprecated)] - let fee_rule = StandardFeeRule::PreZip313; - - // TODO: generate_next_block_from_tx does not currently support transparent outputs. - let to = TransparentAddress::PublicKeyHash([7; 20]).into(); - let min_confirmations = NonZeroU32::new(1).unwrap(); - let proposal = st - .propose_standard_transfer::( - account, - fee_rule, - min_confirmations, - &to, - NonNegativeAmount::const_from_u64(50000), - None, - None, - ShieldedProtocol::Sapling, - ) - .unwrap(); - - // Executing the proposal should succeed - assert_matches!( - st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal), - Ok(txids) if txids.len() == 1 - ); + pub(crate) fn test_prover() -> impl SpendProver + OutputProver { + LocalTxProver::bundled() } #[test] - fn change_note_spends_succeed() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet in a single note owned by the internal spending key - let value = NonNegativeAmount::const_from_u64(60000); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::Internal, value); - st.scan_cached_blocks(h, 1); - - // Spendable balance matches total balance at 1 confirmation. - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - - // Value is considered pending at 10 confirmations. - assert_eq!(st.get_pending_shielded_balance(account, 10), value); - assert_eq!( - st.get_spendable_balance(account, 10), - NonNegativeAmount::ZERO - ); - - let change_note_scope = st.wallet().conn.query_row( - "SELECT recipient_key_scope - FROM sapling_received_notes - WHERE value = ?", - params![u64::from(value)], - |row| Ok(parse_scope(row.get(0)?)), - ); - assert_matches!(change_note_scope, Ok(Some(Scope::Internal))); - - // TODO: This test was originally written to use the pre-zip-313 fee rule - // and has not yet been updated. - #[allow(deprecated)] - let fee_rule = StandardFeeRule::PreZip313; - - // TODO: generate_next_block_from_tx does not currently support transparent outputs. - let to = TransparentAddress::PublicKeyHash([7; 20]).into(); - let min_confirmations = NonZeroU32::new(1).unwrap(); - let proposal = st - .propose_standard_transfer::( - account, - fee_rule, - min_confirmations, - &to, - NonNegativeAmount::const_from_u64(50000), - None, - None, - ShieldedProtocol::Sapling, - ) - .unwrap(); - - // Executing the proposal should succeed - assert_matches!( - st.create_proposed_transactions::(&usk, OvkPolicy::Sender, &proposal), - Ok(txids) if txids.len() == 1 - ); + fn send_single_step_proposed_transfer() { + testing::pool::send_single_step_proposed_transfer::() } #[test] - fn external_address_change_spends_detected_in_restore_from_seed() { - let mut st = TestBuilder::new().with_block_cache().build(); - - // Add two accounts to the wallet. - let seed = Secret::new([0u8; 32].to_vec()); - let birthday = AccountBirthday::from_sapling_activation(&st.network()); - let (account, usk) = st - .wallet_mut() - .create_account(&seed, birthday.clone()) - .unwrap(); - let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); - - let (account2, usk2) = st - .wallet_mut() - .create_account(&seed, birthday.clone()) - .unwrap(); - let dfvk2 = usk2.sapling().to_diversifiable_full_viewing_key(); - - // Add funds to the wallet in a single note - let value = NonNegativeAmount::from_u64(100000).unwrap(); - let (h, _, _) = st.generate_next_block(&dfvk, AddressType::DefaultExternal, value); - st.scan_cached_blocks(h, 1); - - // Spendable balance matches total balance - assert_eq!(st.get_total_balance(account), value); - assert_eq!(st.get_spendable_balance(account, 1), value); - assert_eq!(st.get_total_balance(account2), NonNegativeAmount::ZERO); - - let amount_sent = NonNegativeAmount::from_u64(20000).unwrap(); - let amount_legacy_change = NonNegativeAmount::from_u64(30000).unwrap(); - let addr = dfvk.default_address().1; - let addr2 = dfvk2.default_address().1; - let req = TransactionRequest::new(vec![ - // payment to an external recipient - Payment { - recipient_address: Address::Sapling(addr2), - amount: amount_sent, - memo: None, - label: None, - message: None, - other_params: vec![], - }, - // payment back to the originating wallet, simulating legacy change - Payment { - recipient_address: Address::Sapling(addr), - amount: amount_legacy_change, - memo: None, - label: None, - message: None, - other_params: vec![], - }, - ]) - .unwrap(); - - #[allow(deprecated)] - let fee_rule = FixedFeeRule::standard(); - let input_selector = GreedyInputSelector::new( - fixed::SingleOutputChangeStrategy::new(fee_rule, None, ShieldedProtocol::Sapling), - DustOutputPolicy::default(), - ); - - let txid = st - .spend( - &input_selector, - &usk, - req, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - ) - .unwrap()[0]; - - let amount_left = (value - (amount_sent + fee_rule.fixed_fee()).unwrap()).unwrap(); - let pending_change = (amount_left - amount_legacy_change).unwrap(); - - // The "legacy change" is not counted by get_pending_change(). - assert_eq!(st.get_pending_change(account, 1), pending_change); - // We spent the only note so we only have pending change. - assert_eq!(st.get_total_balance(account), pending_change); - - let (h, _) = st.generate_next_block_including(txid); - st.scan_cached_blocks(h, 1); - - assert_eq!(st.get_total_balance(account2), amount_sent,); - assert_eq!(st.get_total_balance(account), amount_left); - - st.reset(); - - // Account creation and DFVK derivation should be deterministic. - let (_, restored_usk) = st - .wallet_mut() - .create_account(&seed, birthday.clone()) - .unwrap(); - assert_eq!( - restored_usk - .sapling() - .to_diversifiable_full_viewing_key() - .to_bytes(), - dfvk.to_bytes() - ); - - let (_, restored_usk2) = st.wallet_mut().create_account(&seed, birthday).unwrap(); - assert_eq!( - restored_usk2 - .sapling() - .to_diversifiable_full_viewing_key() - .to_bytes(), - dfvk2.to_bytes() - ); - - st.scan_cached_blocks(st.sapling_activation_height(), 2); - - assert_eq!(st.get_total_balance(account2), amount_sent,); - assert_eq!(st.get_total_balance(account), amount_left); + #[cfg(feature = "transparent-inputs")] + fn send_multi_step_proposed_transfer() { + testing::pool::send_multi_step_proposed_transfer::() } #[test] - fn zip317_spend() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Add funds to the wallet - let (h1, _, _) = st.generate_next_block( - &dfvk, - AddressType::Internal, - NonNegativeAmount::const_from_u64(50000), - ); - - // Add 10 dust notes to the wallet - for _ in 1..=10 { - st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(1000), - ); - } + #[allow(deprecated)] + fn create_to_address_fails_on_incorrect_usk() { + testing::pool::create_to_address_fails_on_incorrect_usk::() + } - st.scan_cached_blocks(h1, 11); - - // Spendable balance matches total balance - let total = NonNegativeAmount::const_from_u64(60000); - assert_eq!(st.get_total_balance(account), total); - assert_eq!(st.get_spendable_balance(account, 1), total); - - let input_selector = - input_selector(StandardFeeRule::Zip317, None, ShieldedProtocol::Sapling); - - // This first request will fail due to insufficient non-dust funds - let req = TransactionRequest::new(vec![Payment { - recipient_address: Address::Sapling(dfvk.default_address().1), - amount: NonNegativeAmount::const_from_u64(50000), - memo: None, - label: None, - message: None, - other_params: vec![], - }]) - .unwrap(); - - assert_matches!( - st.spend( - &input_selector, - &usk, - req, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - ), - Err(Error::InsufficientFunds { available, required }) - if available == NonNegativeAmount::const_from_u64(51000) - && required == NonNegativeAmount::const_from_u64(60000) - ); - - // This request will succeed, spending a single dust input to pay the 10000 - // ZAT fee in addition to the 41000 ZAT output to the recipient - let req = TransactionRequest::new(vec![Payment { - recipient_address: Address::Sapling(dfvk.default_address().1), - amount: NonNegativeAmount::const_from_u64(41000), - memo: None, - label: None, - message: None, - other_params: vec![], - }]) - .unwrap(); - - let txid = st - .spend( - &input_selector, - &usk, - req, - OvkPolicy::Sender, - NonZeroU32::new(1).unwrap(), - ) - .unwrap()[0]; - - let (h, _) = st.generate_next_block_including(txid); - st.scan_cached_blocks(h, 1); - - // TODO: send to an account so that we can check its balance. - // We sent back to the same account so the amount_sent should be included - // in the total balance. - assert_eq!( - st.get_total_balance(account), - (total - NonNegativeAmount::const_from_u64(10000)).unwrap() - ); + #[test] + #[allow(deprecated)] + fn proposal_fails_with_no_blocks() { + testing::pool::proposal_fails_with_no_blocks::() } #[test] - #[cfg(feature = "transparent-inputs")] - fn shield_transparent() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account_id, usk, _) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - let uaddr = st - .wallet() - .get_current_address(account_id) - .unwrap() - .unwrap(); - let taddr = uaddr.transparent().unwrap(); - - // Ensure that the wallet has at least one block - let (h, _, _) = st.generate_next_block( - &dfvk, - AddressType::Internal, - NonNegativeAmount::const_from_u64(50000), - ); - st.scan_cached_blocks(h, 1); - - let utxo = WalletTransparentOutput::from_parts( - OutPoint::new([1u8; 32], 1), - TxOut { - value: NonNegativeAmount::const_from_u64(10000), - script_pubkey: taddr.script(), - }, - h, - ) - .unwrap(); - - let res0 = st.wallet_mut().put_received_transparent_utxo(&utxo); - assert!(matches!(res0, Ok(_))); - - // TODO: This test was originally written to use the pre-zip-313 fee rule - // and has not yet been updated. - #[allow(deprecated)] - let fee_rule = StandardFeeRule::PreZip313; - - let input_selector = GreedyInputSelector::new( - standard::SingleOutputChangeStrategy::new(fee_rule, None, ShieldedProtocol::Sapling), - DustOutputPolicy::default(), - ); - - assert_matches!( - st.shield_transparent_funds( - &input_selector, - NonNegativeAmount::from_u64(10000).unwrap(), - &usk, - &[*taddr], - 1 - ), - Ok(_) - ); + fn spend_fails_on_unverified_notes() { + testing::pool::spend_fails_on_unverified_notes::() } #[test] - fn birthday_in_anchor_shard() { - let (mut st, dfvk, birthday, _) = test_with_canopy_birthday(); - - // Set up the following situation: - // - // |<------ 500 ------->|<--- 10 --->|<--- 10 --->| - // last_shard_start wallet_birthday received_tx anchor_height - // - // Set up some shard root history before the wallet birthday. - let prev_shard_start = birthday.height() - 500; - st.wallet_mut() - .put_sapling_subtree_roots( - 0, - &[CommitmentTreeRoot::from_parts( - prev_shard_start, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - )], - ) - .unwrap(); + fn spend_fails_on_locked_notes() { + testing::pool::spend_fails_on_locked_notes::() + } - let received_tx_height = birthday.height() + 10; + #[test] + fn ovk_policy_prevents_recovery_from_chain() { + testing::pool::ovk_policy_prevents_recovery_from_chain::() + } - let initial_sapling_tree_size = - u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) - .try_into() - .unwrap(); - - // Generate 9 blocks that have no value for us, starting at the birthday height. - let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); - let not_our_value = NonNegativeAmount::const_from_u64(10000); - st.generate_block_at( - birthday.height(), - BlockHash([0; 32]), - ¬_our_key, - AddressType::DefaultExternal, - not_our_value, - initial_sapling_tree_size, - ); - for _ in 1..9 { - st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); - } + #[test] + fn spend_succeeds_to_t_addr_zero_change() { + testing::pool::spend_succeeds_to_t_addr_zero_change::() + } - // Now, generate a block that belongs to our wallet - st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(500000), - ); + #[test] + fn change_note_spends_succeed() { + testing::pool::change_note_spends_succeed::() + } - // Generate some more blocks to get above our anchor height - for _ in 0..15 { - st.generate_next_block(¬_our_key, AddressType::DefaultExternal, not_our_value); - } + #[test] + fn external_address_change_spends_detected_in_restore_from_seed() { + testing::pool::external_address_change_spends_detected_in_restore_from_seed::< + SaplingPoolTester, + >() + } - // Scan a block range that includes our received note, but skips some blocks we need to - // make it spendable. - st.scan_cached_blocks(birthday.height() + 5, 20); - - // Verify that the received note is not considered spendable - let account = st.test_account().unwrap(); - let spendable = select_spendable_sapling_notes( - &st.wallet().conn, - &st.wallet().params, - account.0, - NonNegativeAmount::const_from_u64(300000), - received_tx_height + 10, - &[], - ) - .unwrap(); + #[test] + fn zip317_spend() { + testing::pool::zip317_spend::() + } - assert_eq!(spendable.len(), 0); + #[test] + #[cfg(feature = "transparent-inputs")] + fn shield_transparent() { + testing::pool::shield_transparent::() + } - // Scan the blocks we skipped - st.scan_cached_blocks(birthday.height(), 5); + // FIXME: This requires fixes to the test framework. + #[test] + #[cfg(feature = "orchard")] + fn birthday_in_anchor_shard() { + testing::pool::birthday_in_anchor_shard::() + } - // Verify that the received note is now considered spendable - let spendable = select_spendable_sapling_notes( - &st.wallet().conn, - &st.wallet().params, - account.0, - NonNegativeAmount::const_from_u64(300000), - received_tx_height + 10, - &[], - ) - .unwrap(); + #[test] + fn checkpoint_gaps() { + testing::pool::checkpoint_gaps::() + } - assert_eq!(spendable.len(), 1); + #[test] + fn scan_cached_blocks_detects_spends_out_of_order() { + testing::pool::scan_cached_blocks_detects_spends_out_of_order::() } #[test] - fn checkpoint_gaps() { - let mut st = TestBuilder::new() - .with_block_cache() - .with_test_account(AccountBirthday::from_sapling_activation) - .build(); - - let (account, usk, birthday) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); - - // Generate a block with funds belonging to our wallet. - st.generate_next_block( - &dfvk, - AddressType::DefaultExternal, - NonNegativeAmount::const_from_u64(500000), - ); - st.scan_cached_blocks(birthday.height(), 1); - - // Create a gap of 10 blocks having no shielded outputs, then add a block that doesn't - // belong to us so that we can get a checkpoint in the tree. - let not_our_key = ExtendedSpendingKey::master(&[]).to_diversifiable_full_viewing_key(); - let not_our_value = NonNegativeAmount::const_from_u64(10000); - st.generate_block_at( - birthday.height() + 10, - BlockHash([0; 32]), - ¬_our_key, - AddressType::DefaultExternal, - not_our_value, - st.latest_cached_block().unwrap().2, - ); - - // Scan the block - st.scan_cached_blocks(birthday.height() + 10, 1); - - // Fake that everything has been scanned - st.wallet() - .conn - .execute_batch("UPDATE scan_queue SET priority = 10") - .unwrap(); - - // Verify that our note is considered spendable - let spendable = select_spendable_sapling_notes( - &st.wallet().conn, - &st.wallet().params, - account, - NonNegativeAmount::const_from_u64(300000), - birthday.height() + 5, - &[], - ) - .unwrap(); - assert_eq!(spendable.len(), 1); - - // Attempt to spend the note with 5 confirmations - let to = not_our_key.default_address().1.into(); - assert_matches!( - st.create_spend_to_address( - &usk, - &to, - NonNegativeAmount::const_from_u64(10000), - None, - OvkPolicy::Sender, - NonZeroU32::new(5).unwrap(), - None, - ShieldedProtocol::Sapling - ), - Ok(_) - ); + #[cfg(feature = "orchard")] + fn cross_pool_exchange() { + use crate::wallet::orchard::tests::OrchardPoolTester; + + testing::pool::cross_pool_exchange::() } } diff --git a/zcash_client_sqlite/src/wallet/scanning.rs b/zcash_client_sqlite/src/wallet/scanning.rs index 59458e06e0..11f1a9d323 100644 --- a/zcash_client_sqlite/src/wallet/scanning.rs +++ b/zcash_client_sqlite/src/wallet/scanning.rs @@ -1,3 +1,4 @@ +use incrementalmerkletree::{Address, Position}; use rusqlite::{self, named_params, types::Value, OptionalExtension}; use shardtree::error::ShardTreeError; use std::cmp::{max, min}; @@ -6,14 +7,14 @@ use std::ops::Range; use std::rc::Rc; use tracing::{debug, trace}; -use incrementalmerkletree::{Address, Position}; -use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; - -use zcash_client_backend::data_api::{ - scanning::{spanning_tree::SpanningTree, ScanPriority, ScanRange}, - SAPLING_SHARD_HEIGHT, +use zcash_client_backend::{ + data_api::{ + scanning::{spanning_tree::SpanningTree, ScanPriority, ScanRange}, + SAPLING_SHARD_HEIGHT, + }, + ShieldedProtocol, }; -use zcash_protocol::{PoolType, ShieldedProtocol}; +use zcash_primitives::consensus::{self, BlockHeight, NetworkUpgrade}; use crate::{ error::SqliteClientError, @@ -23,6 +24,12 @@ use crate::{ use super::wallet_birthday; +#[cfg(feature = "orchard")] +use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; + +#[cfg(not(feature = "orchard"))] +use zcash_client_backend::PoolType; + pub(crate) fn priority_code(priority: &ScanPriority) -> i64 { use ScanPriority::*; match priority { @@ -301,6 +308,7 @@ pub(crate) fn scan_complete( wallet_note_positions: &[(ShieldedProtocol, Position)], ) -> Result<(), SqliteClientError> { // Read the wallet birthday (if known). + // TODO: use per-pool birthdays? let wallet_birthday = wallet_birthday(conn)?; // Determine the range of block heights for which we will be updating the scan queue. @@ -310,6 +318,8 @@ pub(crate) fn scan_complete( // the note commitment tree subtrees containing the positions of the discovered notes. // We will query by subtree index to find these bounds. let mut required_sapling_subtrees = BTreeSet::new(); + #[cfg(feature = "orchard")] + let mut required_orchard_subtrees = BTreeSet::new(); for (protocol, position) in wallet_note_positions { match protocol { ShieldedProtocol::Sapling => { @@ -318,6 +328,12 @@ pub(crate) fn scan_complete( ); } ShieldedProtocol::Orchard => { + #[cfg(feature = "orchard")] + required_orchard_subtrees.insert( + Address::above_position(ORCHARD_SHARD_HEIGHT.into(), *position).index(), + ); + + #[cfg(not(feature = "orchard"))] return Err(SqliteClientError::UnsupportedPoolType(PoolType::Shielded( *protocol, ))); @@ -325,14 +341,28 @@ pub(crate) fn scan_complete( } } - extend_range( + let extended_range = extend_range( conn, &range, required_sapling_subtrees, SAPLING_TABLES_PREFIX, params.activation_height(NetworkUpgrade::Sapling), wallet_birthday, + )?; + + #[cfg(feature = "orchard")] + let extended_range = extend_range( + conn, + extended_range.as_ref().unwrap_or(&range), + required_orchard_subtrees, + ORCHARD_TABLES_PREFIX, + params.activation_height(NetworkUpgrade::Nu5), + wallet_birthday, )? + .or(extended_range); + + #[allow(clippy::let_and_return)] + extended_range }; let query_range = extended_range.clone().unwrap_or_else(|| range.clone()); @@ -351,15 +381,12 @@ pub(crate) fn scan_complete( .map(|extended| ScanRange::from_parts(range.end..extended.end, ScanPriority::FoundNote)) .filter(|range| !range.is_empty()); - replace_queue_entries::( - conn, - &query_range, - Some(scanned) - .into_iter() - .chain(extended_before) - .chain(extended_after), - false, - )?; + let replacement = Some(scanned) + .into_iter() + .chain(extended_before) + .chain(extended_after); + + replace_queue_entries::(conn, &query_range, replacement, false)?; Ok(()) } @@ -415,9 +442,21 @@ pub(crate) fn update_chain_tip( // `ScanRange` uses an exclusive upper bound. let chain_end = new_tip + 1; - // Read the maximum height from the shards table. + // Read the maximum height from each of the shards tables. The minimum of the two + // gives the start of a height range that covers the last incomplete shard of both the + // Sapling and Orchard pools. let sapling_shard_tip = tip_shard_end_height(conn, SAPLING_TABLES_PREFIX)?; - + #[cfg(feature = "orchard")] + let orchard_shard_tip = tip_shard_end_height(conn, ORCHARD_TABLES_PREFIX)?; + + #[cfg(feature = "orchard")] + let min_shard_tip = match (sapling_shard_tip, orchard_shard_tip) { + (None, None) => None, + (None, Some(o)) => Some(o), + (Some(s), None) => Some(s), + (Some(s), Some(o)) => Some(std::cmp::min(s, o)), + }; + #[cfg(not(feature = "orchard"))] let min_shard_tip = sapling_shard_tip; // Create a scanning range for the fragment of the last shard leading up to new tip. @@ -542,29 +581,52 @@ pub(crate) fn update_chain_tip( pub(crate) mod tests { use incrementalmerkletree::{frontier::Frontier, Hashable, Level, Position}; - use sapling::{zip32::DiversifiableFullViewingKey, Node}; + use sapling::Node; use secrecy::SecretVec; use zcash_client_backend::data_api::{ chain::CommitmentTreeRoot, scanning::{spanning_tree::testing::scan_range, ScanPriority}, - AccountBirthday, Ratio, WalletCommitmentTrees, WalletRead, WalletWrite, - SAPLING_SHARD_HEIGHT, + AccountBirthday, Ratio, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; use zcash_primitives::{ block::BlockHash, consensus::{BlockHeight, NetworkUpgrade, Parameters}, transaction::components::amount::NonNegativeAmount, }; + use zcash_protocol::ShieldedProtocol; use crate::{ error::SqliteClientError, - testing::{AddressType, BlockCache, TestBuilder, TestState}, - wallet::scanning::{insert_queue_entries, replace_queue_entries, suggest_scan_ranges}, + testing::{pool::ShieldedPoolTester, AddressType, BlockCache, TestBuilder, TestState}, + wallet::{ + sapling::tests::SaplingPoolTester, + scanning::{insert_queue_entries, replace_queue_entries, suggest_scan_ranges}, + }, VERIFY_LOOKAHEAD, }; + #[cfg(feature = "orchard")] + use { + crate::wallet::orchard::tests::OrchardPoolTester, orchard::tree::MerkleHashOrchard, + zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT, + }; + + // FIXME: This requires fixes to the test framework. + #[test] + #[cfg(feature = "orchard")] + fn sapling_scan_complete() { + scan_complete::(); + } + #[test] - fn scan_complete() { + #[cfg(feature = "orchard")] + fn orchard_scan_complete() { + scan_complete::(); + } + + // FIXME: This requires fixes to the test framework. + #[allow(dead_code)] + fn scan_complete() { use ScanPriority::*; let mut st = TestBuilder::new() @@ -572,26 +634,27 @@ pub(crate) mod tests { .with_test_account(AccountBirthday::from_sapling_activation) .build(); - let dfvk = st.test_account_sapling().unwrap(); + let dfvk = T::test_account_fvk(&st); let sapling_activation_height = st.sapling_activation_height(); assert_matches!( // In the following, we don't care what the root hashes are, they just need to be // distinct. - st.wallet_mut().put_sapling_subtree_roots( + T::put_subtree_roots( + &mut st, 0, &[ CommitmentTreeRoot::from_parts( sapling_activation_height + 100, - Node::empty_root(Level::from(0)) + T::empty_tree_root(Level::from(0)) ), CommitmentTreeRoot::from_parts( sapling_activation_height + 200, - Node::empty_root(Level::from(1)) + T::empty_tree_root(Level::from(1)) ), CommitmentTreeRoot::from_parts( sapling_activation_height + 300, - Node::empty_root(Level::from(2)) + T::empty_tree_root(Level::from(2)) ), ] ), @@ -601,7 +664,9 @@ pub(crate) mod tests { // We'll start inserting leaf notes 5 notes after the end of the third subtree, with a gap // of 10 blocks. After `scan_cached_blocks`, the scan queue should have a requested scan // range of 300..310 with `FoundNote` priority, 310..320 with `Scanned` priority. + // We set both Sapling and Orchard to the same initial tree size for simplicity. let initial_sapling_tree_size = (0x1 << 16) * 3 + 5; + let initial_orchard_tree_size = (0x1 << 16) * 3 + 5; let initial_height = sapling_activation_height + 310; let value = NonNegativeAmount::const_from_u64(50000); @@ -612,6 +677,7 @@ pub(crate) mod tests { AddressType::DefaultExternal, value, initial_sapling_tree_size, + initial_orchard_tree_size, ); for _ in 1..=10 { @@ -684,52 +750,71 @@ pub(crate) mod tests { st.wallet() .get_wallet_summary(0) .unwrap() - .map(|s| s.next_sapling_subtree_index()), + .map(|s| T::next_subtree_index(&s)), Some(2), ); } - pub(crate) fn test_with_canopy_birthday() -> ( - TestState, - DiversifiableFullViewingKey, - AccountBirthday, - u32, - ) { + pub(crate) fn test_with_nu5_birthday_offset( + offset: u32, + ) -> (TestState, T::Fvk, AccountBirthday, u32) { let st = TestBuilder::new() .with_block_cache() .with_test_account(|network| { - // We use Canopy activation as an arbitrary birthday height that's greater than Sapling - // activation. We set the Canopy frontier to be 1234 notes into the second shard. - let birthday_height = network.activation_height(NetworkUpgrade::Canopy).unwrap(); + // We set the Sapling and Orchard frontiers at the birthday height to be + // 1234 notes into the second shard. + let birthday_height = + network.activation_height(NetworkUpgrade::Nu5).unwrap() + offset; let frontier_position = Position::from((0x1 << 16) + 1234); - let frontier = Frontier::from_parts( + let sapling_frontier = Frontier::from_parts( frontier_position, Node::empty_leaf(), vec![Node::empty_leaf(); frontier_position.past_ommer_count().into()], ) .unwrap(); + #[cfg(feature = "orchard")] + let orchard_frontier = Frontier::from_parts( + frontier_position, + MerkleHashOrchard::empty_leaf(), + vec![ + MerkleHashOrchard::empty_leaf(); + frontier_position.past_ommer_count().into() + ], + ) + .unwrap(); AccountBirthday::from_parts( birthday_height, - frontier, + sapling_frontier, #[cfg(feature = "orchard")] - Frontier::empty(), + orchard_frontier, None, ) }) .build(); let (_, _, birthday) = st.test_account().unwrap(); - let dfvk = st.test_account_sapling().unwrap(); + let dfvk = T::test_account_fvk(&st); let sap_active = st.sapling_activation_height(); (st, dfvk, birthday, sap_active.into()) } #[test] - fn create_account_creates_ignored_range() { + fn sapling_create_account_creates_ignored_range() { + create_account_creates_ignored_range::(); + } + + #[cfg(feature = "orchard")] + #[test] + fn orchard_create_account_creates_ignored_range() { + create_account_creates_ignored_range::(); + } + + fn create_account_creates_ignored_range() { use ScanPriority::*; - let (st, _, birthday, sap_active) = test_with_canopy_birthday(); + // Use a non-zero birthday offset because Sapling and NU5 are activated at the same height. + let (st, _, birthday, sap_active) = test_with_nu5_birthday_offset::(76); let birthday_height = birthday.height().into(); let expected = vec![ @@ -785,10 +870,21 @@ pub(crate) mod tests { } #[test] - fn update_chain_tip_with_no_subtree_roots() { + fn sapling_update_chain_tip_with_no_subtree_roots() { + update_chain_tip_with_no_subtree_roots::(); + } + + #[cfg(feature = "orchard")] + #[test] + fn orchard_update_chain_tip_with_no_subtree_roots() { + update_chain_tip_with_no_subtree_roots::(); + } + + fn update_chain_tip_with_no_subtree_roots() { use ScanPriority::*; - let (mut st, _, birthday, sap_active) = test_with_canopy_birthday(); + // Use a non-zero birthday offset because Sapling and NU5 are activated at the same height. + let (mut st, _, birthday, sap_active) = test_with_nu5_birthday_offset::(76); // Set up the following situation: // @@ -816,10 +912,21 @@ pub(crate) mod tests { } #[test] - fn update_chain_tip_when_never_scanned() { + fn sapling_update_chain_tip_when_never_scanned() { + update_chain_tip_when_never_scanned::(); + } + + #[cfg(feature = "orchard")] + #[test] + fn orchard_update_chain_tip_when_never_scanned() { + update_chain_tip_when_never_scanned::(); + } + + fn update_chain_tip_when_never_scanned() { use ScanPriority::*; - let (mut st, _, birthday, sap_active) = test_with_canopy_birthday(); + // Use a non-zero birthday offset because Sapling and NU5 are activated at the same height. + let (mut st, _, birthday, sap_active) = test_with_nu5_birthday_offset::(76); // Set up the following situation: // @@ -830,16 +937,16 @@ pub(crate) mod tests { // Set up some shard root history before the wallet birthday. let last_shard_start = birthday.height() - 1000; - st.wallet_mut() - .put_sapling_subtree_roots( - 0, - &[CommitmentTreeRoot::from_parts( - last_shard_start, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - )], - ) - .unwrap(); + T::put_subtree_roots( + &mut st, + 0, + &[CommitmentTreeRoot::from_parts( + last_shard_start, + // fake a hash, the value doesn't matter + T::empty_tree_leaf(), + )], + ) + .unwrap(); // Update the chain tip. let tip_height = prior_tip_height + 500; @@ -860,12 +967,27 @@ pub(crate) mod tests { assert_eq!(actual, expected); } + // FIXME: This requires fixes to the test framework. #[test] - fn update_chain_tip_unstable_max_scanned() { + #[cfg(feature = "orchard")] + fn sapling_update_chain_tip_unstable_max_scanned() { + update_chain_tip_unstable_max_scanned::(); + } + + #[cfg(feature = "orchard")] + #[test] + fn orchard_update_chain_tip_unstable_max_scanned() { + update_chain_tip_unstable_max_scanned::(); + } + + // FIXME: This requires fixes to the test framework. + #[allow(dead_code)] + fn update_chain_tip_unstable_max_scanned() { use ScanPriority::*; + // Use a non-zero birthday offset because Sapling and NU5 are activated at the same height. // this birthday is 1234 notes into the second shard - let (mut st, dfvk, birthday, sap_active) = test_with_canopy_birthday(); + let (mut st, dfvk, birthday, sap_active) = test_with_nu5_birthday_offset::(76); // Set up the following situation: // @@ -877,16 +999,16 @@ pub(crate) mod tests { // Set up some shard root history before the wallet birthday. let initial_shard_end = birthday.height() - 1000; - st.wallet_mut() - .put_sapling_subtree_roots( - 0, - &[CommitmentTreeRoot::from_parts( - initial_shard_end, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - )], - ) - .unwrap(); + T::put_subtree_roots( + &mut st, + 0, + &[CommitmentTreeRoot::from_parts( + initial_shard_end, + // fake a hash, the value doesn't matter + T::empty_tree_leaf(), + )], + ) + .unwrap(); // Set up prior chain state. This simulates us having imported a wallet // with a birthday 520 blocks below the chain tip. @@ -902,6 +1024,23 @@ pub(crate) mod tests { assert_eq!(actual, expected); // Now, scan the max scanned block. + let initial_sapling_tree_size = birthday + .sapling_frontier() + .value() + .map(|f| u64::from(f.position() + 1)) + .unwrap_or(0) + .try_into() + .unwrap(); + #[cfg(feature = "orchard")] + let initial_orchard_tree_size = birthday + .orchard_frontier() + .value() + .map(|f| u64::from(f.position() + 1)) + .unwrap_or(0) + .try_into() + .unwrap(); + #[cfg(not(feature = "orchard"))] + let initial_orchard_tree_size = 0; st.generate_block_at( max_scanned, BlockHash([0u8; 32]), @@ -909,9 +1048,8 @@ pub(crate) mod tests { AddressType::DefaultExternal, // 1235 notes into into the second shard NonNegativeAmount::const_from_u64(10000), - u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) - .try_into() - .unwrap(), + initial_sapling_tree_size, + initial_orchard_tree_size, ); st.scan_cached_blocks(max_scanned, 1); @@ -929,16 +1067,16 @@ pub(crate) mod tests { // Now simulate shutting down, and then restarting 90 blocks later, after a shard // has been completed. let last_shard_start = prior_tip + 70; - st.wallet_mut() - .put_sapling_subtree_roots( - 0, - &[CommitmentTreeRoot::from_parts( - last_shard_start, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - )], - ) - .unwrap(); + T::put_subtree_roots( + &mut st, + 0, + &[CommitmentTreeRoot::from_parts( + last_shard_start, + // fake a hash, the value doesn't matter + T::empty_tree_leaf(), + )], + ) + .unwrap(); let new_tip = last_shard_start + 20; st.wallet_mut().update_chain_tip(new_tip).unwrap(); @@ -972,11 +1110,26 @@ pub(crate) mod tests { assert_eq!(actual, expected); } + // FIXME: This requires fixes to the test framework. + #[test] + #[cfg(feature = "orchard")] + fn sapling_update_chain_tip_stable_max_scanned() { + update_chain_tip_stable_max_scanned::(); + } + #[test] - fn update_chain_tip_stable_max_scanned() { + #[cfg(feature = "orchard")] + fn orchard_update_chain_tip_stable_max_scanned() { + update_chain_tip_stable_max_scanned::(); + } + + // FIXME: This requires fixes to the test framework. + #[allow(dead_code)] + fn update_chain_tip_stable_max_scanned() { use ScanPriority::*; - let (mut st, dfvk, birthday, sap_active) = test_with_canopy_birthday(); + // Use a non-zero birthday offset because Sapling and NU5 are activated at the same height. + let (mut st, dfvk, birthday, sap_active) = test_with_nu5_birthday_offset::(76); // Set up the following situation: // @@ -989,16 +1142,16 @@ pub(crate) mod tests { // Set up some shard root history before the wallet birthday. let second_to_last_shard_start = birthday.height() - 1000; - st.wallet_mut() - .put_sapling_subtree_roots( - 0, - &[CommitmentTreeRoot::from_parts( - second_to_last_shard_start, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - )], - ) - .unwrap(); + T::put_subtree_roots( + &mut st, + 0, + &[CommitmentTreeRoot::from_parts( + second_to_last_shard_start, + // fake a hash, the value doesn't matter + T::empty_tree_leaf(), + )], + ) + .unwrap(); // We have scan ranges and a subtree, but have scanned no blocks. let summary = st.get_wallet_summary(1); @@ -1018,43 +1171,62 @@ pub(crate) mod tests { assert_eq!(actual, expected); // Now, scan the max scanned block. + let initial_sapling_tree_size = birthday + .sapling_frontier() + .value() + .map(|f| u64::from(f.position() + 1)) + .unwrap_or(0) + .try_into() + .unwrap(); + #[cfg(feature = "orchard")] + let initial_orchard_tree_size = birthday + .orchard_frontier() + .value() + .map(|f| u64::from(f.position() + 1)) + .unwrap_or(0) + .try_into() + .unwrap(); + #[cfg(not(feature = "orchard"))] + let initial_orchard_tree_size = 0; st.generate_block_at( max_scanned, BlockHash([0u8; 32]), &dfvk, AddressType::DefaultExternal, NonNegativeAmount::const_from_u64(10000), - u64::from(birthday.sapling_frontier().value().unwrap().position() + 1) - .try_into() - .unwrap(), + initial_sapling_tree_size, + initial_orchard_tree_size, ); st.scan_cached_blocks(max_scanned, 1); // We have scanned a block, so we now have a starting tree position, 500 blocks above the // wallet birthday but before the end of the shard. let summary = st.get_wallet_summary(1); - assert_eq!( - summary.as_ref().map(|s| s.next_sapling_subtree_index()), - Some(0), - ); + assert_eq!(summary.as_ref().map(|s| T::next_subtree_index(s)), Some(0),); + + // Progress denominator depends on which pools are enabled (which changes the + // initial tree states in `test_with_nu5_birthday_offset`). + let expected_denom = 1 << SAPLING_SHARD_HEIGHT; + #[cfg(feature = "orchard")] + let expected_denom = expected_denom + (1 << ORCHARD_SHARD_HEIGHT); assert_eq!( summary.and_then(|s| s.scan_progress()), - Some(Ratio::new(1, 0x1 << SAPLING_SHARD_HEIGHT)) + Some(Ratio::new(1, expected_denom)) ); // Now simulate shutting down, and then restarting 70 blocks later, after a shard // has been completed. let last_shard_start = prior_tip + 50; - st.wallet_mut() - .put_sapling_subtree_roots( - 0, - &[CommitmentTreeRoot::from_parts( - last_shard_start, - // fake a hash, the value doesn't matter - Node::empty_leaf(), - )], - ) - .unwrap(); + T::put_subtree_roots( + &mut st, + 0, + &[CommitmentTreeRoot::from_parts( + last_shard_start, + // fake a hash, the value doesn't matter + T::empty_tree_leaf(), + )], + ) + .unwrap(); let new_tip = last_shard_start + 20; st.wallet_mut().update_chain_tip(new_tip).unwrap(); @@ -1077,10 +1249,18 @@ pub(crate) mod tests { // We've crossed a subtree boundary, and so still only have one scanned note but have two // shards worth of notes to scan. + let expected_denom = expected_denom + + match T::SHIELDED_PROTOCOL { + ShieldedProtocol::Sapling => 1 << SAPLING_SHARD_HEIGHT, + #[cfg(feature = "orchard")] + ShieldedProtocol::Orchard => 1 << ORCHARD_SHARD_HEIGHT, + #[cfg(not(feature = "orchard"))] + ShieldedProtocol::Orchard => unreachable!(), + }; let summary = st.get_wallet_summary(1); assert_eq!( summary.and_then(|s| s.scan_progress()), - Some(Ratio::new(1, 0x1 << (SAPLING_SHARD_HEIGHT + 1))) + Some(Ratio::new(1, expected_denom)) ); } diff --git a/zcash_extensions/Cargo.toml b/zcash_extensions/Cargo.toml index d1bb69c45d..009a16606a 100644 --- a/zcash_extensions/Cargo.toml +++ b/zcash_extensions/Cargo.toml @@ -16,7 +16,7 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] blake2b_simd.workspace = true -zcash_primitives = { workspace = true, features = ["zfuture" ] } +zcash_primitives.workspace = true [dev-dependencies] ff.workspace = true @@ -29,7 +29,6 @@ zcash_proofs.workspace = true [features] transparent-inputs = [] -unstable-nu6 = ["zcash_primitives/unstable-nu6"] [lib] bench = false diff --git a/zcash_extensions/src/lib.rs b/zcash_extensions/src/lib.rs index 2b88fa20be..5dc0c812cc 100644 --- a/zcash_extensions/src/lib.rs +++ b/zcash_extensions/src/lib.rs @@ -3,5 +3,10 @@ // Catch documentation errors caused by code changes. #![deny(rustdoc::broken_intra_doc_links)] +// For workspace compilation reasons, we have this crate in the workspace and just leave +// it empty if `zfuture` is not enabled. + +#[cfg(zcash_unstable = "zfuture")] pub mod consensus; +#[cfg(zcash_unstable = "zfuture")] pub mod transparent; diff --git a/zcash_extensions/src/transparent/demo.rs b/zcash_extensions/src/transparent/demo.rs index 6f26629932..40389cbb31 100644 --- a/zcash_extensions/src/transparent/demo.rs +++ b/zcash_extensions/src/transparent/demo.rs @@ -513,7 +513,7 @@ mod tests { NetworkUpgrade::Heartwood => Some(BlockHeight::from_u32(903_800)), NetworkUpgrade::Canopy => Some(BlockHeight::from_u32(1_028_500)), NetworkUpgrade::Nu5 => Some(BlockHeight::from_u32(1_200_000)), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] NetworkUpgrade::Nu6 => Some(BlockHeight::from_u32(1_300_000)), NetworkUpgrade::ZFuture => Some(BlockHeight::from_u32(1_400_000)), } diff --git a/zcash_keys/src/keys.rs b/zcash_keys/src/keys.rs index 8eefd34f0a..fd5be613b4 100644 --- a/zcash_keys/src/keys.rs +++ b/zcash_keys/src/keys.rs @@ -1549,7 +1549,7 @@ mod tests { #[cfg(any(feature = "orchard", feature = "sapling"))] { let uivk = uivk.expect("Orchard or Sapling ivk is present."); - let encoded = uivk.to_uivk().encode(&Network::Main); + let encoded = uivk.to_uivk().encode(&NetworkType::Main); // Test encoded form against known values; these test vectors contain Orchard receivers // that will be treated as unknown if the `orchard` feature is not enabled. diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 35723435aa..ffc62e8990 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -33,6 +33,9 @@ and this library adheres to Rust's notion of - `impl From for orchard::NoteValue` - The `local_consensus` module and feature flag have been removed; use the module from the `zcash_protocol` crate instead. +- `unstable-nu6` and `zfuture` feature flags (use `--cfg zcash_unstable=\"nu6\"` + or `--cfg zcash_unstable=\"zfuture\"` in `RUSTFLAGS` and `RUSTDOCFLAGS` + instead). ## [0.14.0] - 2024-03-01 ### Added diff --git a/zcash_primitives/Cargo.toml b/zcash_primitives/Cargo.toml index c349215600..7bbc788a0a 100644 --- a/zcash_primitives/Cargo.toml +++ b/zcash_primitives/Cargo.toml @@ -123,16 +123,6 @@ test-dependencies = [ "zcash_protocol/test-dependencies", ] -#! ### Experimental features -#! -#! ⚠️ Enabling these features will likely make your code incompatible with current Zcash -#! consensus rules! - -## Exposes the in-development NU6 features. -unstable-nu6 = ["zcash_protocol/unstable-nu6"] - -## Exposes early in-development features that are not yet planned for any network upgrade. -zfuture = ["zcash_protocol/zfuture"] [lib] bench = false diff --git a/zcash_primitives/src/lib.rs b/zcash_primitives/src/lib.rs index 37c37d653a..f514fb1777 100644 --- a/zcash_primitives/src/lib.rs +++ b/zcash_primitives/src/lib.rs @@ -26,6 +26,6 @@ pub mod merkle_tree; use sapling; pub mod transaction; pub use zip32; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] pub mod extensions; pub mod zip339; diff --git a/zcash_primitives/src/transaction/builder.rs b/zcash_primitives/src/transaction/builder.rs index 2fc016bd8f..ef352d7e0f 100644 --- a/zcash_primitives/src/transaction/builder.rs +++ b/zcash_primitives/src/transaction/builder.rs @@ -35,7 +35,7 @@ use crate::transaction::components::transparent::builder::TransparentInputInfo; #[cfg(not(feature = "transparent-inputs"))] use std::convert::Infallible; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] use crate::{ extensions::transparent::{ExtensionTxBuilder, ToPayload}, transaction::{ @@ -100,7 +100,7 @@ pub enum Error { /// spend or output was added. OrchardBuilderNotAvailable, /// An error occurred in constructing the TZE parts of a transaction. - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TzeBuild(tze::builder::Error), } @@ -132,7 +132,7 @@ impl fmt::Display for Error { f, "Cannot create Orchard transactions without an Orchard anchor, or before NU5 activation" ), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] Error::TzeBuild(err) => err.fmt(f), } } @@ -284,9 +284,9 @@ pub struct Builder<'a, P, U: sapling::builder::ProverProgress> { // derivatives for proving and signing to complete transaction creation. sapling_asks: Vec, orchard_saks: Vec, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_builder: TzeBuilder<'a, TransactionData>, - #[cfg(not(feature = "zfuture"))] + #[cfg(not(zcash_unstable = "zfuture"))] tze_builder: std::marker::PhantomData<&'a ()>, progress_notifier: U, } @@ -369,9 +369,9 @@ impl<'a, P: consensus::Parameters> Builder<'a, P, ()> { orchard_builder, sapling_asks: vec![], orchard_saks: Vec::new(), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_builder: TzeBuilder::empty(), - #[cfg(not(feature = "zfuture"))] + #[cfg(not(zcash_unstable = "zfuture"))] tze_builder: std::marker::PhantomData, progress_notifier: (), } @@ -521,7 +521,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< .map_err(|_| BalanceError::Overflow) }, )?, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] self.tze_builder.value_balance()?, ]; @@ -577,7 +577,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< .map_err(FeeError::FeeRule) } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub fn get_fee_zfuture( &self, fee_rule: &FR, @@ -641,7 +641,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< /// /// Upon success, returns a tuple containing the final transaction, and the /// [`SaplingMetadata`] generated during the build process. - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub fn build_zfuture< R: RngCore + CryptoRng, SP: SpendProver, @@ -734,7 +734,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< None => (None, orchard::builder::BundleMetadata::empty()), }; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let (tze_bundle, tze_signers) = self.tze_builder.build(); let unauthed_tx: TransactionData = TransactionData { @@ -746,7 +746,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< sprout_bundle: None, sapling_bundle, orchard_bundle, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle, }; @@ -764,7 +764,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< ) }); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let tze_bundle = unauthed_tx .tze_bundle .clone() @@ -814,7 +814,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< sprout_bundle: unauthed_tx.sprout_bundle, sapling_bundle, orchard_bundle, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle, }; @@ -828,7 +828,7 @@ impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> Builder< } } -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] impl<'a, P: consensus::Parameters, U: sapling::builder::ProverProgress> ExtensionTxBuilder<'a> for Builder<'a, P, U> { @@ -934,7 +934,7 @@ mod tests { use super::{Builder, Error}; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] #[cfg(feature = "transparent-inputs")] use super::TzeBuilder; @@ -969,9 +969,9 @@ mod tests { expiry_height: sapling_activation_height + DEFAULT_TX_EXPIRY_DELTA, transparent_builder: TransparentBuilder::empty(), sapling_builder: None, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_builder: TzeBuilder::empty(), - #[cfg(not(feature = "zfuture"))] + #[cfg(not(zcash_unstable = "zfuture"))] tze_builder: std::marker::PhantomData, progress_notifier: (), orchard_builder: None, diff --git a/zcash_primitives/src/transaction/components.rs b/zcash_primitives/src/transaction/components.rs index bba5ddf951..f239b0ba01 100644 --- a/zcash_primitives/src/transaction/components.rs +++ b/zcash_primitives/src/transaction/components.rs @@ -16,6 +16,7 @@ pub mod orchard; pub mod sapling; pub mod sprout; pub mod transparent; +#[cfg(zcash_unstable = "zfuture")] pub mod tze; pub use self::{ @@ -25,7 +26,7 @@ pub use self::{ }; pub use crate::sapling::bundle::{OutputDescription, SpendDescription}; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] pub use self::tze::{TzeIn, TzeOut}; // π_A + π_B + π_C diff --git a/zcash_primitives/src/transaction/components/tze.rs b/zcash_primitives/src/transaction/components/tze.rs index 0e88d35d15..26e849cbc5 100644 --- a/zcash_primitives/src/transaction/components/tze.rs +++ b/zcash_primitives/src/transaction/components/tze.rs @@ -1,5 +1,4 @@ //! Structs representing the TZE components within Zcash transactions. -#![cfg(feature = "zfuture")] use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use std::convert::TryFrom; diff --git a/zcash_primitives/src/transaction/components/tze/builder.rs b/zcash_primitives/src/transaction/components/tze/builder.rs index ff8ca49bf4..974dac9ce8 100644 --- a/zcash_primitives/src/transaction/components/tze/builder.rs +++ b/zcash_primitives/src/transaction/components/tze/builder.rs @@ -1,5 +1,4 @@ //! Types and functions for building TZE transaction components -#![cfg(feature = "zfuture")] use std::fmt; diff --git a/zcash_primitives/src/transaction/fees.rs b/zcash_primitives/src/transaction/fees.rs index 313391be6e..0dc82acf2b 100644 --- a/zcash_primitives/src/transaction/fees.rs +++ b/zcash_primitives/src/transaction/fees.rs @@ -9,7 +9,7 @@ pub mod fixed; pub mod transparent; pub mod zip317; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] pub mod tze; /// A trait that represents the ability to compute the fees that must be paid @@ -37,7 +37,7 @@ pub trait FeeRule { /// A trait that represents the ability to compute the fees that must be paid by a transaction /// having a specified set of inputs and outputs, for use when experimenting with the TZE feature. -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] pub trait FutureFeeRule: FeeRule { /// Computes the total fee required for a transaction given the provided inputs and outputs. /// diff --git a/zcash_primitives/src/transaction/fees/fixed.rs b/zcash_primitives/src/transaction/fees/fixed.rs index 660eb5f781..1bb1b38527 100644 --- a/zcash_primitives/src/transaction/fees/fixed.rs +++ b/zcash_primitives/src/transaction/fees/fixed.rs @@ -4,7 +4,7 @@ use crate::{ transaction::fees::{transparent, zip317}, }; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] use crate::transaction::fees::tze; /// A fee rule that always returns a fixed fee, irrespective of the structure of @@ -62,7 +62,7 @@ impl super::FeeRule for FeeRule { } } -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] impl super::FutureFeeRule for FeeRule { fn fee_required_zfuture( &self, diff --git a/zcash_primitives/src/transaction/mod.rs b/zcash_primitives/src/transaction/mod.rs index 7e3802a18d..b7e9943295 100644 --- a/zcash_primitives/src/transaction/mod.rs +++ b/zcash_primitives/src/transaction/mod.rs @@ -38,7 +38,7 @@ use self::{ util::sha256d::{HashReader, HashWriter}, }; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] use self::components::tze::{self, TzeIn, TzeOut}; const OVERWINTER_VERSION_GROUP_ID: u32 = 0x03C48270; @@ -55,9 +55,9 @@ const V5_VERSION_GROUP_ID: u32 = 0x26A7270A; /// using these constants should be inspected, and use of these constants /// should be removed as appropriate in favor of the new consensus /// transaction version and group. -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZFUTURE_VERSION_GROUP_ID: u32 = 0xFFFFFFFF; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZFUTURE_TX_VERSION: u32 = 0x0000FFFF; /// The identifier for a Zcash transaction. @@ -131,7 +131,7 @@ pub enum TxVersion { Overwinter, Sapling, Zip225, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] ZFuture, } @@ -146,7 +146,7 @@ impl TxVersion { (OVERWINTER_TX_VERSION, OVERWINTER_VERSION_GROUP_ID) => Ok(TxVersion::Overwinter), (SAPLING_TX_VERSION, SAPLING_VERSION_GROUP_ID) => Ok(TxVersion::Sapling), (V5_TX_VERSION, V5_VERSION_GROUP_ID) => Ok(TxVersion::Zip225), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] (ZFUTURE_TX_VERSION, ZFUTURE_VERSION_GROUP_ID) => Ok(TxVersion::ZFuture), _ => Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -176,7 +176,7 @@ impl TxVersion { TxVersion::Overwinter => OVERWINTER_TX_VERSION, TxVersion::Sapling => SAPLING_TX_VERSION, TxVersion::Zip225 => V5_TX_VERSION, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => ZFUTURE_TX_VERSION, } } @@ -187,7 +187,7 @@ impl TxVersion { TxVersion::Overwinter => OVERWINTER_VERSION_GROUP_ID, TxVersion::Sapling => SAPLING_VERSION_GROUP_ID, TxVersion::Zip225 => V5_VERSION_GROUP_ID, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => ZFUTURE_VERSION_GROUP_ID, } } @@ -206,7 +206,7 @@ impl TxVersion { TxVersion::Sprout(v) => *v >= 2u32, TxVersion::Overwinter | TxVersion::Sapling => true, TxVersion::Zip225 => false, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => true, } } @@ -221,7 +221,7 @@ impl TxVersion { TxVersion::Sprout(_) | TxVersion::Overwinter => false, TxVersion::Sapling => true, TxVersion::Zip225 => true, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => true, } } @@ -231,12 +231,12 @@ impl TxVersion { match self { TxVersion::Sprout(_) | TxVersion::Overwinter | TxVersion::Sapling => false, TxVersion::Zip225 => true, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => true, } } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub fn has_tze(&self) -> bool { matches!(self, TxVersion::ZFuture) } @@ -250,9 +250,9 @@ impl TxVersion { TxVersion::Sapling } BranchId::Nu5 => TxVersion::Zip225, - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] BranchId::Nu6 => TxVersion::Zip225, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] BranchId::ZFuture => TxVersion::ZFuture, } } @@ -264,7 +264,7 @@ pub trait Authorization { type SaplingAuth: sapling::bundle::Authorization; type OrchardAuth: orchard::bundle::Authorization; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeAuth: tze::Authorization; } @@ -277,7 +277,7 @@ impl Authorization for Authorized { type SaplingAuth = sapling::bundle::Authorized; type OrchardAuth = orchard::bundle::Authorized; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeAuth = tze::Authorized; } @@ -294,7 +294,7 @@ impl Authorization for Unauthorized { type OrchardAuth = orchard::builder::InProgress; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeAuth = tze::builder::Unauthorized; } @@ -330,7 +330,7 @@ pub struct TransactionData { sprout_bundle: Option, sapling_bundle: Option>, orchard_bundle: Option>, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle: Option>, } @@ -356,14 +356,14 @@ impl TransactionData { sprout_bundle, sapling_bundle, orchard_bundle, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle: None, } } /// Constructs a `TransactionData` from its constituent parts, including speculative /// future parts that are not in the current Zcash consensus rules. - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] #[allow(clippy::too_many_arguments)] pub fn from_parts_zfuture( version: TxVersion, @@ -423,7 +423,7 @@ impl TransactionData { self.orchard_bundle.as_ref() } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub fn tze_bundle(&self) -> Option<&tze::Bundle> { self.tze_bundle.as_ref() } @@ -469,7 +469,7 @@ impl TransactionData { digester.digest_transparent(self.transparent_bundle.as_ref()), digester.digest_sapling(self.sapling_bundle.as_ref()), digester.digest_orchard(self.orchard_bundle.as_ref()), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] digester.digest_tze(self.tze_bundle.as_ref()), ) } @@ -489,9 +489,10 @@ impl TransactionData { f_orchard: impl FnOnce( Option>, ) -> Option>, - #[cfg(feature = "zfuture")] f_tze: impl FnOnce( + #[cfg(zcash_unstable = "zfuture")] f_tze: impl FnOnce( Option>, - ) -> Option>, + ) + -> Option>, ) -> TransactionData { TransactionData { version: self.version, @@ -502,7 +503,7 @@ impl TransactionData { sprout_bundle: self.sprout_bundle, sapling_bundle: f_sapling(self.sapling_bundle), orchard_bundle: f_orchard(self.orchard_bundle), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle: f_tze(self.tze_bundle), } } @@ -512,7 +513,7 @@ impl TransactionData { f_transparent: impl transparent::MapAuth, mut f_sapling: impl sapling_serialization::MapAuth, mut f_orchard: impl orchard_serialization::MapAuth, - #[cfg(feature = "zfuture")] f_tze: impl tze::MapAuth, + #[cfg(zcash_unstable = "zfuture")] f_tze: impl tze::MapAuth, ) -> TransactionData { TransactionData { version: self.version, @@ -539,7 +540,7 @@ impl TransactionData { |f, a| f.map_authorization(a), ) }), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle: self.tze_bundle.map(|b| b.map_authorization(f_tze)), } } @@ -566,7 +567,7 @@ impl Transaction { Self::from_data_v4(data) } TxVersion::Zip225 => Ok(Self::from_data_v5(data)), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => Ok(Self::from_data_v5(data)), } } @@ -609,7 +610,7 @@ impl Transaction { Self::read_v4(reader, version, consensus_branch_id) } TxVersion::Zip225 => Self::read_v5(reader.into_base_reader(), version), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => Self::read_v5(reader.into_base_reader(), version), } } @@ -685,7 +686,7 @@ impl Transaction { ) }), orchard_bundle: None, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle: None, }, }) @@ -721,7 +722,7 @@ impl Transaction { let sapling_bundle = sapling_serialization::read_v5_bundle(&mut reader)?; let orchard_bundle = orchard_serialization::read_v5_bundle(&mut reader)?; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let tze_bundle = if version.has_tze() { Self::read_tze(&mut reader)? } else { @@ -737,7 +738,7 @@ impl Transaction { sprout_bundle: None, sapling_bundle, orchard_bundle, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_bundle, }; @@ -765,7 +766,7 @@ impl Transaction { sapling_serialization::read_v5_bundle(reader) } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] fn read_tze(mut reader: &mut R) -> io::Result>> { let vin = Vector::read(&mut reader, TzeIn::read)?; let vout = Vector::read(&mut reader, TzeOut::read)?; @@ -786,7 +787,7 @@ impl Transaction { self.write_v4(writer) } TxVersion::Zip225 => self.write_v5(writer), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => self.write_v5(writer), } } @@ -855,7 +856,7 @@ impl Transaction { self.write_transparent(&mut writer)?; self.write_v5_sapling(&mut writer)?; orchard_serialization::write_v5_bundle(self.orchard_bundle.as_ref(), &mut writer)?; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] self.write_tze(&mut writer)?; Ok(()) } @@ -880,7 +881,7 @@ impl Transaction { sapling_serialization::write_v5_bundle(writer, self.sapling_bundle.as_ref()) } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub fn write_tze(&self, mut writer: W) -> io::Result<()> { if let Some(bundle) = &self.tze_bundle { Vector::write(&mut writer, &bundle.vin, |w, e| e.write(w))?; @@ -919,7 +920,7 @@ pub struct TxDigests { pub transparent_digests: Option>, pub sapling_digest: Option, pub orchard_digest: Option, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] pub tze_digests: Option>, } @@ -929,7 +930,7 @@ pub trait TransactionDigest { type SaplingDigest; type OrchardDigest; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeDigest; type Digest; @@ -957,7 +958,7 @@ pub trait TransactionDigest { orchard_bundle: Option<&orchard::Bundle>, ) -> Self::OrchardDigest; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] fn digest_tze(&self, tze_bundle: Option<&tze::Bundle>) -> Self::TzeDigest; fn combine( @@ -966,7 +967,7 @@ pub trait TransactionDigest { transparent_digest: Self::TransparentDigest, sapling_digest: Self::SaplingDigest, orchard_digest: Self::OrchardDigest, - #[cfg(feature = "zfuture")] tze_digest: Self::TzeDigest, + #[cfg(zcash_unstable = "zfuture")] tze_digest: Self::TzeDigest, ) -> Self::Digest; } @@ -989,7 +990,7 @@ pub mod testing { Authorized, Transaction, TransactionData, TxId, TxVersion, }; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] use super::components::tze::testing::{self as tze}; pub fn arb_txid() -> impl Strategy { @@ -1004,14 +1005,14 @@ pub mod testing { Just(TxVersion::Sapling).boxed() } BranchId::Nu5 => Just(TxVersion::Zip225).boxed(), - #[cfg(feature = "unstable-nu6")] + #[cfg(zcash_unstable = "nu6")] BranchId::Nu6 => Just(TxVersion::Zip225).boxed(), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] BranchId::ZFuture => Just(TxVersion::ZFuture).boxed(), } } - #[cfg(not(feature = "zfuture"))] + #[cfg(not(zcash_unstable = "zfuture"))] prop_compose! { pub fn arb_txdata(consensus_branch_id: BranchId)( version in arb_tx_version(consensus_branch_id), @@ -1036,7 +1037,7 @@ pub mod testing { } } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] prop_compose! { pub fn arb_txdata(consensus_branch_id: BranchId)( version in arb_tx_version(consensus_branch_id), diff --git a/zcash_primitives/src/transaction/sighash.rs b/zcash_primitives/src/transaction/sighash.rs index 93ca0eef2c..702ed17575 100644 --- a/zcash_primitives/src/transaction/sighash.rs +++ b/zcash_primitives/src/transaction/sighash.rs @@ -11,7 +11,7 @@ use crate::{ sapling::{self, bundle::GrothProofBytes}, }; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] use {super::components::Amount, crate::extensions::transparent::Precondition}; pub const SIGHASH_ALL: u8 = 0x01; @@ -29,7 +29,7 @@ pub enum SignableInput<'a> { script_pubkey: &'a Script, value: NonNegativeAmount, }, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] Tze { index: usize, precondition: &'a Precondition, @@ -42,7 +42,7 @@ impl<'a> SignableInput<'a> { match self { SignableInput::Shielded => SIGHASH_ALL, SignableInput::Transparent { hash_type, .. } => *hash_type, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] SignableInput::Tze { .. } => SIGHASH_ALL, } } @@ -92,7 +92,7 @@ pub fn signature_hash< TxVersion::Zip225 => v5_signature_hash(tx, signable_input, txid_parts), - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] TxVersion::ZFuture => v5_signature_hash(tx, signable_input, txid_parts), }) } diff --git a/zcash_primitives/src/transaction/sighash_v4.rs b/zcash_primitives/src/transaction/sighash_v4.rs index e50c6902f3..ad136e5f8d 100644 --- a/zcash_primitives/src/transaction/sighash_v4.rs +++ b/zcash_primitives/src/transaction/sighash_v4.rs @@ -255,7 +255,7 @@ pub fn v4_signature_hash< } } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] SignableInput::Tze { .. } => { panic!("A request has been made to sign a TZE input, but the transaction version is not ZFuture"); } diff --git a/zcash_primitives/src/transaction/sighash_v5.rs b/zcash_primitives/src/transaction/sighash_v5.rs index 72d15d94ff..60b02b8b75 100644 --- a/zcash_primitives/src/transaction/sighash_v5.rs +++ b/zcash_primitives/src/transaction/sighash_v5.rs @@ -16,20 +16,18 @@ use crate::transaction::{ Authorization, TransactionData, TransparentDigests, TxDigests, }; -#[cfg(feature = "zfuture")] -use byteorder::WriteBytesExt; - -#[cfg(feature = "zfuture")] -use zcash_encoding::{CompactSize, Vector}; - -#[cfg(feature = "zfuture")] -use crate::transaction::{components::tze, TzeDigests}; +#[cfg(zcash_unstable = "zfuture")] +use { + crate::transaction::{components::tze, TzeDigests}, + byteorder::WriteBytesExt, + zcash_encoding::{CompactSize, Vector}, +}; const ZCASH_TRANSPARENT_INPUT_HASH_PERSONALIZATION: &[u8; 16] = b"Zcash___TxInHash"; const ZCASH_TRANSPARENT_AMOUNTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxTrAmountsHash"; const ZCASH_TRANSPARENT_SCRIPTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxTrScriptsHash"; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZCASH_TZE_INPUT_HASH_PERSONALIZATION: &[u8; 16] = b"Zcash__TzeInHash"; fn hasher(personal: &[u8; 16]) -> State { @@ -140,7 +138,7 @@ fn transparent_sig_digest( } } -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] fn tze_input_sigdigests( bundle: &tze::Bundle, input: &SignableInput<'_>, @@ -197,7 +195,7 @@ pub fn v5_signature_hash< ), txid_parts.sapling_digest, txid_parts.orchard_digest, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tx.tze_bundle .as_ref() .zip(txid_parts.tze_digests.as_ref()) diff --git a/zcash_primitives/src/transaction/tests.rs b/zcash_primitives/src/transaction/tests.rs index e74fe63709..09552226e1 100644 --- a/zcash_primitives/src/transaction/tests.rs +++ b/zcash_primitives/src/transaction/tests.rs @@ -21,7 +21,7 @@ use super::{ Authorization, Transaction, TransactionData, TxDigests, TxIn, }; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] use super::components::tze; #[test] @@ -44,7 +44,7 @@ fn check_roundtrip(tx: Transaction) -> Result<(), TestCaseError> { let txo = Transaction::read(&txn_bytes[..], tx.consensus_branch_id).unwrap(); prop_assert_eq!(tx.version, txo.version); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] prop_assert_eq!(tx.tze_bundle.as_ref(), txo.tze_bundle.as_ref()); prop_assert_eq!(tx.lock_time, txo.lock_time); prop_assert_eq!( @@ -115,7 +115,7 @@ proptest! { } } -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] proptest! { #[test] #[ignore] @@ -196,7 +196,7 @@ impl Authorization for TestUnauthorized { type SaplingAuth = sapling::bundle::Authorized; type OrchardAuth = orchard::bundle::Authorized; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeAuth = tze::Authorized; } @@ -246,7 +246,7 @@ fn zip_0244() { }, }); - #[cfg(not(feature = "zfuture"))] + #[cfg(not(zcash_unstable = "zfuture"))] let tdata = TransactionData::from_parts( txdata.version(), txdata.consensus_branch_id(), @@ -257,7 +257,7 @@ fn zip_0244() { txdata.sapling_bundle().cloned(), txdata.orchard_bundle().cloned(), ); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] let tdata = TransactionData::from_parts_zfuture( txdata.version(), txdata.consensus_branch_id(), diff --git a/zcash_primitives/src/transaction/txid.rs b/zcash_primitives/src/transaction/txid.rs index ba3b45d377..84941271c9 100644 --- a/zcash_primitives/src/transaction/txid.rs +++ b/zcash_primitives/src/transaction/txid.rs @@ -23,7 +23,7 @@ use super::{ Authorization, Authorized, TransactionDigest, TransparentDigests, TxDigests, TxId, TxVersion, }; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] use super::{ components::tze::{self, TzeIn, TzeOut}, TzeDigests, @@ -36,7 +36,7 @@ const ZCASH_TX_PERSONALIZATION_PREFIX: &[u8; 12] = b"ZcashTxHash_"; const ZCASH_HEADERS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdHeadersHash"; pub(crate) const ZCASH_TRANSPARENT_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdTranspaHash"; const ZCASH_SAPLING_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdSaplingHash"; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZCASH_TZE_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdTZE____Hash"; // TxId transparent level 2 node personalization @@ -45,9 +45,9 @@ const ZCASH_SEQUENCE_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdSequencHash"; const ZCASH_OUTPUTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdOutputsHash"; // TxId tze level 2 node personalization -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZCASH_TZE_INPUTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdTZEIns_Hash"; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZCASH_TZE_OUTPUTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxIdTZEOutsHash"; // TxId sapling level 2 node personalization @@ -63,7 +63,7 @@ const ZCASH_SAPLING_OUTPUTS_NONCOMPACT_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxId const ZCASH_AUTH_PERSONALIZATION_PREFIX: &[u8; 12] = b"ZTxAuthHash_"; const ZCASH_TRANSPARENT_SCRIPTS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxAuthTransHash"; const ZCASH_SAPLING_SIGS_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxAuthSapliHash"; -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] const ZCASH_TZE_WITNESSES_HASH_PERSONALIZATION: &[u8; 16] = b"ZTxAuthTZE__Hash"; fn hasher(personal: &[u8; 16]) -> State { @@ -112,7 +112,7 @@ pub(crate) fn transparent_outputs_hash>(vout: &[T]) -> Blake2bH /// witness data, to a hash personalized by ZCASH_TZE_INPUTS_HASH_PERSONALIZATION. /// In the case that no inputs are provided, this produces a default /// hash from just the personalization string. -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] pub(crate) fn hash_tze_inputs(tze_inputs: &[TzeIn]) -> Blake2bHash { let mut h = hasher(ZCASH_TZE_INPUTS_HASH_PERSONALIZATION); for tzein in tze_inputs { @@ -125,7 +125,7 @@ pub(crate) fn hash_tze_inputs(tze_inputs: &[TzeIn]) -> Blake2bHash { /// to a hash personalized by ZCASH_TZE_OUTPUTS_HASH_PERSONALIZATION. /// In the case that no outputs are provided, this produces a default /// hash from just the personalization string. -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] pub(crate) fn hash_tze_outputs(tze_outputs: &[TzeOut]) -> Blake2bHash { let mut h = hasher(ZCASH_TZE_OUTPUTS_HASH_PERSONALIZATION); for tzeout in tze_outputs { @@ -210,7 +210,7 @@ fn transparent_digests( } } -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] fn tze_digests(bundle: &tze::Bundle) -> TzeDigests { // The txid commits to the hash for all outputs. TzeDigests { @@ -276,7 +276,7 @@ fn hash_sapling_txid_empty() -> Blake2bHash { hasher(ZCASH_SAPLING_HASH_PERSONALIZATION).finalize() } -#[cfg(feature = "zfuture")] +#[cfg(zcash_unstable = "zfuture")] fn hash_tze_txid_data(tze_digests: Option<&TzeDigests>) -> Blake2bHash { let mut h = hasher(ZCASH_TZE_HASH_PERSONALIZATION); if let Some(d) = tze_digests { @@ -304,7 +304,7 @@ impl TransactionDigest for TxIdDigester { type SaplingDigest = Option; type OrchardDigest = Option; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeDigest = Option>; type Digest = TxDigests; @@ -340,7 +340,7 @@ impl TransactionDigest for TxIdDigester { orchard_bundle.map(|b| b.commitment().0) } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] fn digest_tze(&self, tze_bundle: Option<&tze::Bundle>) -> Self::TzeDigest { tze_bundle.map(tze_digests) } @@ -351,14 +351,14 @@ impl TransactionDigest for TxIdDigester { transparent_digests: Self::TransparentDigest, sapling_digest: Self::SaplingDigest, orchard_digest: Self::OrchardDigest, - #[cfg(feature = "zfuture")] tze_digests: Self::TzeDigest, + #[cfg(zcash_unstable = "zfuture")] tze_digests: Self::TzeDigest, ) -> Self::Digest { TxDigests { header_digest, transparent_digests, sapling_digest, orchard_digest, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] tze_digests, } } @@ -371,7 +371,7 @@ pub(crate) fn to_hash( transparent_digest: Blake2bHash, sapling_digest: Option, orchard_digest: Option, - #[cfg(feature = "zfuture")] tze_digests: Option<&TzeDigests>, + #[cfg(zcash_unstable = "zfuture")] tze_digests: Option<&TzeDigests>, ) -> Blake2bHash { let mut personal = [0; 16]; personal[..12].copy_from_slice(ZCASH_TX_PERSONALIZATION_PREFIX); @@ -395,7 +395,7 @@ pub(crate) fn to_hash( ) .unwrap(); - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] if _txversion.has_tze() { h.write_all(hash_tze_txid_data(tze_digests).as_bytes()) .unwrap(); @@ -416,7 +416,7 @@ pub fn to_txid( hash_transparent_txid_data(digests.transparent_digests.as_ref()), digests.sapling_digest, digests.orchard_digest, - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] digests.tze_digests.as_ref(), ); @@ -437,7 +437,7 @@ impl TransactionDigest for BlockTxCommitmentDigester { type SaplingDigest = Blake2bHash; type OrchardDigest = Blake2bHash; - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] type TzeDigest = Blake2bHash; type Digest = Blake2bHash; @@ -499,7 +499,7 @@ impl TransactionDigest for BlockTxCommitmentDigester { }) } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] fn digest_tze(&self, tze_bundle: Option<&tze::Bundle>) -> Blake2bHash { let mut h = hasher(ZCASH_TZE_WITNESSES_HASH_PERSONALIZATION); if let Some(bundle) = tze_bundle { @@ -516,7 +516,7 @@ impl TransactionDigest for BlockTxCommitmentDigester { transparent_digest: Self::TransparentDigest, sapling_digest: Self::SaplingDigest, orchard_digest: Self::OrchardDigest, - #[cfg(feature = "zfuture")] tze_digest: Self::TzeDigest, + #[cfg(zcash_unstable = "zfuture")] tze_digest: Self::TzeDigest, ) -> Self::Digest { let digests = [transparent_digest, sapling_digest, orchard_digest]; @@ -531,7 +531,7 @@ impl TransactionDigest for BlockTxCommitmentDigester { h.write_all(digest.as_bytes()).unwrap(); } - #[cfg(feature = "zfuture")] + #[cfg(zcash_unstable = "zfuture")] if TxVersion::suggested_for_branch(consensus_branch_id).has_tze() { h.write_all(tze_digest.as_bytes()).unwrap(); }