Merge pull request #1289 from nuttycom/sqlite_wallet/branching_chain_…

…test_fixes

zcash_client_sqlite: Fix `scan_complete` tests.

Cargo.toml

@@ -122,6 +122,7 @@ panic = 'abort'
 codegen-units = 1
 
 [patch.crates-io]
-orchard = { git = "https://github.com/zcash/orchard", rev = "e74879dd0ad0918f4ffe0826e03905cd819981bd" }
-incrementalmerkletree = { git = "https://github.com/nuttycom/incrementalmerkletree", rev = "fa147c89c6c98a03bba745538f4e68d4eaed5146" }
-shardtree = { git = "https://github.com/nuttycom/incrementalmerkletree", rev = "fa147c89c6c98a03bba745538f4e68d4eaed5146" }
+orchard = { git = "https://github.com/zcash/orchard", rev = "33474bdbfd7268e1f84718078d47f63d01a879d5" }
+incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree", rev = "e1a7a80212c22e5a8912d05860f7eb6899c56a7c" }
+sapling = { git = "https://github.com/zcash/sapling-crypto", package = "sapling-crypto", rev = "22412ae07644813253feb064d1692b0823242853" }
+shardtree = { git = "https://github.com/zcash/incrementalmerkletree", rev = "e1a7a80212c22e5a8912d05860f7eb6899c56a7c" }

zcash_client_backend/CHANGELOG.md

@@ -30,6 +30,7 @@ and this library adheres to Rust's notion of
   - `WalletSummary::next_orchard_subtree_index`
   - `chain::ChainState`
   - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}`
+  - `impl Debug for chain::CommitmentTreeRoot`
 - `zcash_client_backend::fees`:
   - `orchard`
   - `ChangeValue::orchard`

zcash_client_backend/src/data_api/chain.rs

@@ -155,7 +155,10 @@ use std::ops::Range;
 
 use incrementalmerkletree::frontier::Frontier;
 use subtle::ConditionallySelectable;
-use zcash_primitives::consensus::{self, BlockHeight};
+use zcash_primitives::{
+    block::BlockHash,
+    consensus::{self, BlockHeight},
+};
 
 use crate::{
     data_api::{NullifierQuery, WalletWrite},
@@ -172,6 +175,7 @@ use super::WalletRead;
 ///
 /// This stores the block height at which the leaf that completed the subtree was
 /// added, and the root hash of the complete subtree.
+#[derive(Debug)]
 pub struct CommitmentTreeRoot<H> {
     subtree_end_height: BlockHeight,
     root_hash: H,
@@ -291,6 +295,7 @@ impl ScanSummary {
 #[derive(Debug, Clone)]
 pub struct ChainState {
     block_height: BlockHeight,
+    block_hash: BlockHash,
     final_sapling_tree: Frontier<sapling::Node, { sapling::NOTE_COMMITMENT_TREE_DEPTH }>,
     #[cfg(feature = "orchard")]
     final_orchard_tree:
@@ -299,9 +304,10 @@ pub struct ChainState {
 
 impl ChainState {
     /// Construct a new empty chain state.
-    pub fn empty(block_height: BlockHeight) -> Self {
+    pub fn empty(block_height: BlockHeight, block_hash: BlockHash) -> Self {
         Self {
             block_height,
+            block_hash,
             final_sapling_tree: Frontier::empty(),
             #[cfg(feature = "orchard")]
             final_orchard_tree: Frontier::empty(),
@@ -311,6 +317,7 @@ impl ChainState {
     /// Construct a new [`ChainState`] from its constituent parts.
     pub fn new(
         block_height: BlockHeight,
+        block_hash: BlockHash,
         final_sapling_tree: Frontier<sapling::Node, { sapling::NOTE_COMMITMENT_TREE_DEPTH }>,
         #[cfg(feature = "orchard")] final_orchard_tree: Frontier<
             orchard::tree::MerkleHashOrchard,
@@ -319,6 +326,7 @@ impl ChainState {
     ) -> Self {
         Self {
             block_height,
+            block_hash,
             final_sapling_tree,
             #[cfg(feature = "orchard")]
             final_orchard_tree,
@@ -330,6 +338,11 @@ impl ChainState {
         self.block_height
     }
 
+    /// Return the hash of the block.
+    pub fn block_hash(&self) -> BlockHash {
+        self.block_hash
+    }
+
     /// Returns the frontier of the Sapling note commitment tree as of the end of the block at
     /// [`Self::block_height`].
     pub fn final_sapling_tree(

zcash_client_backend/src/proto.rs

@@ -295,10 +295,22 @@ impl service::TreeState {
     ///
     /// [`scan_cached_blocks`]: crate::data_api::chain::scan_cached_blocks
     pub fn to_chain_state(&self) -> io::Result<ChainState> {
+        let mut hash_bytes = hex::decode(&self.hash).map_err(|e| {
+            io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("Block hash is not valid hex: {:?}", e),
+            )
+        })?;
+        // Zcashd hex strings for block hashes are byte-reversed.
+        hash_bytes.reverse();
+
         Ok(ChainState::new(
             self.height
                 .try_into()
                 .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid block height"))?,
+            BlockHash::try_from_slice(&hash_bytes).ok_or_else(|| {
+                io::Error::new(io::ErrorKind::InvalidData, "Invalid block hash length.")
+            })?,
             self.sapling_tree()?.to_frontier(),
             #[cfg(feature = "orchard")]
             self.orchard_tree()?.to_frontier(),

zcash_client_sqlite/src/testing.rs

@@ -6,12 +6,14 @@ use std::{collections::BTreeMap, convert::Infallible};
 use std::fs::File;
 
 use group::ff::Field;
+use incrementalmerkletree::Retention;
 use nonempty::NonEmpty;
 use prost::Message;
 use rand_chacha::ChaChaRng;
 use rand_core::{CryptoRng, RngCore, SeedableRng};
 use rusqlite::{params, Connection};
 use secrecy::{Secret, SecretVec};
+use shardtree::error::ShardTreeError;
 use tempfile::NamedTempFile;
 
 #[cfg(feature = "unstable")]
@@ -28,13 +30,14 @@ use zcash_client_backend::{
     address::Address,
     data_api::{
         self,
-        chain::{scan_cached_blocks, BlockSource, ScanSummary},
+        chain::{scan_cached_blocks, BlockSource, CommitmentTreeRoot, ScanSummary},
         wallet::{
             create_proposed_transactions, create_spend_to_address,
             input_selection::{GreedyInputSelector, GreedyInputSelectorError, InputSelector},
             propose_standard_transfer_to_address, propose_transfer, spend,
         },
-        AccountBalance, AccountBirthday, WalletRead, WalletSummary, WalletWrite,
+        AccountBalance, AccountBirthday, WalletCommitmentTrees, WalletRead, WalletSummary,
+        WalletWrite,
     },
     keys::UnifiedSpendingKey,
     proposal::Proposal,
@@ -190,7 +193,6 @@ impl<Cache> TestBuilder<Cache> {
 
 #[derive(Clone, Debug)]
 pub(crate) struct CachedBlock {
-    hash: BlockHash,
     chain_state: ChainState,
     sapling_end_size: u32,
     orchard_end_size: u32,
@@ -199,19 +201,13 @@ pub(crate) struct CachedBlock {
 impl CachedBlock {
     fn none(sapling_activation_height: BlockHeight) -> Self {
         Self {
-            hash: BlockHash([0; 32]),
-            chain_state: ChainState::empty(sapling_activation_height),
+            chain_state: ChainState::empty(sapling_activation_height, BlockHash([0; 32])),
             sapling_end_size: 0,
             orchard_end_size: 0,
         }
     }
 
-    fn at(
-        hash: BlockHash,
-        chain_state: ChainState,
-        sapling_end_size: u32,
-        orchard_end_size: u32,
-    ) -> Self {
+    fn at(chain_state: ChainState, sapling_end_size: u32, orchard_end_size: u32) -> Self {
         assert_eq!(
             chain_state.final_sapling_tree().tree_size() as u32,
             sapling_end_size
@@ -223,7 +219,6 @@ impl CachedBlock {
         );
 
         Self {
-            hash,
             chain_state,
             sapling_end_size,
             orchard_end_size,
@@ -258,9 +253,9 @@ impl CachedBlock {
         });
 
         Self {
-            hash: cb.hash(),
             chain_state: ChainState::new(
                 cb.height(),
+                cb.hash(),
                 sapling_final_tree,
                 #[cfg(feature = "orchard")]
                 orchard_final_tree,
@@ -323,6 +318,67 @@ where
         self.cache.insert(&compact_block)
     }
 
+    /// Ensure that the provided chain state and subtree roots exist in the wallet's note
+    /// commitment tree(s). This may result in a conflict if either the provided subtree roots or
+    /// the chain state conflict with existing note commitment tree data.
+    pub(crate) fn establish_chain_state(
+        &mut self,
+        state: ChainState,
+        prior_sapling_roots: &[CommitmentTreeRoot<sapling::Node>],
+        #[cfg(feature = "orchard")] prior_orchard_roots: &[CommitmentTreeRoot<MerkleHashOrchard>],
+    ) -> Result<(), ShardTreeError<commitment_tree::Error>> {
+        self.wallet_mut()
+            .put_sapling_subtree_roots(0, prior_sapling_roots)?;
+        #[cfg(feature = "orchard")]
+        self.wallet_mut()
+            .put_orchard_subtree_roots(0, prior_orchard_roots)?;
+
+        self.wallet_mut().with_sapling_tree_mut(|t| {
+            t.insert_frontier(
+                state.final_sapling_tree().clone(),
+                Retention::Checkpoint {
+                    id: state.block_height(),
+                    is_marked: false,
+                },
+            )
+        })?;
+        let final_sapling_tree_size = state.final_sapling_tree().tree_size() as u32;
+
+        #[cfg(feature = "orchard")]
+        self.wallet_mut().with_orchard_tree_mut(|t| {
+            t.insert_frontier(
+                state.final_orchard_tree().clone(),
+                Retention::Checkpoint {
+                    id: state.block_height(),
+                    is_marked: false,
+                },
+            )
+        })?;
+
+        let _final_orchard_tree_size = 0;
+        #[cfg(feature = "orchard")]
+        let _final_orchard_tree_size = state.final_orchard_tree().tree_size() as u32;
+
+        self.insert_cached_block(state, final_sapling_tree_size, _final_orchard_tree_size);
+        Ok(())
+    }
+
+    fn insert_cached_block(
+        &mut self,
+        chain_state: ChainState,
+        sapling_end_size: u32,
+        orchard_end_size: u32,
+    ) {
+        self.cached_blocks.insert(
+            chain_state.block_height(),
+            CachedBlock {
+                chain_state,
+                sapling_end_size,
+                orchard_end_size,
+            },
+        );
+    }
+
     /// Creates a fake block at the expected next height containing a single output of the
     /// given value, and inserts it into the cache.
     pub(crate) fn generate_next_block<Fvk: TestFvk>(
@@ -337,7 +393,7 @@ where
 
         let (res, nf) = self.generate_block_at(
             height,
-            prior_cached_block.hash,
+            prior_cached_block.chain_state.block_hash(),
             fvk,
             req,
             value,
@@ -376,7 +432,7 @@ where
         // we need to generate a new prior cached block that the block to be generated can
         // successfully chain from, with the provided tree sizes.
         if prior_cached_block.chain_state.block_height() == height - 1 {
-            assert_eq!(prev_hash, prior_cached_block.hash);
+            assert_eq!(prev_hash, prior_cached_block.chain_state.block_hash());
         } else {
             let final_sapling_tree =
                 (prior_cached_block.sapling_end_size..initial_sapling_tree_size).fold(
@@ -400,9 +456,9 @@ where
                 );
 
             prior_cached_block = CachedBlock::at(
-                prev_hash,
                 ChainState::new(
                     height - 1,
+                    prev_hash,
                     final_sapling_tree,
                     #[cfg(feature = "orchard")]
                     final_orchard_tree,
@@ -452,7 +508,7 @@ where
         let cb = fake_compact_block_spending(
             &self.network(),
             height,
-            prior_cached_block.hash,
+            prior_cached_block.chain_state.block_hash(),
             note,
             fvk,
             to.into(),
@@ -508,7 +564,7 @@ where
 
         let cb = fake_compact_block_from_tx(
             height,
-            prior_cached_block.hash,
+            prior_cached_block.chain_state.block_hash(),
             tx_index,
             tx,
             prior_cached_block.sapling_end_size,
@@ -613,6 +669,11 @@ impl<Cache> TestState<Cache> {
         self.db_data.params
     }
 
+    /// Exposes the random number source for the test state
+    pub(crate) fn rng(&mut self) -> &mut ChaChaRng {
+        &mut self.rng
+    }
+
     /// Convenience method for obtaining the Sapling activation height for the network under test.
     pub(crate) fn sapling_activation_height(&self) -> BlockHeight {
         self.db_data

zcash_client_sqlite/src/wallet/orchard.rs

@@ -378,6 +378,7 @@ pub(crate) mod tests {
     impl ShieldedPoolTester for OrchardPoolTester {
         const SHIELDED_PROTOCOL: ShieldedProtocol = ShieldedProtocol::Orchard;
         const TABLES_PREFIX: &'static str = ORCHARD_TABLES_PREFIX;
+        // const MERKLE_TREE_DEPTH: u8 = {orchard::NOTE_COMMITMENT_TREE_DEPTH as u8};
 
         type Sk = SpendingKey;
         type Fvk = FullViewingKey;

zcash_client_sqlite/src/wallet/sapling.rs

@@ -399,6 +399,7 @@ pub(crate) mod tests {
     impl ShieldedPoolTester for SaplingPoolTester {
         const SHIELDED_PROTOCOL: ShieldedProtocol = ShieldedProtocol::Sapling;
         const TABLES_PREFIX: &'static str = SAPLING_TABLES_PREFIX;
+        // const MERKLE_TREE_DEPTH: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH;
 
         type Sk = ExtendedSpendingKey;
         type Fvk = DiversifiableFullViewingKey;