Merge pull request #1284 from fluidvanadium/conditional_rescan

Conditional rescan

darkside-tests/tests/advanced_reorg_tests.rs

@@ -607,7 +607,7 @@ async fn reorg_changes_outgoing_tx_height() {
             .find_map(|v| match v.kind() {
                 ValueTransferKind::Sent => {
                     if let Some(addr) = v.recipient_address() {
-                        if addr.to_string() == recipient_string && v.value() == 100_000 {
+                        if addr == recipient_string && v.value() == 100_000 {
                             Some(v.blockheight())
                         } else {
                             None
@@ -839,7 +839,7 @@ async fn reorg_expires_outgoing_tx_height() {
             .find_map(|v| match v.kind() {
                 ValueTransferKind::Sent => {
                     if let Some(addr) = v.recipient_address() {
-                        if addr.to_string() == recipient_string && v.value() == 100_000 {
+                        if addr == recipient_string && v.value() == 100_000 {
                             Some(v.blockheight())
                         } else {
                             None
@@ -1026,7 +1026,7 @@ async fn reorg_changes_outgoing_tx_index() {
             .find_map(|v| match v.kind() {
                 ValueTransferKind::Sent => {
                     if let Some(addr) = v.recipient_address() {
-                        if addr.to_string() == recipient_string && v.value() == 100_000 {
+                        if addr == recipient_string && v.value() == 100_000 {
                             Some(v.blockheight())
                         } else {
                             None

libtonode-tests/tests/concrete.rs

@@ -2959,9 +2959,10 @@ mod slow {
             let txid2 = utils::conversion::txid_from_hex_encoded_str("7a9d41caca143013ebd2f710e4dad04f0eb9f0ae98b42af0f58f25c61a9d439e").unwrap();
             let expected_txids = vec![txid1, txid2];
             // in case the txids are in reverse order
-            if output_error != expected_txids {
+            let missing_index_txids: Vec<zcash_primitives::transaction::TxId> = output_error.into_iter().map(|(txid, _)| txid).collect();
+            if missing_index_txids != expected_txids {
                 let expected_txids = vec![txid2, txid1];
-                assert!(output_error == expected_txids, "{:?}\n\n{:?}", output_error, expected_txids);
+                assert!(missing_index_txids == expected_txids, "{:?}\n\n{:?}", missing_index_txids, expected_txids);
             }
         };
     }

zingolib/src/blaze.rs

@@ -2,8 +2,11 @@
 
 pub(super) mod block_management_reorg_detection;
 pub(super) mod fetch_compact_blocks;
-pub(super) mod fetch_full_transaction;
 pub(super) mod fetch_taddr_transactions;
+/// alternative name: daemon_for_txid_lookup_and_record_updates
+/// this function can update a few details about a TransactionRecord. it has numerous gaps
+/// It is the closest thing Zingo has to conditional rescan. it needs to be conditional rescan. as of this git commit 41cf89b it is not.
+pub(super) mod full_transactions_processor;
 pub(super) mod sync_status;
 pub(super) mod syncdata;
 pub(super) mod trial_decryptions;

zingolib/src/blaze/fetch_full_transaction.rs → zingolib/src/blaze/full_transactions_processor.rs

@@ -21,6 +21,7 @@ use zcash_primitives::{
 
 use zingo_status::confirmation_status::ConfirmationStatus;
 
+// despite being called fetch_full_transaction, this function sends a txid somewhere else to be fetched as a Transaction. then processes it.
 pub async fn start(
     transaction_context: TransactionContext,
     fulltx_fetcher: UnboundedSender<(TxId, oneshot::Sender<Result<Transaction, String>>)>,
@@ -51,8 +52,7 @@ pub async fn start(
 
     let bsync_data_i = bsync_data.clone();
 
-    let (transaction_id_transmitter, mut transaction_id_receiver) =
-        unbounded_channel::<(TxId, BlockHeight)>();
+    let (txid_sender, mut transaction_id_receiver) = unbounded_channel::<(TxId, BlockHeight)>();
     let h1: JoinHandle<Result<(), String>> = tokio::spawn(async move {
         let last_progress = Arc::new(AtomicU64::new(0));
         let mut workers = FuturesUnordered::new();
@@ -65,7 +65,7 @@ pub async fn start(
                 .block_data
                 .get_block_timestamp(&height)
                 .await;
-            let full_transaction_fetcher = fulltx_fetcher.clone();
+            let fulltx_fetcher_sub = fulltx_fetcher.clone();
             let bsync_data = bsync_data_i.clone();
             let last_progress = last_progress.clone();
 
@@ -74,7 +74,7 @@ pub async fn start(
                 let transaction = {
                     // Fetch the TxId from LightwalletD and process all the parts of it.
                     let (transmitter, receiver) = oneshot::channel();
-                    full_transaction_fetcher
+                    fulltx_fetcher_sub
                         .send((transaction_id, transmitter))
                         .unwrap();
                     receiver.await.unwrap()?
@@ -120,13 +120,13 @@ pub async fn start(
     });
 
     let transaction_context = transaction_context.clone(); // TODO: Delete and study error.
-    let (transaction_transmitter, mut transaction_receiver) =
+    let (full_transaction_sender, mut full_transaction_receiver) =
         unbounded_channel::<(Transaction, BlockHeight)>();
 
     let h2: JoinHandle<Result<(), String>> = tokio::spawn(async move {
         let bsync_data = bsync_data.clone();
 
-        while let Some((transaction, height)) = transaction_receiver.recv().await {
+        while let Some((transaction, height)) = full_transaction_receiver.recv().await {
             let block_time = bsync_data
                 .read()
                 .await
@@ -150,5 +150,5 @@ pub async fn start(
             .try_for_each(|r| r.map_err(|e| format!("{}", e))?)
     });
 
-    (h, transaction_id_transmitter, transaction_transmitter)
+    (h, txid_sender, full_transaction_sender)
 }

zingolib/src/lightclient/sync.rs

@@ -429,22 +429,35 @@ impl LightClient {
             self.wallet.wallet_capability(),
             self.wallet.transactions(),
         );
-        let (
-            fetch_full_transactions_handle,
-            fetch_full_transaction_transmitter,
-            fetch_taddr_transactions_transmitter,
-        ) = crate::blaze::fetch_full_transaction::start(
-            transaction_context,
-            full_transaction_fetcher_transmitter.clone(),
-            bsync_data.clone(),
-        )
-        .await;
+
+        // fv believes that sending either a transaction or a txid along the txid_sender or full_transaction_sender will result in a scan.
+        let (fetch_full_transactions_handle, txid_sender, full_transaction_sender) =
+            crate::blaze::full_transactions_processor::start(
+                transaction_context.clone(),
+                full_transaction_fetcher_transmitter.clone(),
+                bsync_data.clone(),
+            )
+            .await;
+
+        // targetted_rescan to update missing output indices
+        if let Some(latest_block) = self.wallet.blocks.read().await.first() {
+            // collect any outdated transaction record that are incomplete and missing output indexes
+            let result = transaction_context
+                .unindexed_records(BlockHeight::from_u32(latest_block.height as u32))
+                .await;
+            // send those TxIds to the newly created output scanner
+            if let Err(incomplete_txids_and_heights) = result {
+                incomplete_txids_and_heights
+                    .into_iter()
+                    .for_each(|t| txid_sender.send(t).unwrap());
+            }
+        }
 
         // The processor to process Transactions detected by the trial decryptions processor
         let update_notes_processor = UpdateNotes::new(self.wallet.transactions());
         let (update_notes_handle, blocks_done_transmitter, detected_transactions_transmitter) =
             update_notes_processor
-                .start(bsync_data.clone(), fetch_full_transaction_transmitter)
+                .start(bsync_data.clone(), txid_sender)
                 .await;
 
         // Do Trial decryptions of all the outputs, and pass on the successful ones to the update_notes processor
@@ -501,7 +514,7 @@ impl LightClient {
             start_block,
             earliest_block,
             taddr_fetcher_transmitter,
-            fetch_taddr_transactions_transmitter,
+            full_transaction_sender,
             self.config.chain,
         )
         .await;

zingolib/src/wallet/notes/interface.rs

@@ -154,6 +154,9 @@ pub trait ShieldedNoteInterface: OutputInterface + OutputConstructor + Sized {
     /// TODO: Add Doc Comment Here!
     fn output_index(&self) -> &Option<u32>;
 
+    /// TODO: Add Doc Comment Here!
+    fn output_index_mut(&mut self) -> &mut Option<u32>;
+
     /// TODO: Add Doc Comment Here!
     fn pending_receipt(&self) -> bool {
         self.nullifier().is_none()

zingolib/src/wallet/notes/orchard.rs

@@ -199,6 +199,10 @@ impl ShieldedNoteInterface for OrchardNote {
         &self.output_index
     }
 
+    fn output_index_mut(&mut self) -> &mut Option<u32> {
+        &mut self.output_index
+    }
+
     fn to_zcb_note(&self) -> zcash_client_backend::wallet::Note {
         zcash_client_backend::wallet::Note::Orchard(*self.note())
     }

zingolib/src/wallet/notes/sapling.rs

@@ -221,6 +221,10 @@ impl ShieldedNoteInterface for SaplingNote {
         &self.output_index
     }
 
+    fn output_index_mut(&mut self) -> &mut Option<u32> {
+        &mut self.output_index
+    }
+
     fn to_zcb_note(&self) -> zcash_client_backend::wallet::Note {
         zcash_client_backend::wallet::Note::Sapling(self.note().clone())
     }

zingolib/src/wallet/transaction_context.rs

@@ -3,6 +3,8 @@
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
+use zcash_client_backend::ShieldedProtocol;
+use zcash_primitives::{consensus::BlockHeight, transaction::TxId};
 use zingoconfig::ZingoConfig;
 
 use crate::wallet::{keys::unified::WalletCapability, tx_map_and_maybe_trees::TxMapAndMaybeTrees};
@@ -31,6 +33,24 @@ impl TransactionContext {
             transaction_metadata_set,
         }
     }
+
+    /// returns any outdated records that need to be rescanned for completeness..
+    /// checks that each record contains output indexes for its notes
+    pub async fn unindexed_records(
+        &self,
+        wallet_height: BlockHeight,
+    ) -> Result<(), Vec<(TxId, BlockHeight)>> {
+        self.transaction_metadata_set
+            .read()
+            .await
+            .transaction_records_by_id
+            .get_spendable_note_ids_and_values(
+                &[ShieldedProtocol::Sapling, ShieldedProtocol::Orchard],
+                wallet_height,
+                &[],
+            )
+            .map(|_| ())
+    }
 }
 
 /// These functions are responsible for receiving a full Transaction and storing it, with a few major caveats.
@@ -460,6 +480,8 @@ pub mod decrypt_transaction {
                     _ => continue,
                 };
                 let memo_bytes = MemoBytes::from_bytes(&memo_bytes.to_bytes()).unwrap();
+                // if status is pending add the whole pending note
+                // otherwise, just update the output index
                 if let Some(height) = status.get_pending_height() {
                     self.transaction_metadata_set
                         .write()
@@ -473,6 +495,18 @@ pub mod decrypt_transaction {
                             to,
                             output_index,
                         );
+                } else {
+                    self.transaction_metadata_set
+                        .write()
+                        .await
+                        .transaction_records_by_id
+                        .update_output_index::<D>(
+                            transaction.txid(),
+                            status,
+                            block_time as u64,
+                            note.clone(),
+                            output_index,
+                        )
                 }
                 let memo = memo_bytes
                     .clone()

zingolib/src/wallet/transaction_record.rs

@@ -854,8 +854,8 @@ mod tests {
             Output::get_record_outputs(&default_nn_transaction_record)
                 .iter()
                 .filter(|o| o.spend_status_query(queried_spend_state))
+                .filter(|&o| o.pool_query(queried_pools))
                 .cloned()
-                .filter(|o| o.pool_query(queried_pools))
                 .collect();
         assert_eq!(requested_outputs.len(), expected);
     }

zingolib/src/wallet/transaction_records_by_id.rs

@@ -538,8 +538,26 @@ impl TransactionRecordsById {
             );
         }
     }
-    /// witness tree requirement:
-    ///
+    pub(crate) fn update_output_index<D: DomainWalletExt>(
+        &mut self,
+        txid: TxId,
+        status: zingo_status::confirmation_status::ConfirmationStatus,
+        timestamp: u64,
+        note: D::Note,
+        output_index: usize,
+    ) {
+        let transaction_record =
+            self.create_modify_get_transaction_metadata(&txid, status, timestamp);
+
+        if let Some(n) = D::WalletNote::transaction_metadata_notes_mut(transaction_record)
+            .iter_mut()
+            .find(|n| n.note() == &note)
+        {
+            if n.output_index().is_none() {
+                *n.output_index_mut() = Some(output_index as u32)
+            }
+        }
+    }
     pub(crate) fn add_pending_note<D: DomainWalletExt>(
         &mut self,
         txid: TxId,
@@ -666,12 +684,13 @@ impl TransactionRecordsById {
     }
 
     /// get a list of spendable NoteIds with associated note values
+    #[allow(clippy::type_complexity)]
     pub(crate) fn get_spendable_note_ids_and_values(
         &self,
         sources: &[zcash_client_backend::ShieldedProtocol],
         anchor_height: zcash_primitives::consensus::BlockHeight,
         exclude: &[NoteId],
-    ) -> Result<Vec<(NoteId, u64)>, Vec<TxId>> {
+    ) -> Result<Vec<(NoteId, u64)>, Vec<(TxId, BlockHeight)>> {
         let mut missing_output_index = vec![];
         let ok = self
             .values()
@@ -685,7 +704,10 @@ impl TransactionRecordsById {
                     {
                         notes_from_tx
                     } else {
-                        missing_output_index.push(transaction_record.txid);
+                        missing_output_index.push((
+                            transaction_record.txid,
+                            transaction_record.status.get_height(),
+                        ));
                         vec![]
                     }
                 } else {

zingolib/src/wallet/transaction_records_by_id/trait_inputsource.rs

@@ -39,7 +39,7 @@ pub enum InputSourceError {
     InvalidValue(BalanceError),
     /// Wallet data is out of date
     #[error("Output index data is missing! Wallet data is out of date, please rescan.")]
-    MissingOutputIndexes(Vec<TxId>),
+    MissingOutputIndexes(Vec<(TxId, zcash_primitives::consensus::BlockHeight)>),
 }
 
 // Calculate remaining difference between target and selected.