fix(consensus)!: defer proposing txs with conflicting unversioned inp…

…uts (#1125) Description --- fix(consensus)!: defer proposing txs with conflicting unversioned inputs refactor: locks take into account if the requested input was versioned or not Motivation and Context --- A transaction may provide an unversioned input. In the multi-shard case, if this input is locked by another transaction, the first transaction should not be proposed until that lock is cleared. How Has This Been Tested? --- Manually, submitting transactions with conflicting unversioned inputs. What process can a PR reviewer use to test or verify this change? --- Submit transactions with transaction_generator and transaction_submitter. The default generated transactions use unversioned inputs. Breaking Changes --- - [ ] None - [x] Requires data directory to be deleted - [ ] Other - Please specify
tari-project · Sep 3, 2024 · b3f1507 · b3f1507
1 parent f91bcd9
commit b3f1507
Show file tree

Hide file tree

Showing 98 changed files with 906 additions and 593 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/applications/tari_dan_app_utilities/src/base_layer_scanner.rs b/applications/tari_dan_app_utilities/src/base_layer_scanner.rs
@@ -43,7 +43,7 @@ use tari_crypto::{
     ristretto::RistrettoPublicKey,
     tari_utilities::{hex::Hex, ByteArray},
 };
-use tari_dan_common_types::{optional::Optional, NodeAddressable};
+use tari_dan_common_types::{optional::Optional, NodeAddressable, VersionedSubstateId};
 use tari_dan_storage::{
     consensus_models::{BurntUtxo, SubstateRecord},
     global::{GlobalDb, MetadataKey},
@@ -59,7 +59,6 @@ use tari_epoch_manager::{base_layer::EpochManagerHandle, EpochManagerError, Epoc
 use tari_shutdown::ShutdownSignal;
 use tari_state_store_sqlite::SqliteStateStore;
 use tari_template_lib::models::{EncryptedData, TemplateAddress, UnclaimedConfidentialOutputAddress};
-use tari_transaction::VersionedSubstateId;
 use tokio::{task, task::JoinHandle, time};
 
 use crate::{

diff --git a/applications/tari_dan_app_utilities/src/transaction_executor.rs b/applications/tari_dan_app_utilities/src/transaction_executor.rs
@@ -8,22 +8,23 @@ use log::*;
 use tari_common::configuration::Network;
 use tari_common_types::types::PublicKey;
 use tari_crypto::tari_utilities::ByteArray;
-use tari_dan_common_types::services::template_provider::TemplateProvider;
+use tari_dan_common_types::{
+    services::template_provider::TemplateProvider,
+    SubstateLockType,
+    SubstateRequirement,
+    VersionedSubstateId,
+};
 use tari_dan_engine::{
     fees::{FeeModule, FeeTable},
     runtime::{AuthParams, RuntimeModule},
     state_store::{memory::MemoryStateStore, StateStoreError},
     template::LoadedTemplate,
     transaction::{TransactionError, TransactionProcessor},
 };
-use tari_dan_storage::consensus_models::{SubstateLockType, VersionedSubstateIdLockIntent};
-use tari_engine_types::{
-    commit_result::ExecuteResult,
-    substate::{Substate, SubstateId},
-    virtual_substate::VirtualSubstates,
-};
+use tari_dan_storage::consensus_models::VersionedSubstateIdLockIntent;
+use tari_engine_types::{commit_result::ExecuteResult, substate::Substate, virtual_substate::VirtualSubstates};
 use tari_template_lib::{crypto::RistrettoPublicKeyBytes, prelude::NonFungibleAddress};
-use tari_transaction::{Transaction, VersionedSubstateId};
+use tari_transaction::Transaction;
 
 const _LOG_TARGET: &str = "tari::dan::transaction_executor";
 
@@ -45,20 +46,23 @@ pub struct ExecutionOutput {
 }
 
 impl ExecutionOutput {
-    pub fn resolve_inputs(&self, inputs: &IndexMap<SubstateId, Substate>) -> Vec<VersionedSubstateIdLockIntent> {
+    pub fn resolve_inputs(
+        &self,
+        inputs: &IndexMap<SubstateRequirement, Substate>,
+    ) -> Vec<VersionedSubstateIdLockIntent> {
         if let Some(diff) = self.result.finalize.accept() {
             inputs
                 .iter()
-                .map(|(substate_id, substate)| {
-                    let lock_flag = if diff.down_iter().any(|(id, _)| id == substate_id) {
+                .map(|(substate_req, substate)| {
+                    let lock_flag = if diff.down_iter().any(|(id, _)| id == substate_req.substate_id()) {
                         // Update all inputs that were DOWNed to be write locked
                         SubstateLockType::Write
                     } else {
                         // Any input not downed, gets a read lock
                         SubstateLockType::Read
                     };
                     VersionedSubstateIdLockIntent::new(
-                        VersionedSubstateId::new(substate_id.clone(), substate.version()),
+                        VersionedSubstateId::new(substate_req.substate_id().clone(), substate.version()),
                         lock_flag,
                     )
                 })
@@ -68,9 +72,9 @@ impl ExecutionOutput {
             // shards involved but do not lock them. We dont actually lock anything for rejected transactions anyway.
             inputs
                 .iter()
-                .map(|(substate_id, substate)| {
+                .map(|(substate_req, substate)| {
                     VersionedSubstateIdLockIntent::new(
-                        VersionedSubstateId::new(substate_id.clone(), substate.version()),
+                        VersionedSubstateId::new(substate_req.substate_id().clone(), substate.version()),
                         SubstateLockType::Read,
                     )
                 })

diff --git a/applications/tari_dan_wallet_cli/src/command/transaction.rs b/applications/tari_dan_wallet_cli/src/command/transaction.rs
@@ -34,7 +34,7 @@ use anyhow::anyhow;
 use clap::{Args, Subcommand};
 use tari_bor::decode_exact;
 use tari_common_types::types::PublicKey;
-use tari_dan_common_types::{Epoch, SubstateAddress};
+use tari_dan_common_types::{Epoch, SubstateAddress, SubstateRequirement};
 use tari_dan_engine::abi::Type;
 use tari_dan_wallet_sdk::apis::confidential_transfer::ConfidentialTransferInputSelection;
 use tari_engine_types::{
@@ -53,7 +53,7 @@ use tari_template_lib::{
     models::{Amount, BucketId, NonFungibleAddress, NonFungibleId},
     prelude::ResourceAddress,
 };
-use tari_transaction::{SubstateRequirement, TransactionId};
+use tari_transaction::TransactionId;
 use tari_transaction_manifest::{parse_manifest, ManifestValue};
 use tari_utilities::{hex::to_hex, ByteArray};
 use tari_wallet_daemon_client::{

diff --git a/applications/tari_dan_wallet_daemon/src/handlers/accounts.rs b/applications/tari_dan_wallet_daemon/src/handlers/accounts.rs
@@ -13,7 +13,7 @@ use tari_crypto::{
     ristretto::{RistrettoComSig, RistrettoPublicKey},
     tari_utilities::ByteArray,
 };
-use tari_dan_common_types::optional::Optional;
+use tari_dan_common_types::{optional::Optional, SubstateRequirement};
 use tari_dan_wallet_crypto::ConfidentialProofStatement;
 use tari_dan_wallet_sdk::{
     apis::{confidential_transfer::TransferParams, jwt::JrpcPermission, key_manager, substate::ValidatorScanResult},
@@ -36,7 +36,7 @@ use tari_template_lib::{
     models::{Amount, UnclaimedConfidentialOutputAddress},
     prelude::CONFIDENTIAL_TARI_RESOURCE_ADDRESS,
 };
-use tari_transaction::{SubstateRequirement, Transaction};
+use tari_transaction::Transaction;
 use tari_wallet_daemon_client::{
     types::{
         AccountGetDefaultRequest,

diff --git a/applications/tari_dan_wallet_daemon/src/handlers/nfts.rs b/applications/tari_dan_wallet_daemon/src/handlers/nfts.rs
@@ -7,6 +7,7 @@ use anyhow::anyhow;
 use log::info;
 use tari_common_types::types::PublicKey;
 use tari_crypto::{keys::PublicKey as PK, ristretto::RistrettoSecretKey, tari_utilities::ByteArray};
+use tari_dan_common_types::SubstateRequirement;
 use tari_dan_wallet_sdk::{
     apis::{jwt::JrpcPermission, key_manager},
     models::Account,
@@ -18,7 +19,7 @@ use tari_template_lib::{
     crypto::RistrettoPublicKeyBytes,
     prelude::{Amount, ComponentAddress, Metadata, NonFungibleAddress, NonFungibleId, ResourceAddress},
 };
-use tari_transaction::{SubstateRequirement, Transaction, TransactionId};
+use tari_transaction::{Transaction, TransactionId};
 use tari_wallet_daemon_client::types::{
     GetAccountNftRequest,
     GetAccountNftResponse,

diff --git a/applications/tari_dan_wallet_daemon/src/indexer_jrpc_impl.rs b/applications/tari_dan_wallet_daemon/src/indexer_jrpc_impl.rs
@@ -5,7 +5,7 @@ use std::sync::{Arc, Mutex};
 
 use axum::async_trait;
 use reqwest::{IntoUrl, Url};
-use tari_dan_common_types::{optional::IsNotFoundError, substate_type::SubstateType};
+use tari_dan_common_types::{optional::IsNotFoundError, substate_type::SubstateType, SubstateRequirement};
 use tari_dan_wallet_sdk::network::{
     SubstateListItem,
     SubstateListResult,
@@ -28,7 +28,7 @@ use tari_indexer_client::{
     },
 };
 use tari_template_lib::models::TemplateAddress;
-use tari_transaction::{SubstateRequirement, Transaction, TransactionId};
+use tari_transaction::{Transaction, TransactionId};
 use url::ParseError;
 
 #[derive(Debug, Clone)]

diff --git a/applications/tari_dan_wallet_daemon/src/services/transaction_service/handle.rs b/applications/tari_dan_wallet_daemon/src/services/transaction_service/handle.rs
@@ -1,9 +1,10 @@
 //   Copyright 2024 The Tari Project
 //   SPDX-License-Identifier: BSD-3-Clause
 
+use tari_dan_common_types::SubstateRequirement;
 use tari_dan_wallet_sdk::models::NewAccountInfo;
 use tari_engine_types::commit_result::ExecuteResult;
-use tari_transaction::{SubstateRequirement, Transaction, TransactionId};
+use tari_transaction::{Transaction, TransactionId};
 use tokio::sync::{mpsc, oneshot};
 
 use super::TransactionServiceError;

diff --git a/applications/tari_dan_wallet_daemon/src/services/transaction_service/service.rs b/applications/tari_dan_wallet_daemon/src/services/transaction_service/service.rs
@@ -4,7 +4,7 @@
 use std::{sync::Arc, time::Duration};
 
 use log::*;
-use tari_dan_common_types::optional::IsNotFoundError;
+use tari_dan_common_types::{optional::IsNotFoundError, SubstateRequirement};
 use tari_dan_wallet_sdk::{
     models::{NewAccountInfo, TransactionStatus},
     network::WalletNetworkInterface,
@@ -13,7 +13,7 @@ use tari_dan_wallet_sdk::{
 };
 use tari_engine_types::commit_result::ExecuteResult;
 use tari_shutdown::ShutdownSignal;
-use tari_transaction::{SubstateRequirement, Transaction, TransactionId};
+use tari_transaction::{Transaction, TransactionId};
 use tokio::{
     sync::{mpsc, watch, Semaphore},
     time,

diff --git a/applications/tari_indexer/src/dry_run/processor.rs b/applications/tari_indexer/src/dry_run/processor.rs
@@ -28,7 +28,7 @@ use tari_dan_app_utilities::{
     template_manager::implementation::TemplateManager,
     transaction_executor::{TariDanTransactionProcessor, TransactionExecutor as _},
 };
-use tari_dan_common_types::{Epoch, PeerAddress, SubstateAddress};
+use tari_dan_common_types::{Epoch, PeerAddress, SubstateAddress, SubstateRequirement};
 use tari_dan_engine::{fees::FeeTable, state_store::new_memory_store};
 use tari_engine_types::{
     commit_result::ExecuteResult,
@@ -42,7 +42,7 @@ use tari_indexer_lib::{
     substate_scanner::SubstateScanner,
     transaction_autofiller::TransactionAutofiller,
 };
-use tari_transaction::{SubstateRequirement, Transaction};
+use tari_transaction::Transaction;
 use tari_validator_node_rpc::client::{
     SubstateResult,
     TariValidatorNodeRpcClientFactory,

diff --git a/applications/tari_indexer/src/transaction_manager/mod.rs b/applications/tari_indexer/src/transaction_manager/mod.rs
@@ -29,6 +29,8 @@ use tari_dan_common_types::{
     optional::{IsNotFoundError, Optional},
     NodeAddressable,
     SubstateAddress,
+    SubstateRequirement,
+    ToSubstateAddress,
 };
 use tari_engine_types::substate::SubstateId;
 use tari_epoch_manager::EpochManagerReader;
@@ -37,7 +39,7 @@ use tari_indexer_lib::{
     substate_scanner::SubstateScanner,
     transaction_autofiller::TransactionAutofiller,
 };
-use tari_transaction::{SubstateRequirement, Transaction, TransactionId};
+use tari_transaction::{Transaction, TransactionId};
 use tari_validator_node_rpc::client::{
     SubstateResult,
     TransactionResultStatus,

diff --git a/applications/tari_validator_node/src/bootstrap.rs b/applications/tari_validator_node/src/bootstrap.rs
@@ -49,7 +49,16 @@ use tari_dan_app_utilities::{
     template_manager::{implementation::TemplateManager, interface::TemplateManagerHandle},
     transaction_executor::TariDanTransactionProcessor,
 };
-use tari_dan_common_types::{shard::Shard, Epoch, NodeAddressable, NodeHeight, NumPreshards, PeerAddress, ShardGroup};
+use tari_dan_common_types::{
+    shard::Shard,
+    Epoch,
+    NodeAddressable,
+    NodeHeight,
+    NumPreshards,
+    PeerAddress,
+    ShardGroup,
+    VersionedSubstateId,
+};
 use tari_dan_engine::fees::FeeTable;
 use tari_dan_p2p::TariMessagingSpec;
 use tari_dan_storage::{
@@ -86,7 +95,7 @@ use tari_template_lib::{
     prelude::{ComponentAccessRules, OwnerRule, ResourceType},
     resource::TOKEN_SYMBOL,
 };
-use tari_transaction::{Transaction, VersionedSubstateId};
+use tari_transaction::Transaction;
 use tari_validator_node_rpc::client::TariValidatorNodeRpcClientFactory;
 use tokio::{sync::mpsc, task::JoinHandle};
 

diff --git a/applications/tari_validator_node/src/consensus/block_transaction_executor.rs b/applications/tari_validator_node/src/consensus/block_transaction_executor.rs
@@ -7,11 +7,11 @@ use indexmap::IndexMap;
 use log::info;
 use tari_consensus::traits::{BlockTransactionExecutor, BlockTransactionExecutorError};
 use tari_dan_app_utilities::transaction_executor::TransactionExecutor;
-use tari_dan_common_types::Epoch;
+use tari_dan_common_types::{Epoch, SubstateRequirement};
 use tari_dan_engine::state_store::{memory::MemoryStateStore, new_memory_store, AtomicDb, StateWriter};
 use tari_dan_storage::{consensus_models::ExecutedTransaction, StateStore};
 use tari_engine_types::{
-    substate::{Substate, SubstateId},
+    substate::Substate,
     virtual_substate::{VirtualSubstate, VirtualSubstateId, VirtualSubstates},
 };
 use tari_transaction::Transaction;
@@ -38,7 +38,7 @@ where TExecutor: TransactionExecutor
 
     fn add_substates_to_memory_db(
         &self,
-        inputs: &IndexMap<SubstateId, Substate>,
+        inputs: &IndexMap<SubstateRequirement, Substate>,
         out: &MemoryStateStore,
     ) -> Result<(), BlockTransactionExecutorError> {
         // TODO: pass the impl SubstateStore directly into the engine
@@ -47,7 +47,7 @@ where TExecutor: TransactionExecutor
             .map_err(|e| BlockTransactionExecutorError::StateStoreError(e.to_string()))?;
         for (id, substate) in inputs {
             access
-                .set_state(id, substate)
+                .set_state(id.substate_id(), substate)
                 .map_err(|e| BlockTransactionExecutorError::StateStoreError(e.to_string()))?;
         }
         access
@@ -81,7 +81,7 @@ where
         &self,
         transaction: Transaction,
         current_epoch: Epoch,
-        resolved_inputs: &IndexMap<SubstateId, Substate>,
+        resolved_inputs: &IndexMap<SubstateRequirement, Substate>,
     ) -> Result<ExecutedTransaction, BlockTransactionExecutorError> {
         let id = *transaction.id();
 

diff --git a/applications/tari_validator_node/src/p2p/rpc/service_impl.rs b/applications/tari_validator_node/src/p2p/rpc/service_impl.rs
@@ -309,10 +309,15 @@ impl ValidatorNodeRpcService for ValidatorNodeRpcServiceImpl {
     }
 
     async fn get_high_qc(&self, _request: Request<GetHighQcRequest>) -> Result<Response<GetHighQcResponse>, RpcStatus> {
+        let current_epoch = self
+            .epoch_manager
+            .current_epoch()
+            .await
+            .map_err(RpcStatus::log_internal_error(LOG_TARGET))?;
         let high_qc = self
             .shard_state_store
             .with_read_tx(|tx| {
-                HighQc::get(tx)
+                HighQc::get(tx, current_epoch)
                     .optional()?
                     .map(|hqc| hqc.get_quorum_certificate(tx))
                     .transpose()

diff --git a/applications/tari_validator_node/src/p2p/services/mempool/service.rs b/applications/tari_validator_node/src/p2p/services/mempool/service.rs
@@ -23,7 +23,14 @@
 use std::{collections::HashSet, fmt::Display, iter};
 
 use log::*;
-use tari_dan_common_types::{optional::Optional, NumPreshards, PeerAddress, ShardGroup, SubstateAddress};
+use tari_dan_common_types::{
+    optional::Optional,
+    NumPreshards,
+    PeerAddress,
+    ShardGroup,
+    SubstateAddress,
+    ToSubstateAddress,
+};
 use tari_dan_p2p::{DanMessage, NewTransactionMessage};
 use tari_dan_storage::{consensus_models::TransactionRecord, StateStore};
 use tari_engine_types::commit_result::RejectReason;

diff --git a/applications/tari_validator_node/src/p2p/services/mempool/traits.rs b/applications/tari_validator_node/src/p2p/services/mempool/traits.rs
@@ -5,12 +5,12 @@ use std::collections::HashSet;
 
 use async_trait::async_trait;
 use indexmap::IndexMap;
-use tari_dan_common_types::Epoch;
+use tari_dan_common_types::{Epoch, SubstateRequirement};
 use tari_engine_types::{
     substate::{Substate, SubstateId},
     virtual_substate::VirtualSubstates,
 };
-use tari_transaction::{SubstateRequirement, Transaction};
+use tari_transaction::Transaction;
 
 pub struct ResolvedSubstates {
     pub local: IndexMap<SubstateId, Substate>,