refactor!: remove mempool execute, add faucet component, remove creat…

…e free test coins (#1082)

Description
---
- remove duplicate and incorrect state bootstrapping code in engine
- remove CreateFreeTestCoins instruction
- remove mempool transaction validation
- add validation to consensus-level transaction executor
- remove deferred decision (decision is now made until after execution
and locking)
- fixes for local-only-rule concurrent execution
- fix(engine): fix incorrectly marking components as changed when no
state change occurred
- adds XtrFaucet builtin template 
- create funded XtrFaucet component in testnet bootstrap 
- update walletd to call XtrFaucet for free test coins
- fix(consensus): only accept missing transactions if they were
requested
- fix(transaction-submitter): ignore race condition when retrieving tx
results

Motivation and Context
---
CreateFreeTestCoins was a quick way to get test funds. However because
it is the only valid transaction that required no inputs, a number of
edge cases had to be handled. This PR replaces this with a faucet
component that is bootstrapped with funds. This component is only
created for non-mainnet networks.

The mempool execution has a number of limitations e.g. cannot handle
versionless inputs.

Next steps:
- fee claiming does not currently work. This is because we need a bunch
of information from the L1/epoch manager to authorize a claim. We may
need to rethink this e.g. at the end of epoch, we create/update an
UnclaimedVnFee substate which can later be claimed/swept by validators.
The "minting" of the validator fee substate may require co-operation
from multiple shards.
- multishard transactions only previously worked for versioned inputs.
After this PR they will not work at all because they require a substate
pledging protocol. This will be implemented in a subsequent PR.
- there is a small chance that a substate may change shards as the
version changes. This chance needs to be completely removed. The easiest
way would be to simply ignore the last 8 (u32) bytes of a substate
address when mapping to a shard, however that would leave many tiny
"holes" in the shard space where state cannot exist.

How Has This Been Tested?
---
Existing tests.
Manually: 5000 transaction stress test on single shard network

What process can a PR reviewer use to test or verify this change?
---
Run a stress test (remember to regenerate stress test transaction bin
file with new XtrFaucet transaction)
Observe that local-only transactions that share the XtrFaucet vault are
executed in the same block

Breaking Changes
---

- [ ] None
- [x] Requires data directory to be deleted
- [x] Other - CreateFreeTestCoins is no longer valid, so all clients
that submit these will have to change to use the XtrFaucet

applications/tari_dan_app_utilities/src/template_manager/implementation/manager.rs

@@ -40,7 +40,12 @@ use tari_dan_engine::{
 use tari_dan_storage::global::{DbTemplate, DbTemplateType, DbTemplateUpdate, GlobalDb, TemplateStatus};
 use tari_dan_storage_sqlite::global::SqliteGlobalDbAdapter;
 use tari_engine_types::calculate_template_binary_hash;
-use tari_template_builtin::{get_template_builtin, ACCOUNT_NFT_TEMPLATE_ADDRESS, ACCOUNT_TEMPLATE_ADDRESS};
+use tari_template_builtin::{
+    get_template_builtin,
+    ACCOUNT_NFT_TEMPLATE_ADDRESS,
+    ACCOUNT_TEMPLATE_ADDRESS,
+    FAUCET_TEMPLATE_ADDRESS,
+};
 use tari_template_lib::models::TemplateAddress;
 
 use super::TemplateConfig;
@@ -90,36 +95,36 @@ impl<TAddr: NodeAddressable> TemplateManager<TAddr> {
 
     fn load_builtin_templates() -> HashMap<TemplateAddress, Template> {
         // for now, we only load the "account" template
-        let mut builtin_templates = HashMap::new();
+        let mut builtin_templates = HashMap::with_capacity(3);
 
         // get the builtin WASM code of the account template
         let compiled_code = get_template_builtin(&ACCOUNT_TEMPLATE_ADDRESS);
-        let template = Self::load_builtin_template("account", ACCOUNT_TEMPLATE_ADDRESS, compiled_code.to_vec());
+        let template = Self::convert_code_to_template("Account", ACCOUNT_TEMPLATE_ADDRESS, compiled_code.to_vec());
         builtin_templates.insert(ACCOUNT_TEMPLATE_ADDRESS, template);
 
         // get the builtin WASM code of the account nft template
         let compiled_code = get_template_builtin(&ACCOUNT_NFT_TEMPLATE_ADDRESS);
-        let template = Self::load_builtin_template("account_nft", ACCOUNT_NFT_TEMPLATE_ADDRESS, compiled_code.to_vec());
+        let template =
+            Self::convert_code_to_template("AccountNft", ACCOUNT_NFT_TEMPLATE_ADDRESS, compiled_code.to_vec());
         builtin_templates.insert(ACCOUNT_NFT_TEMPLATE_ADDRESS, template);
 
+        // get the builtin WASM code of the account nft template
+        let compiled_code = get_template_builtin(&FAUCET_TEMPLATE_ADDRESS);
+        let template = Self::convert_code_to_template("XtrFaucet", FAUCET_TEMPLATE_ADDRESS, compiled_code.to_vec());
+        builtin_templates.insert(FAUCET_TEMPLATE_ADDRESS, template);
+
         builtin_templates
     }
 
-    fn load_builtin_template(name: &str, address: TemplateAddress, compiled_code: Vec<u8>) -> Template {
-        let compiled_code_len = compiled_code.len();
-        info!(
-            target: LOG_TARGET,
-            "Loading builtin {} template: {} bytes", name, compiled_code_len
-        );
-
+    fn convert_code_to_template(name: &str, address: TemplateAddress, compiled_code: Vec<u8>) -> Template {
         // build the template object of the account template
         let binary_sha = calculate_template_binary_hash(&compiled_code);
         Template {
             metadata: TemplateMetadata {
                 name: name.to_string(),
                 address,
                 url: "".to_string(),
-                binary_sha: binary_sha.to_vec(),
+                binary_sha,
                 height: 0,
             },
             executable: TemplateExecutable::CompiledWasm(compiled_code),

applications/tari_dan_app_utilities/src/template_manager/interface/types.rs

@@ -20,6 +20,7 @@
 //  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 //  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+use tari_common_types::types::FixedHash;
 use tari_dan_storage::global::{DbTemplate, DbTemplateType};
 use tari_template_lib::models::TemplateAddress;
 use tari_validator_node_client::types::TemplateAbi;
@@ -34,7 +35,7 @@ pub struct TemplateMetadata {
     // this must be in the form of "https://example.com/my_template.wasm"
     pub url: String,
     /// SHA hash of binary
-    pub binary_sha: Vec<u8>,
+    pub binary_sha: FixedHash,
     /// Block height in which the template was published
     pub height: u64,
 }
@@ -45,7 +46,9 @@ impl From<TemplateRegistration> for TemplateMetadata {
             name: reg.template_name,
             address: reg.template_address,
             url: reg.registration.binary_url.into_string(),
-            binary_sha: reg.registration.binary_sha.into_vec(),
+            binary_sha: FixedHash::try_from(reg.registration.binary_sha.into_vec())
+                // TODO: impl Fallible conversion
+                .expect("binary_sha must be 32 bytes long"),
             height: reg.mined_height,
         }
     }
@@ -58,7 +61,7 @@ impl From<DbTemplate> for TemplateMetadata {
             name: record.template_name,
             address: (*record.template_address).into(),
             url: record.url,
-            binary_sha: vec![],
+            binary_sha: FixedHash::zero(),
             height: record.height,
         }
     }
@@ -87,7 +90,7 @@ impl From<DbTemplate> for Template {
                 address: (*record.template_address).into(),
                 url: record.url,
                 // TODO: add field to db
-                binary_sha: vec![],
+                binary_sha: FixedHash::zero(),
                 height: record.height,
             },
             executable: match record.template_type {

applications/tari_dan_wallet_daemon/src/handlers/accounts.rs

@@ -17,7 +17,7 @@ use tari_dan_common_types::optional::Optional;
 use tari_dan_wallet_crypto::ConfidentialProofStatement;
 use tari_dan_wallet_sdk::{
     apis::{confidential_transfer::TransferParams, jwt::JrpcPermission, key_manager, substate::ValidatorScanResult},
-    models::{NewAccountInfo, VersionedSubstateId},
+    models::NewAccountInfo,
     storage::WalletStore,
     DanWalletSdk,
 };
@@ -32,6 +32,7 @@ use tari_key_manager::key_manager::DerivedKey;
 use tari_template_builtin::ACCOUNT_TEMPLATE_ADDRESS;
 use tari_template_lib::{
     args,
+    constants::{XTR_FAUCET_COMPONENT_ADDRESS, XTR_FAUCET_VAULT_ADDRESS},
     models::{Amount, UnclaimedConfidentialOutputAddress},
     prelude::CONFIDENTIAL_TARI_RESOURCE_ADDRESS,
 };
@@ -124,11 +125,7 @@ pub async fn handle_create(
     let transaction = Transaction::builder()
         .fee_transaction_pay_from_component(default_account.address.as_component_address().unwrap(), max_fee)
         .create_account(owner_pk.clone())
-        .with_inputs(
-            inputs
-                .iter()
-                .map(|addr| SubstateRequirement::new(addr.substate_id.clone(), Some(addr.version))),
-        )
+        .with_inputs(inputs)
         .sign(&signing_key.key)
         .build();
 
@@ -541,12 +538,8 @@ pub async fn handle_claim_burn(
 
     // Add all versioned account child addresses as inputs
     // add the commitment substate id as input to the claim burn transaction
-    let commitment_substate_address = VersionedSubstateId {
-        substate_id: SubstateId::UnclaimedConfidentialOutput(UnclaimedConfidentialOutputAddress::try_from(
-            commitment.as_slice(),
-        )?),
-        version: 0,
-    };
+    let commitment_substate_address =
+        SubstateRequirement::unversioned(UnclaimedConfidentialOutputAddress::try_from(commitment.as_slice())?);
     inputs.push(commitment_substate_address.clone());
 
     info!(
@@ -561,7 +554,7 @@ pub async fn handle_claim_burn(
         .substate_api()
         .scan_for_substate(
             &commitment_substate_address.substate_id,
-            Some(commitment_substate_address.version),
+            commitment_substate_address.version,
         )
         .await?;
     let output = output.into_unclaimed_confidential_output().unwrap();
@@ -648,7 +641,7 @@ async fn finish_claiming<T: WalletStore>(
     account_address: SubstateId,
     new_account_name: Option<String>,
     sdk: &DanWalletSdk<SqliteWalletStore, IndexerJsonRpcNetworkInterface>,
-    mut inputs: Vec<VersionedSubstateId>,
+    mut inputs: Vec<SubstateRequirement>,
     account_public_key: &RistrettoPublicKey,
     max_fee: Amount,
     account_secret_key: DerivedKey<RistrettoPublicKey>,
@@ -670,7 +663,7 @@ async fn finish_claiming<T: WalletStore>(
     if new_account_name.is_none() {
         // Add all versioned account child addresses as inputs unless the account is new
         let child_addresses = sdk.substate_api().load_dependent_substates(&[&account_address])?;
-        inputs.extend(child_addresses);
+        inputs.extend(child_addresses.into_iter().map(Into::into));
         instructions.push(Instruction::CallMethod {
             component_address: account_component_address,
             method: "deposit".to_string(),
@@ -687,9 +680,6 @@ async fn finish_claiming<T: WalletStore>(
         method: "pay_fee".to_string(),
         args: args![max_fee],
     });
-    let inputs = inputs
-        .into_iter()
-        .map(|s| SubstateRequirement::new(s.substate_id.clone(), Some(s.version)));
     let transaction = Transaction::builder()
         .with_fee_instructions(instructions)
         .with_inputs(inputs)
@@ -727,7 +717,6 @@ async fn finish_claiming<T: WalletStore>(
 }
 
 /// Mints free test coins into an account. If an account name is provided which does not exist, that account is created
-#[allow(clippy::too_many_lines)]
 pub async fn handle_create_free_test_coins(
     context: &HandlerContext,
     token: Option<String>,
@@ -748,24 +737,21 @@ pub async fn handle_create_free_test_coins(
         return Err(invalid_params("fee", Some("cannot be negative")));
     }
 
-    let mut inputs = vec![];
+    let mut inputs = vec![
+        SubstateRequirement::unversioned(XTR_FAUCET_COMPONENT_ADDRESS),
+        SubstateRequirement::unversioned(XTR_FAUCET_VAULT_ADDRESS),
+    ];
     let accounts_api = sdk.accounts_api();
     let (account_address, account_secret_key, new_account_name) =
         get_or_create_account(&account, &accounts_api, key_id, sdk, &mut inputs)?;
 
     let account_public_key = PublicKey::from_secret_key(&account_secret_key.key);
-    let output = sdk
-        .confidential_crypto_api()
-        .generate_output_for_dest(&account_public_key, amount)?;
-
-    let instructions = vec![
-        // TODO: We create double what is expected, amount confidential and amount revealed. Should let the caller
-        //       specify these values separately.
-        Instruction::CreateFreeTestCoins {
-            revealed_amount: amount,
-            output: Some(output),
-        },
-    ];
+
+    let instructions = vec![Instruction::CallMethod {
+        component_address: XTR_FAUCET_COMPONENT_ADDRESS,
+        method: "take".to_string(),
+        args: args![amount],
+    }];
 
     // ------------------------------
     let (tx_id, finalized) = finish_claiming(
@@ -799,7 +785,7 @@ fn get_or_create_account<T: WalletStore>(
     accounts_api: &tari_dan_wallet_sdk::apis::accounts::AccountsApi<'_, T>,
     key_id: Option<u64>,
     sdk: &DanWalletSdk<SqliteWalletStore, IndexerJsonRpcNetworkInterface>,
-    inputs: &mut Vec<VersionedSubstateId>,
+    inputs: &mut Vec<SubstateRequirement>,
 ) -> Result<(SubstateId, DerivedKey<RistrettoPublicKey>, Option<String>), anyhow::Error> {
     let maybe_account = match account {
         Some(ref addr_or_name) => get_account(addr_or_name, accounts_api).optional()?,
@@ -819,7 +805,7 @@ fn get_or_create_account<T: WalletStore>(
                 .key_manager_api()
                 .derive_key(key_manager::TRANSACTION_BRANCH, key_index)?;
             let account_substate = sdk.substate_api().get_substate(&account.address)?;
-            inputs.push(account_substate.address);
+            inputs.push(account_substate.address.into());
 
             (account.address, account_secret_key, None)
         },

applications/tari_dan_wallet_daemon/src/handlers/nfts.rs

@@ -180,15 +180,11 @@ async fn mint_account_nft(
     let sdk = context.wallet_sdk();
     sdk.jwt_api().check_auth(token, &[JrpcPermission::Admin])?;
 
-    let inputs = sdk
+    let mut inputs = sdk
         .substate_api()
         .locate_dependent_substates(&[account.address.clone()])
         .await?;
 
-    let mut inputs = inputs
-        .iter()
-        .map(|v| SubstateRequirement::new(v.substate_id.clone(), Some(v.version)))
-        .collect::<Vec<_>>();
     inputs.extend([SubstateRequirement::new(SubstateId::Component(component_address), None)]);
 
     let instructions = vec![
@@ -252,9 +248,6 @@ async fn create_account_nft(
         .substate_api()
         .locate_dependent_substates(&[account.address.clone()])
         .await?;
-    let inputs = inputs
-        .iter()
-        .map(|addr| SubstateRequirement::new(addr.substate_id.clone(), Some(addr.version)));
 
     let transaction = Transaction::builder()
         .fee_transaction_pay_from_component(account.address.as_component_address().unwrap(), fee)

applications/tari_dan_wallet_daemon/src/handlers/transaction.rs

@@ -121,13 +121,7 @@ pub async fn handle_submit(
                 .unwrap_or(&req.fee_instructions),
         )?);
         let substates = substates.into_iter().collect::<Vec<_>>();
-        let loaded_dependent_substates = sdk
-            .substate_api()
-            .locate_dependent_substates(&substates)
-            .await?
-            .into_iter()
-            .map(Into::into)
-            .collect();
+        let loaded_dependent_substates = sdk.substate_api().locate_dependent_substates(&substates).await?;
         [req.inputs, loaded_dependent_substates].concat()
     };
 

applications/tari_dan_wallet_web_ui/src/routes/AssetVault/Components/ActionMenu.tsx

@@ -37,7 +37,7 @@ function ActionMenu() {
   const onClaimFreeCoins = () => {
     mutate({
       accountName: accountName,
-      amount: 100000,
+      amount: 200000,
       fee: 1000,
     });
   };

applications/tari_dan_wallet_web_ui/src/routes/Onboarding/Onboarding.tsx

@@ -45,7 +45,7 @@ function Onboarding() {
     mutate(
       {
         accountName: accountFormState.accountName,
-        amount: 100000,
+        amount: 200000,
         fee: 1000,
       },
       {

applications/tari_dan_wallet_web_ui/src/routes/Wallet/Components/Accounts.tsx

@@ -161,7 +161,7 @@ function Accounts() {
   const onClaimFreeCoins = async () => {
     await mutateCreateFeeTestCoins({
       accountName: "TestAccount",
-      amount: 100000,
+      amount: 200000,
       fee: 1000,
     });
   };

applications/tari_indexer/src/dry_run/processor.rs

@@ -29,11 +29,7 @@ use tari_dan_app_utilities::{
     transaction_executor::{TariDanTransactionProcessor, TransactionExecutor as _},
 };
 use tari_dan_common_types::{Epoch, PeerAddress, SubstateAddress};
-use tari_dan_engine::{
-    bootstrap_state,
-    fees::FeeTable,
-    state_store::{memory::MemoryStateStore, AtomicDb, StateWriter},
-};
+use tari_dan_engine::{fees::FeeTable, state_store::new_memory_store};
 use tari_engine_types::{
     commit_result::ExecuteResult,
     instruction::Instruction,
@@ -115,7 +111,7 @@ where TSubstateCache: SubstateCache + 'static
 
         let virtual_substates = self.get_virtual_substates(&transaction, epoch).await?;
 
-        let state_store = new_state_store();
+        let state_store = new_memory_store();
         state_store.set_many(found_substates)?;
 
         // execute the payload in the WASM engine and return the result
@@ -271,11 +267,3 @@ where TSubstateCache: SubstateCache + 'static
         Ok(virtual_substates)
     }
 }
-
-fn new_state_store() -> MemoryStateStore {
-    let state_store = MemoryStateStore::new();
-    let mut tx = state_store.write_access().unwrap();
-    bootstrap_state(&mut tx).unwrap();
-    tx.commit().unwrap();
-    state_store
-}

applications/tari_indexer/src/json_rpc/handlers.rs

@@ -617,7 +617,7 @@ impl JsonRpcHandlers {
                     name: t.name,
                     address: t.address,
                     url: t.url,
-                    binary_sha: to_hex(&t.binary_sha),
+                    binary_sha: to_hex(t.binary_sha.as_slice()),
                     height: t.height,
                 })
                 .collect(),