diff --git a/integration-tests/src/test_loop/tests/congestion_control.rs b/integration-tests/src/test_loop/tests/congestion_control.rs index 57c11454ab3..1fd3cbbb757 100644 --- a/integration-tests/src/test_loop/tests/congestion_control.rs +++ b/integration-tests/src/test_loop/tests/congestion_control.rs @@ -114,7 +114,7 @@ fn do_deploy_contract( let code = near_test_contracts::rs_contract().to_vec(); let tx = deploy_contract(test_loop, node_datas, rpc_id, contract_id, code, 1); test_loop.run_for(Duration::seconds(5)); - check_txs(&*test_loop, node_datas, rpc_id, &[tx]); + check_txs(&test_loop.data, node_datas, rpc_id, &[tx]); } /// Call the contract from all accounts and wait until the transactions are executed. @@ -144,7 +144,7 @@ fn do_call_contract( txs.push(tx); } test_loop.run_for(Duration::seconds(20)); - check_txs(&*test_loop, node_datas, &rpc_id, &txs); + check_txs(&test_loop.data, node_datas, &rpc_id, &txs); } /// The condition that can be used for the test loop to wait until the chain diff --git a/integration-tests/src/test_loop/tests/contract_distribution_cross_shard.rs b/integration-tests/src/test_loop/tests/contract_distribution_cross_shard.rs index ae37fabd066..af89ff790f8 100644 --- a/integration-tests/src/test_loop/tests/contract_distribution_cross_shard.rs +++ b/integration-tests/src/test_loop/tests/contract_distribution_cross_shard.rs @@ -143,7 +143,7 @@ fn deploy_contracts( contracts.push(contract); } env.test_loop.run_for(Duration::seconds(2)); - check_txs(&env.test_loop, &env.datas, rpc_id, &txs); + check_txs(&env.test_loop.data, &env.datas, rpc_id, &txs); contracts } @@ -175,5 +175,5 @@ fn call_contracts( } } env.test_loop.run_for(Duration::seconds(2)); - check_txs(&env.test_loop, &env.datas, &rpc_id, &txs); + check_txs(&env.test_loop.data, &env.datas, &rpc_id, &txs); } diff --git a/integration-tests/src/test_loop/tests/create_delete_account.rs b/integration-tests/src/test_loop/tests/create_delete_account.rs index 66a87355e82..22681d07ed5 100644 --- a/integration-tests/src/test_loop/tests/create_delete_account.rs +++ b/integration-tests/src/test_loop/tests/create_delete_account.rs @@ -33,7 +33,7 @@ fn do_call_contract(env: &mut TestLoopEnv, rpc_id: &AccountId, contract_id: &Acc nonce, ); env.test_loop.run_for(Duration::seconds(5)); - check_txs(&env.test_loop, &env.datas, rpc_id, &[tx]); + check_txs(&env.test_loop.data, &env.datas, rpc_id, &[tx]); } /// Tracks latest block heights and checks that all chunks are produced. diff --git a/integration-tests/src/test_loop/tests/max_receipt_size.rs b/integration-tests/src/test_loop/tests/max_receipt_size.rs index 9d386a608bd..7d11ae4ceab 100644 --- a/integration-tests/src/test_loop/tests/max_receipt_size.rs +++ b/integration-tests/src/test_loop/tests/max_receipt_size.rs @@ -31,7 +31,7 @@ fn slow_test_max_receipt_size() { &account0, vec![0u8; 2_000_000], account0_signer, - get_shared_block_hash(&env.datas, &env.test_loop), + get_shared_block_hash(&env.datas, &env.test_loop.data), ); let large_tx_exec_res = execute_tx(&mut env.test_loop, &rpc_id, large_tx, &env.datas, Duration::seconds(5)); @@ -43,7 +43,7 @@ fn slow_test_max_receipt_size() { &account0, near_test_contracts::rs_contract().into(), &account0_signer, - get_shared_block_hash(&env.datas, &env.test_loop), + get_shared_block_hash(&env.datas, &env.test_loop.data), ); run_tx(&mut env.test_loop, &rpc_id, deploy_contract_tx, &env.datas, Duration::seconds(5)); @@ -59,7 +59,7 @@ fn slow_test_max_receipt_size() { "generate_large_receipt".into(), r#"{"account_id": "account0", "method_name": "noop", "total_args_size": 3000000}"#.into(), 300 * TGAS, - get_shared_block_hash(&env.datas, &env.test_loop), + get_shared_block_hash(&env.datas, &env.test_loop.data), ); run_tx(&mut env.test_loop, &rpc_id, large_receipt_tx, &env.datas, Duration::seconds(5)); @@ -73,7 +73,7 @@ fn slow_test_max_receipt_size() { "generate_large_receipt".into(), r#"{"account_id": "account0", "method_name": "noop", "total_args_size": 5000000}"#.into(), 300 * TGAS, - get_shared_block_hash(&env.datas, &env.test_loop), + get_shared_block_hash(&env.datas, &env.test_loop.data), ); let too_large_receipt_tx_exec_res = execute_tx( &mut env.test_loop, @@ -115,7 +115,7 @@ fn slow_test_max_receipt_size() { "sum_n".into(), 5_u64.to_le_bytes().to_vec(), 300 * TGAS, - get_shared_block_hash(&env.datas, &env.test_loop), + get_shared_block_hash(&env.datas, &env.test_loop.data), ); let sum_4_res = run_tx(&mut env.test_loop, &rpc_id, sum_4_tx, &env.datas, Duration::seconds(5)); assert_eq!(sum_4_res, 10u64.to_le_bytes().to_vec()); diff --git a/integration-tests/src/test_loop/tests/resharding_v3.rs b/integration-tests/src/test_loop/tests/resharding_v3.rs index 4c0864056f0..f6064417095 100644 --- a/integration-tests/src/test_loop/tests/resharding_v3.rs +++ b/integration-tests/src/test_loop/tests/resharding_v3.rs @@ -1,48 +1,38 @@ +use assert_matches::assert_matches; use itertools::Itertools; use near_async::test_loop::data::TestLoopData; use near_async::time::Duration; use near_chain_configs::test_genesis::{TestGenesisBuilder, ValidatorsSpec}; -use near_chain_configs::DEFAULT_GC_NUM_EPOCHS_TO_KEEP; -use near_client::Query; use near_o11y::testonly::init_test_logger; use near_primitives::epoch_manager::EpochConfigStore; use near_primitives::shard_layout::ShardLayout; -use near_primitives::types::{ - AccountId, BlockHeightDelta, BlockId, BlockReference, Gas, ShardId, ShardIndex, -}; +use near_primitives::types::{AccountId, BlockHeightDelta, ShardId, ShardIndex}; use near_primitives::version::{ProtocolFeature, PROTOCOL_VERSION}; -use rand::seq::SliceRandom; -use rand::Rng; -use rand::SeedableRng; -use rand_chacha::ChaCha20Rng; use std::cell::Cell; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use crate::test_loop::builder::TestLoopBuilder; -use crate::test_loop::env::{TestData, TestLoopEnv}; use crate::test_loop::utils::loop_action::{LoopAction, LoopActionStatus}; use crate::test_loop::utils::receipts::{ check_receipts_presence_after_resharding_block, check_receipts_presence_at_resharding_block, ReceiptKind, }; -use crate::test_loop::utils::sharding::{ - next_block_has_new_shard_layout, print_and_assert_shard_accounts, +#[cfg(feature = "test_features")] +use crate::test_loop::utils::resharding::fork_before_resharding_block; +use crate::test_loop::utils::resharding::{ + call_burn_gas_contract, call_promise_yield, execute_money_transfers, + temporary_account_during_resharding, }; +use crate::test_loop::utils::sharding::print_and_assert_shard_accounts; use crate::test_loop::utils::transactions::{ - check_txs, create_account, delete_account, deploy_contract, get_anchor_hash, get_next_nonce, - get_smallest_height_head, store_and_submit_tx, submit_tx, + check_txs, create_account, deploy_contract, get_smallest_height_head, }; use crate::test_loop::utils::trie_sanity::{ check_state_shard_uid_mapping_after_resharding, TrieSanityCheck, }; -use crate::test_loop::utils::{get_node_data, retrieve_client_actor, ONE_NEAR, TGAS}; -use assert_matches::assert_matches; -use near_crypto::Signer; +use crate::test_loop::utils::{ONE_NEAR, TGAS}; use near_parameters::{vm, RuntimeConfig, RuntimeConfigStore}; -use near_primitives::test_utils::create_user_test_signer; -use near_primitives::transaction::SignedTransaction; -use near_primitives::views::{FinalExecutionStatus, QueryRequest}; /// Default and minimal epoch length used in resharding tests. const DEFAULT_EPOCH_LENGTH: u64 = 6; @@ -82,10 +72,13 @@ struct TestReshardingParameters { validators: Vec, #[builder(setter(skip))] rpcs: Vec, + // Index of the client used to serve requests (RPC node if available, otherwise first from `clients`) #[builder(setter(skip))] - rpc_client_index: Option, + client_index: usize, #[builder(setter(skip))] archivals: Vec, + #[builder(setter(skip))] + new_boundary_account: AccountId, initial_balance: u128, epoch_length: BlockHeightDelta, chunk_ranges_to_drop: HashMap>, @@ -113,6 +106,11 @@ struct TestReshardingParameters { // TODO(resharding) Remove this when negative refcounts are properly handled. /// Whether to allow negative refcount being a result of the database update. allow_negative_refcount: bool, + /// If not disabled, use testloop action that will delete an account after resharding + /// and check that the account is accessible through archival node but not through a regular node. + disable_temporary_account_test: bool, + #[builder(setter(skip))] + temporary_account_id: AccountId, } impl TestReshardingParametersBuilder { @@ -159,17 +157,34 @@ impl TestReshardingParametersBuilder { let validators = validators.to_vec(); let (rpcs, tmp) = tmp.split_at(num_rpcs as usize); let rpcs = rpcs.to_vec(); - let rpc_client_index = - rpcs.first().map(|_| num_producers as usize + num_validators as usize); let (archivals, _) = tmp.split_at(num_archivals as usize); let archivals = archivals.to_vec(); + let client_index = + if rpcs.is_empty() { 0 } else { num_producers + num_validators } as usize; + let client_id = clients[client_index].clone(); + println!("Clients setup:"); println!("Producers: {producers:?}"); println!("Validators: {validators:?}"); - println!("Rpcs: {rpcs:?}, first RPC node uses client at index: {rpc_client_index:?}"); + println!("Rpcs: {rpcs:?}, to serve requests we use client: {client_id}"); println!("Archivals: {archivals:?}"); + let new_boundary_account: AccountId = "account6".parse().unwrap(); + let temporary_account_id: AccountId = + format!("{}.{}", new_boundary_account, new_boundary_account).parse().unwrap(); + let mut loop_actions = self.loop_actions.unwrap_or_default(); + let disable_temporary_account_test = self.disable_temporary_account_test.unwrap_or(false); + if !disable_temporary_account_test { + let archival_id = archivals.iter().next().cloned(); + loop_actions.push(temporary_account_during_resharding( + archival_id, + client_id, + new_boundary_account.clone(), + temporary_account_id.clone(), + )); + } + TestReshardingParameters { base_shard_layout_version: self.base_shard_layout_version.unwrap_or(2), num_accounts, @@ -183,8 +198,9 @@ impl TestReshardingParametersBuilder { producers, validators, rpcs, - rpc_client_index, + client_index, archivals, + new_boundary_account, initial_balance: self.initial_balance.unwrap_or(1_000_000 * ONE_NEAR), epoch_length, chunk_ranges_to_drop: self.chunk_ranges_to_drop.unwrap_or_default(), @@ -195,13 +211,15 @@ impl TestReshardingParametersBuilder { load_mem_tries_for_tracked_shards: self .load_mem_tries_for_tracked_shards .unwrap_or(true), - loop_actions: self.loop_actions.unwrap_or_default(), + loop_actions, all_chunks_expected: self.all_chunks_expected.unwrap_or(false), deploy_test_contract: self.deploy_test_contract.unwrap_or_default(), limit_outgoing_gas: self.limit_outgoing_gas.unwrap_or(false), delay_flat_state_resharding: self.delay_flat_state_resharding.unwrap_or(0), short_yield_timeout: self.short_yield_timeout.unwrap_or(false), allow_negative_refcount: self.allow_negative_refcount.unwrap_or(false), + disable_temporary_account_test, + temporary_account_id, } } @@ -222,334 +240,6 @@ impl TestReshardingParametersBuilder { } } -// Returns a callable function that, when invoked inside a test loop iteration, can force the creation of a chain fork. -#[cfg(feature = "test_features")] -fn fork_before_resharding_block(double_signing: bool) -> LoopAction { - use crate::test_loop::utils::retrieve_client_actor; - use near_client::client_actor::AdvProduceBlockHeightSelection; - - let (done, succeeded) = LoopAction::shared_success_flag(); - let action_fn = Box::new( - move |node_datas: &[TestData], - test_loop_data: &mut TestLoopData, - client_account_id: AccountId| { - // It must happen only for the first resharding block encountered. - if done.get() { - return; - } - let client_actor = - retrieve_client_actor(node_datas, test_loop_data, &client_account_id); - let tip = client_actor.client.chain.head().unwrap(); - - // If there's a new shard layout force a chain fork. - if next_block_has_new_shard_layout(client_actor.client.epoch_manager.as_ref(), &tip) { - println!("creating chain fork at height {}", tip.height); - let height_selection = if double_signing { - // In the double signing scenario we want a new block on top of prev block, with consecutive height. - AdvProduceBlockHeightSelection::NextHeightOnSelectedBlock { - base_block_height: tip.height - 1, - } - } else { - // To avoid double signing skip already produced height. - AdvProduceBlockHeightSelection::SelectedHeightOnSelectedBlock { - produced_block_height: tip.height + 1, - base_block_height: tip.height - 1, - } - }; - client_actor.adv_produce_blocks_on(3, true, height_selection); - done.set(true); - } - }, - ); - LoopAction::new(action_fn, succeeded) -} - -fn execute_money_transfers(account_ids: Vec) -> LoopAction { - const NUM_TRANSFERS_PER_BLOCK: usize = 20; - - let latest_height = Cell::new(0); - let seed = rand::thread_rng().gen::(); - println!("Random seed: {}", seed); - - let (ran_transfers, succeeded) = LoopAction::shared_success_flag(); - let action_fn = Box::new( - move |node_datas: &[TestData], - test_loop_data: &mut TestLoopData, - client_account_id: AccountId| { - let client_actor = - retrieve_client_actor(node_datas, test_loop_data, &client_account_id); - let tip = client_actor.client.chain.head().unwrap(); - - // Run this action only once at every block height. - if latest_height.get() == tip.height { - return; - } - latest_height.set(tip.height); - - let mut slice = [0u8; 32]; - slice[0..8].copy_from_slice(&seed.to_le_bytes()); - slice[8..16].copy_from_slice(&tip.height.to_le_bytes()); - let mut rng: ChaCha20Rng = SeedableRng::from_seed(slice); - - for _ in 0..NUM_TRANSFERS_PER_BLOCK { - let sender = account_ids.choose(&mut rng).unwrap().clone(); - let receiver = account_ids.choose(&mut rng).unwrap().clone(); - - let clients = node_datas - .iter() - .map(|test_data| { - &test_loop_data.get(&test_data.client_sender.actor_handle()).client - }) - .collect_vec(); - - let anchor_hash = get_anchor_hash(&clients); - let nonce = get_next_nonce(&test_loop_data, &node_datas, &sender); - let amount = ONE_NEAR * rng.gen_range(1..=10); - let tx = SignedTransaction::send_money( - nonce, - sender.clone(), - receiver.clone(), - &create_user_test_signer(&sender).into(), - amount, - anchor_hash, - ); - submit_tx(&node_datas, &client_account_id, tx); - } - ran_transfers.set(true); - }, - ); - LoopAction::new(action_fn, succeeded) -} - -/// Returns a loop action that invokes a costly method from a contract -/// `CALLS_PER_BLOCK_HEIGHT` times per block height. -/// -/// The account invoking the contract is taken in sequential order from `signed_ids`. -/// -/// The account receiving the contract call is taken in sequential order from `receiver_ids`. -fn call_burn_gas_contract( - signer_ids: Vec, - receiver_ids: Vec, - gas_burnt_per_call: Gas, - epoch_length: u64, -) -> LoopAction { - const CALLS_PER_BLOCK_HEIGHT: usize = 5; - // Set to a value large enough, so that transactions from the past epoch are settled. - // Must be less than epoch length, otherwise won't be triggered before the test is finished. - let tx_check_blocks_after_resharding = epoch_length - 2; - - let resharding_height = Cell::new(None); - let nonce = Cell::new(102); - let txs = Cell::new(vec![]); - let latest_height = Cell::new(0); - let (checked_transactions, succeeded) = LoopAction::shared_success_flag(); - - let action_fn = Box::new( - move |node_datas: &[TestData], - test_loop_data: &mut TestLoopData, - client_account_id: AccountId| { - let client_actor = - retrieve_client_actor(node_datas, test_loop_data, &client_account_id); - let tip = client_actor.client.chain.head().unwrap(); - - // Run this action only once at every block height. - if latest_height.get() == tip.height { - return; - } - latest_height.set(tip.height); - - // After resharding: wait some blocks and check that all txs have been executed correctly. - if let Some(height) = resharding_height.get() { - if tip.height > height + tx_check_blocks_after_resharding { - for (tx, tx_height) in txs.take() { - let tx_outcome = - client_actor.client.chain.get_partial_transaction_result(&tx); - let status = tx_outcome.as_ref().map(|o| o.status.clone()); - let status = status.unwrap(); - tracing::debug!(target: "test", ?tx_height, ?tx, ?status, "transaction status"); - assert_matches!(status, FinalExecutionStatus::SuccessValue(_)); - } - checked_transactions.set(true); - } - } else { - if next_block_has_new_shard_layout(client_actor.client.epoch_manager.as_ref(), &tip) - { - tracing::debug!(target: "test", height=tip.height, "resharding height set"); - resharding_height.set(Some(tip.height)); - } - } - // Before resharding and one block after: call the test contract a few times per block. - // The objective is to pile up receipts (e.g. delayed). - if tip.height <= resharding_height.get().unwrap_or(1000) + 1 { - for i in 0..CALLS_PER_BLOCK_HEIGHT { - // Note that if the number of signers and receivers is the - // same then the traffic will always flow the same way. It - // would be nice to randomize it a bit. - let signer_id = &signer_ids[i % signer_ids.len()]; - let receiver_id = &receiver_ids[i % receiver_ids.len()]; - let signer: Signer = create_user_test_signer(signer_id).into(); - nonce.set(nonce.get() + 1); - let method_name = "burn_gas_raw".to_owned(); - let burn_gas: u64 = gas_burnt_per_call; - let args = burn_gas.to_le_bytes().to_vec(); - let tx = SignedTransaction::call( - nonce.get(), - signer_id.clone(), - receiver_id.clone(), - &signer, - 1, - method_name, - args, - gas_burnt_per_call + 10 * TGAS, - tip.last_block_hash, - ); - store_and_submit_tx( - &node_datas, - &client_account_id, - &txs, - &signer_id, - &receiver_id, - tip.height, - tx, - ); - } - } - }, - ); - LoopAction::new(action_fn, succeeded) -} - -/// Sends a promise-yield transaction before resharding. Then, if `call_resume` is `true` also sends -/// a yield-resume transaction after resharding, otherwise it lets the promise-yield go into timeout. -/// -/// Each `signer_id` sends transaction to the corresponding `receiver_id`. -/// -/// A few blocks after resharding all transactions outcomes are checked for successful execution. -fn call_promise_yield( - call_resume: bool, - signer_ids: Vec, - receiver_ids: Vec, -) -> LoopAction { - let resharding_height: Cell> = Cell::new(None); - let txs = Cell::new(vec![]); - let latest_height = Cell::new(0); - let promise_txs_sent = Cell::new(false); - let nonce = Cell::new(102); - let yield_payload = vec![]; - let (checked_transactions, succeeded) = LoopAction::shared_success_flag(); - - let action_fn = Box::new( - move |node_datas: &[TestData], - test_loop_data: &mut TestLoopData, - client_account_id: AccountId| { - let client_actor = - retrieve_client_actor(node_datas, test_loop_data, &client_account_id); - let tip = client_actor.client.chain.head().unwrap(); - - // Run this action only once at every block height. - if latest_height.get() == tip.height { - return; - } - latest_height.set(tip.height); - - // The operation to be done depends on the current block height in relation to the - // resharding height. - match (resharding_height.get(), latest_height.get()) { - // Resharding happened in the previous block. - // Maybe send the resume transaction. - (Some(resharding), latest) if latest == resharding + 1 && call_resume => { - for (signer_id, receiver_id) in - signer_ids.clone().into_iter().zip(receiver_ids.clone().into_iter()) - { - let signer: Signer = create_user_test_signer(&signer_id).into(); - nonce.set(nonce.get() + 1); - let tx = SignedTransaction::call( - nonce.get(), - signer_id.clone(), - receiver_id.clone(), - &signer, - 1, - "call_yield_resume_read_data_id_from_storage".to_string(), - yield_payload.clone(), - 300 * TGAS, - tip.last_block_hash, - ); - store_and_submit_tx( - &node_datas, - &client_account_id, - &txs, - &signer_id, - &receiver_id, - tip.height, - tx, - ); - } - } - // Resharding happened a few blocks in the past. - // Check transactions' outcomes. - (Some(resharding), latest) if latest == resharding + 4 => { - let txs = txs.take(); - assert_ne!(txs.len(), 0); - for (tx, tx_height) in txs { - let tx_outcome = - client_actor.client.chain.get_partial_transaction_result(&tx); - let status = tx_outcome.as_ref().map(|o| o.status.clone()); - let status = status.unwrap(); - tracing::debug!(target: "test", ?tx_height, ?tx, ?status, "transaction status"); - assert_matches!(status, FinalExecutionStatus::SuccessValue(_)); - } - checked_transactions.set(true); - } - (Some(_resharding), _latest) => {} - // Resharding didn't happen in the past. - (None, _) => { - // Check if resharding will happen in this block. - if next_block_has_new_shard_layout( - client_actor.client.epoch_manager.as_ref(), - &tip, - ) { - tracing::debug!(target: "test", height=tip.height, "resharding height set"); - resharding_height.set(Some(tip.height)); - return; - } - // Before resharding, send a set of promise transactions, just once. - if promise_txs_sent.get() { - return; - } - for (signer_id, receiver_id) in - signer_ids.clone().into_iter().zip(receiver_ids.clone().into_iter()) - { - let signer: Signer = create_user_test_signer(&signer_id).into(); - nonce.set(nonce.get() + 1); - let tx = SignedTransaction::call( - nonce.get(), - signer_id.clone(), - receiver_id.clone(), - &signer, - 0, - "call_yield_create_return_promise".to_string(), - yield_payload.clone(), - 300 * TGAS, - tip.last_block_hash, - ); - store_and_submit_tx( - &node_datas, - &client_account_id, - &txs, - &signer_id, - &receiver_id, - tip.height, - tx, - ); - } - promise_txs_sent.set(true); - } - } - }, - ); - LoopAction::new(action_fn, succeeded) -} - fn get_base_shard_layout(version: u64) -> ShardLayout { let boundary_accounts = vec!["account1".parse().unwrap(), "account3".parse().unwrap()]; match version { @@ -568,40 +258,6 @@ fn get_base_shard_layout(version: u64) -> ShardLayout { } } -// After resharding and gc-period, assert the deleted `account_id` -// is still accessible through archival node view client (if available), -// and it is not accessible through a regular, RPC node. -fn check_deleted_account_availability( - env: &mut TestLoopEnv, - archival_id: &Option<&AccountId>, - rpc_id: &AccountId, - account_id: AccountId, - height: u64, -) { - let rpc_node_data = get_node_data(&env.datas, &rpc_id); - let rpc_view_client_handle = rpc_node_data.view_client_sender.actor_handle(); - - let block_reference = BlockReference::BlockId(BlockId::Height(height)); - let request = QueryRequest::ViewAccount { account_id }; - let msg = Query::new(block_reference, request); - - let rpc_node_result = { - let view_client = env.test_loop.data.get_mut(&rpc_view_client_handle); - near_async::messaging::Handler::handle(view_client, msg.clone()) - }; - assert!(!rpc_node_result.is_ok()); - - if let Some(archival_id) = archival_id { - let archival_node_data = get_node_data(&env.datas, &archival_id); - let archival_view_client_handle = archival_node_data.view_client_sender.actor_handle(); - let archival_node_result = { - let view_client = env.test_loop.data.get_mut(&archival_view_client_handle); - near_async::messaging::Handler::handle(view_client, msg) - }; - assert!(archival_node_result.is_ok()); - } -} - /// Base setup to check sanity of Resharding V3. fn test_resharding_v3_base(params: TestReshardingParameters) { if !ProtocolFeature::SimpleNightshadeV4.enabled(PROTOCOL_VERSION) { @@ -637,7 +293,7 @@ fn test_resharding_v3_base(params: TestReshardingParameters) { let base_shard_layout = get_base_shard_layout(params.base_shard_layout_version); base_epoch_config.shard_layout = base_shard_layout.clone(); - let new_boundary_account = "account6".parse().unwrap(); + let new_boundary_account = params.new_boundary_account; let parent_shard_uid = base_shard_layout.account_id_to_shard_uid(&new_boundary_account); let mut epoch_config = base_epoch_config.clone(); epoch_config.shard_layout = @@ -687,10 +343,8 @@ fn test_resharding_v3_base(params: TestReshardingParameters) { builder = builder.runtime_config_store(runtime_config_store); } - let archival_id = params.archivals.iter().next(); - // Try to use an RPC client, if available. Otherwise fallback to the client with the lowest index. - let client_index = params.rpc_client_index.unwrap_or(0); - let client_account_id = params.rpcs.get(0).unwrap_or_else(|| ¶ms.clients[0]).clone(); + let client_index = params.client_index; + let client_account_id = params.clients[client_index].clone(); let mut env = builder .genesis(genesis) @@ -716,27 +370,20 @@ fn test_resharding_v3_base(params: TestReshardingParameters) { ); test_setup_transactions.push(deploy_contract_tx); } - - // Create an account that is: - // 1) Subaccount of a future resharding boundary account. - // 2) Temporary, because we will remove it after resharding. - // The goal is to test removing some state and see if it is kept on archival node. - // The secondary goal is to catch potential bugs due to the above two conditions making it a special case. - let temporary_account = - format!("{}.{}", new_boundary_account, new_boundary_account).parse().unwrap(); - let create_account_tx = create_account( - &mut env, - &client_account_id, - &new_boundary_account, - &temporary_account, - 10 * ONE_NEAR, - 2, - ); - test_setup_transactions.push(create_account_tx); - + if !params.disable_temporary_account_test { + let create_account_tx = create_account( + &mut env, + &client_account_id, + &new_boundary_account, + ¶ms.temporary_account_id, + 10 * ONE_NEAR, + 2, + ); + test_setup_transactions.push(create_account_tx); + } // Wait for the test setup transactions to settle and ensure they all succeeded. env.test_loop.run_for(Duration::seconds(2)); - check_txs(&env.test_loop, &env.datas, &client_account_id, &test_setup_transactions); + check_txs(&env.test_loop.data, &env.datas, &client_account_id, &test_setup_transactions); let client_handles = env.datas.iter().map(|data| data.client_sender.actor_handle()).collect_vec(); @@ -756,8 +403,13 @@ fn test_resharding_v3_base(params: TestReshardingParameters) { client_handles.iter().map(|handle| &env.test_loop.data.get(handle).client).collect_vec(); let mut trie_sanity_check = TrieSanityCheck::new(&clients, params.load_mem_tries_for_tracked_shards); + let gc_num_epochs_to_keep = clients[client_index].config.gc.gc_num_epochs_to_keep; - let latest_block_height = std::cell::Cell::new(0u64); + let latest_block_height = Cell::new(0u64); + // Height of a block after resharding. + let new_layout_block_height = Cell::new(None); + // Height of an epoch after resharding. + let new_layout_epoch_height = Cell::new(None); let success_condition = |test_loop_data: &mut TestLoopData| -> bool { params .loop_actions @@ -766,38 +418,58 @@ fn test_resharding_v3_base(params: TestReshardingParameters) { let clients = client_handles.iter().map(|handle| &test_loop_data.get(handle).client).collect_vec(); - let client = clients[client_index]; + // Skip if we already checked the latest height let tip = get_smallest_height_head(&clients); + if latest_block_height.get() == tip.height { + return false; + } + if latest_block_height.get() == 0 { + println!("State before resharding:"); + print_and_assert_shard_accounts(&clients, &tip); + } + latest_block_height.set(tip.height); // Check that all chunks are included. + let client = clients[client_index]; let block_header = client.chain.get_block_header(&tip.last_block_hash).unwrap(); - if latest_block_height.get() < tip.height { - if latest_block_height.get() == 0 { - println!("State before resharding:"); - print_and_assert_shard_accounts(&clients, &tip); - } - trie_sanity_check.assert_state_sanity(&clients, expected_num_shards); - latest_block_height.set(tip.height); - if params.all_chunks_expected && params.chunk_ranges_to_drop.is_empty() { - assert!(block_header.chunk_mask().iter().all(|chunk_bit| *chunk_bit)); - } + if params.all_chunks_expected && params.chunk_ranges_to_drop.is_empty() { + assert!(block_header.chunk_mask().iter().all(|chunk_bit| *chunk_bit)); } - // Return true if we passed an epoch with increased number of shards. + trie_sanity_check.assert_state_sanity(&clients, expected_num_shards); + let epoch_height = client.epoch_manager.get_epoch_height_from_prev_block(&tip.prev_block_hash).unwrap(); - assert!(epoch_height < 6); - let prev_epoch_id = - client.epoch_manager.get_prev_epoch_id_from_prev_block(&tip.prev_block_hash).unwrap(); - let epoch_config = client.epoch_manager.get_epoch_config(&prev_epoch_id).unwrap(); - if epoch_config.shard_layout.num_shards() != expected_num_shards { - return false; + + // Return false if we have not yet passed an epoch with increased number of shards. + if new_layout_epoch_height.get().is_none() { + assert!(epoch_height < 6); + let prev_epoch_id = client + .epoch_manager + .get_prev_epoch_id_from_prev_block(&tip.prev_block_hash) + .unwrap(); + let epoch_config = client.epoch_manager.get_epoch_config(&prev_epoch_id).unwrap(); + if epoch_config.shard_layout.num_shards() != expected_num_shards { + return false; + } + // Just passed an epoch with increased number of shards. + new_layout_block_height.set(Some(latest_block_height.get())); + new_layout_epoch_height.set(Some(epoch_height)); + println!("State after resharding:"); + print_and_assert_shard_accounts(&clients, &tip); } - println!("State after resharding:"); - print_and_assert_shard_accounts(&clients, &tip); - check_state_shard_uid_mapping_after_resharding(&client, parent_shard_uid); + check_state_shard_uid_mapping_after_resharding( + &client, + parent_shard_uid, + params.allow_negative_refcount, + ); + + // Return false if garbage collection window has not passed yet since resharding. + if epoch_height <= new_layout_epoch_height.get().unwrap() + gc_num_epochs_to_keep { + return false; + } for loop_action in ¶ms.loop_actions { assert_matches!(loop_action.get_status(), LoopActionStatus::Succeeded); } @@ -811,21 +483,6 @@ fn test_resharding_v3_base(params: TestReshardingParameters) { ); let client = &env.test_loop.data.get(&client_handles[client_index]).client; trie_sanity_check.check_epochs(client); - let height_after_resharding = latest_block_height.get(); - - // Delete `temporary_account`. - delete_account(&mut env, &client_account_id, &temporary_account, &client_account_id); - // Wait for garbage collection to kick in. - env.test_loop - .run_for(Duration::seconds((DEFAULT_GC_NUM_EPOCHS_TO_KEEP * params.epoch_length) as i64)); - // Check that the deleted account is still accessible at archival node, but not at a regular node. - check_deleted_account_availability( - &mut env, - &archival_id, - &client_account_id, - temporary_account, - height_after_resharding, - ); env.shutdown_and_drain_remaining_events(Duration::seconds(20)); } @@ -851,6 +508,7 @@ fn test_resharding_v3_drop_chunks_before() { test_resharding_v3_base( TestReshardingParametersBuilder::default() .chunk_ranges_to_drop(chunk_ranges_to_drop) + .epoch_length(INCREASED_EPOCH_LENGTH) .build(), ); } @@ -871,6 +529,7 @@ fn test_resharding_v3_drop_chunks_before_and_after() { test_resharding_v3_base( TestReshardingParametersBuilder::default() .chunk_ranges_to_drop(chunk_ranges_to_drop) + .epoch_length(INCREASED_EPOCH_LENGTH) .build(), ); } diff --git a/integration-tests/src/test_loop/utils/mod.rs b/integration-tests/src/test_loop/utils/mod.rs index 792899d6a45..fcacfb0e16b 100644 --- a/integration-tests/src/test_loop/utils/mod.rs +++ b/integration-tests/src/test_loop/utils/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod contract_distribution; pub(crate) mod loop_action; pub(crate) mod network; pub(crate) mod receipts; +pub(crate) mod resharding; pub(crate) mod setups; pub(crate) mod sharding; pub(crate) mod transactions; diff --git a/integration-tests/src/test_loop/utils/resharding.rs b/integration-tests/src/test_loop/utils/resharding.rs new file mode 100644 index 00000000000..005fc3f3fad --- /dev/null +++ b/integration-tests/src/test_loop/utils/resharding.rs @@ -0,0 +1,483 @@ +use std::cell::Cell; + +use assert_matches::assert_matches; +use itertools::Itertools; +use near_async::test_loop::data::TestLoopData; +use near_client::{Query, QueryError::GarbageCollectedBlock}; +use near_crypto::Signer; +use near_primitives::test_utils::create_user_test_signer; +use near_primitives::transaction::SignedTransaction; +use near_primitives::types::{AccountId, BlockId, BlockReference, Gas}; +use near_primitives::views::{ + FinalExecutionStatus, QueryRequest, QueryResponse, QueryResponseKind, +}; +use rand::seq::SliceRandom; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; + +use super::sharding::this_block_has_new_shard_layout; +use crate::test_loop::env::TestData; +use crate::test_loop::utils::loop_action::LoopAction; +use crate::test_loop::utils::sharding::next_block_has_new_shard_layout; +use crate::test_loop::utils::transactions::{ + check_txs, delete_account, get_anchor_hash, get_next_nonce, store_and_submit_tx, submit_tx, +}; +use crate::test_loop::utils::{get_node_data, retrieve_client_actor, ONE_NEAR, TGAS}; + +// Returns a callable function that, when invoked inside a test loop iteration, can force the creation of a chain fork. +#[cfg(feature = "test_features")] +pub(crate) fn fork_before_resharding_block(double_signing: bool) -> LoopAction { + use near_client::client_actor::AdvProduceBlockHeightSelection; + + let (done, succeeded) = LoopAction::shared_success_flag(); + let action_fn = Box::new( + move |node_datas: &[TestData], + test_loop_data: &mut TestLoopData, + client_account_id: AccountId| { + // It must happen only for the first resharding block encountered. + if done.get() { + return; + } + let client_actor = + retrieve_client_actor(node_datas, test_loop_data, &client_account_id); + let tip = client_actor.client.chain.head().unwrap(); + + // If there's a new shard layout force a chain fork. + if next_block_has_new_shard_layout(client_actor.client.epoch_manager.as_ref(), &tip) { + println!("creating chain fork at height {}", tip.height); + let height_selection = if double_signing { + // In the double signing scenario we want a new block on top of prev block, with consecutive height. + AdvProduceBlockHeightSelection::NextHeightOnSelectedBlock { + base_block_height: tip.height - 1, + } + } else { + // To avoid double signing skip already produced height. + AdvProduceBlockHeightSelection::SelectedHeightOnSelectedBlock { + produced_block_height: tip.height + 1, + base_block_height: tip.height - 1, + } + }; + client_actor.adv_produce_blocks_on(3, true, height_selection); + done.set(true); + } + }, + ); + LoopAction::new(action_fn, succeeded) +} + +pub(crate) fn execute_money_transfers(account_ids: Vec) -> LoopAction { + const NUM_TRANSFERS_PER_BLOCK: usize = 20; + + let latest_height = Cell::new(0); + let seed = rand::thread_rng().gen::(); + println!("Random seed: {}", seed); + + let (ran_transfers, succeeded) = LoopAction::shared_success_flag(); + let action_fn = Box::new( + move |node_datas: &[TestData], + test_loop_data: &mut TestLoopData, + client_account_id: AccountId| { + let client_actor = + retrieve_client_actor(node_datas, test_loop_data, &client_account_id); + let tip = client_actor.client.chain.head().unwrap(); + + // Run this action only once at every block height. + if latest_height.get() == tip.height { + return; + } + latest_height.set(tip.height); + + let mut slice = [0u8; 32]; + slice[0..8].copy_from_slice(&seed.to_le_bytes()); + slice[8..16].copy_from_slice(&tip.height.to_le_bytes()); + let mut rng: ChaCha20Rng = SeedableRng::from_seed(slice); + + for _ in 0..NUM_TRANSFERS_PER_BLOCK { + let sender = account_ids.choose(&mut rng).unwrap().clone(); + let receiver = account_ids.choose(&mut rng).unwrap().clone(); + + let clients = node_datas + .iter() + .map(|test_data| { + &test_loop_data.get(&test_data.client_sender.actor_handle()).client + }) + .collect_vec(); + + let anchor_hash = get_anchor_hash(&clients); + let nonce = get_next_nonce(&test_loop_data, &node_datas, &sender); + let amount = ONE_NEAR * rng.gen_range(1..=10); + let tx = SignedTransaction::send_money( + nonce, + sender.clone(), + receiver.clone(), + &create_user_test_signer(&sender).into(), + amount, + anchor_hash, + ); + submit_tx(&node_datas, &client_account_id, tx); + } + ran_transfers.set(true); + }, + ); + LoopAction::new(action_fn, succeeded) +} + +/// Returns a loop action that invokes a costly method from a contract +/// `CALLS_PER_BLOCK_HEIGHT` times per block height. +/// +/// The account invoking the contract is taken in sequential order from `signed_ids`. +/// +/// The account receiving the contract call is taken in sequential order from `receiver_ids`. +pub(crate) fn call_burn_gas_contract( + signer_ids: Vec, + receiver_ids: Vec, + gas_burnt_per_call: Gas, + epoch_length: u64, +) -> LoopAction { + const CALLS_PER_BLOCK_HEIGHT: usize = 5; + // Set to a value large enough, so that transactions from the past epoch are settled. + // Must be less than epoch length, otherwise won't be triggered before the test is finished. + let tx_check_blocks_after_resharding = epoch_length - 2; + + let resharding_height = Cell::new(None); + let nonce = Cell::new(102); + let txs = Cell::new(vec![]); + let latest_height = Cell::new(0); + let (checked_transactions, succeeded) = LoopAction::shared_success_flag(); + + let action_fn = Box::new( + move |node_datas: &[TestData], + test_loop_data: &mut TestLoopData, + client_account_id: AccountId| { + let client_actor = + retrieve_client_actor(node_datas, test_loop_data, &client_account_id); + let tip = client_actor.client.chain.head().unwrap(); + + // Run this action only once at every block height. + if latest_height.get() == tip.height { + return; + } + latest_height.set(tip.height); + + // After resharding: wait some blocks and check that all txs have been executed correctly. + if let Some(height) = resharding_height.get() { + if tip.height > height + tx_check_blocks_after_resharding { + for (tx, tx_height) in txs.take() { + let tx_outcome = + client_actor.client.chain.get_partial_transaction_result(&tx); + let status = tx_outcome.as_ref().map(|o| o.status.clone()); + let status = status.unwrap(); + tracing::debug!(target: "test", ?tx_height, ?tx, ?status, "transaction status"); + assert_matches!(status, FinalExecutionStatus::SuccessValue(_)); + } + checked_transactions.set(true); + } + } else { + if next_block_has_new_shard_layout(client_actor.client.epoch_manager.as_ref(), &tip) + { + tracing::debug!(target: "test", height=tip.height, "resharding height set"); + resharding_height.set(Some(tip.height)); + } + } + // Before resharding and one block after: call the test contract a few times per block. + // The objective is to pile up receipts (e.g. delayed). + if tip.height <= resharding_height.get().unwrap_or(1000) + 1 { + for i in 0..CALLS_PER_BLOCK_HEIGHT { + // Note that if the number of signers and receivers is the + // same then the traffic will always flow the same way. It + // would be nice to randomize it a bit. + let signer_id = &signer_ids[i % signer_ids.len()]; + let receiver_id = &receiver_ids[i % receiver_ids.len()]; + let signer: Signer = create_user_test_signer(signer_id).into(); + nonce.set(nonce.get() + 1); + let method_name = "burn_gas_raw".to_owned(); + let burn_gas: u64 = gas_burnt_per_call; + let args = burn_gas.to_le_bytes().to_vec(); + let tx = SignedTransaction::call( + nonce.get(), + signer_id.clone(), + receiver_id.clone(), + &signer, + 1, + method_name, + args, + gas_burnt_per_call + 10 * TGAS, + tip.last_block_hash, + ); + store_and_submit_tx( + &node_datas, + &client_account_id, + &txs, + &signer_id, + &receiver_id, + tip.height, + tx, + ); + } + } + }, + ); + LoopAction::new(action_fn, succeeded) +} + +/// Sends a promise-yield transaction before resharding. Then, if `call_resume` is `true` also sends +/// a yield-resume transaction after resharding, otherwise it lets the promise-yield go into timeout. +/// +/// Each `signer_id` sends transaction to the corresponding `receiver_id`. +/// +/// A few blocks after resharding all transactions outcomes are checked for successful execution. +pub(crate) fn call_promise_yield( + call_resume: bool, + signer_ids: Vec, + receiver_ids: Vec, +) -> LoopAction { + let resharding_height: Cell> = Cell::new(None); + let txs = Cell::new(vec![]); + let latest_height = Cell::new(0); + let promise_txs_sent = Cell::new(false); + let nonce = Cell::new(102); + let yield_payload = vec![]; + let (checked_transactions, succeeded) = LoopAction::shared_success_flag(); + + let action_fn = Box::new( + move |node_datas: &[TestData], + test_loop_data: &mut TestLoopData, + client_account_id: AccountId| { + let client_actor = + retrieve_client_actor(node_datas, test_loop_data, &client_account_id); + let tip = client_actor.client.chain.head().unwrap(); + + // Run this action only once at every block height. + if latest_height.get() == tip.height { + return; + } + latest_height.set(tip.height); + + // The operation to be done depends on the current block height in relation to the + // resharding height. + match (resharding_height.get(), latest_height.get()) { + // Resharding happened in the previous block. + // Maybe send the resume transaction. + (Some(resharding), latest) if latest == resharding + 1 && call_resume => { + for (signer_id, receiver_id) in + signer_ids.clone().into_iter().zip(receiver_ids.clone().into_iter()) + { + let signer: Signer = create_user_test_signer(&signer_id).into(); + nonce.set(nonce.get() + 1); + let tx = SignedTransaction::call( + nonce.get(), + signer_id.clone(), + receiver_id.clone(), + &signer, + 1, + "call_yield_resume_read_data_id_from_storage".to_string(), + yield_payload.clone(), + 300 * TGAS, + tip.last_block_hash, + ); + store_and_submit_tx( + &node_datas, + &client_account_id, + &txs, + &signer_id, + &receiver_id, + tip.height, + tx, + ); + } + } + // Resharding happened a few blocks in the past. + // Check transactions' outcomes. + (Some(resharding), latest) if latest == resharding + 4 => { + let txs = txs.take(); + assert_ne!(txs.len(), 0); + for (tx, tx_height) in txs { + let tx_outcome = + client_actor.client.chain.get_partial_transaction_result(&tx); + let status = tx_outcome.as_ref().map(|o| o.status.clone()); + let status = status.unwrap(); + tracing::debug!(target: "test", ?tx_height, ?tx, ?status, "transaction status"); + assert_matches!(status, FinalExecutionStatus::SuccessValue(_)); + } + checked_transactions.set(true); + } + (Some(_resharding), _latest) => {} + // Resharding didn't happen in the past. + (None, _) => { + // Check if resharding will happen in this block. + if next_block_has_new_shard_layout( + client_actor.client.epoch_manager.as_ref(), + &tip, + ) { + tracing::debug!(target: "test", height=tip.height, "resharding height set"); + resharding_height.set(Some(tip.height)); + return; + } + // Before resharding, send a set of promise transactions, just once. + if promise_txs_sent.get() { + return; + } + for (signer_id, receiver_id) in + signer_ids.clone().into_iter().zip(receiver_ids.clone().into_iter()) + { + let signer: Signer = create_user_test_signer(&signer_id).into(); + nonce.set(nonce.get() + 1); + let tx = SignedTransaction::call( + nonce.get(), + signer_id.clone(), + receiver_id.clone(), + &signer, + 0, + "call_yield_create_return_promise".to_string(), + yield_payload.clone(), + 300 * TGAS, + tip.last_block_hash, + ); + store_and_submit_tx( + &node_datas, + &client_account_id, + &txs, + &signer_id, + &receiver_id, + tip.height, + tx, + ); + } + promise_txs_sent.set(true); + } + } + }, + ); + LoopAction::new(action_fn, succeeded) +} + +/// After resharding and gc-period, assert the deleted `account_id` +/// is still accessible through archival node view client (if available), +/// and it is not accessible through a regular, RPC node. +fn check_deleted_account_availability( + node_datas: &[TestData], + test_loop_data: &mut TestLoopData, + archival_id: &Option, + rpc_id: &AccountId, + account_id: &AccountId, + height: u64, +) { + let rpc_node_data = get_node_data(node_datas, &rpc_id); + let rpc_view_client_handle = rpc_node_data.view_client_sender.actor_handle(); + + let block_reference = BlockReference::BlockId(BlockId::Height(height)); + let request = QueryRequest::ViewAccount { account_id: account_id.clone() }; + let msg = Query::new(block_reference, request); + + let rpc_node_result = { + let view_client = test_loop_data.get_mut(&rpc_view_client_handle); + near_async::messaging::Handler::handle(view_client, msg.clone()) + }; + assert_matches!(rpc_node_result, Err(GarbageCollectedBlock { .. })); + + if let Some(archival_id) = archival_id { + let archival_node_data = get_node_data(node_datas, &archival_id); + let archival_view_client_handle = archival_node_data.view_client_sender.actor_handle(); + let archival_node_result = { + let view_client = test_loop_data.get_mut(&archival_view_client_handle); + near_async::messaging::Handler::handle(view_client, msg) + }; + assert_matches!( + archival_node_result, + Ok(QueryResponse { kind: QueryResponseKind::ViewAccount(_), .. }) + ); + } +} + +/// Loop action testing a scenario where a temporary account is deleted after resharding. +/// After `gc_num_epochs_to_keep epochs` we assert that the account +/// is not accesible through RPC node but it is still accesible through archival node. +/// +/// The `temporary_account_id` must be a subaccount of the `originator_id`. +pub(crate) fn temporary_account_during_resharding( + archival_id: Option, + rpc_id: AccountId, + originator_id: AccountId, + temporary_account_id: AccountId, +) -> LoopAction { + let latest_height = Cell::new(0); + let resharding_height = Cell::new(None); + let target_height = Cell::new(None); + + let delete_account_tx_hash = Cell::new(None); + let checked_deleted_account = Cell::new(false); + + let (done, succeeded) = LoopAction::shared_success_flag(); + let action_fn = Box::new( + move |node_datas: &[TestData], + test_loop_data: &mut TestLoopData, + client_account_id: AccountId| { + if done.get() { + return; + } + + let client_actor = + retrieve_client_actor(node_datas, test_loop_data, &client_account_id); + let tip = client_actor.client.chain.head().unwrap(); + + // Run this action only once at every block height. + if latest_height.get() == tip.height { + return; + } + latest_height.set(tip.height); + let epoch_length = client_actor.client.config.epoch_length; + let gc_num_epochs_to_keep = client_actor.client.config.gc.gc_num_epochs_to_keep; + + if resharding_height.get().is_none() { + if !this_block_has_new_shard_layout( + client_actor.client.epoch_manager.as_ref(), + &tip, + ) { + return; + } + // Just resharded. Delete the temporary account and set the target height + // high enough so that the delete account transaction will be garbage collected. + let tx_hash = delete_account( + test_loop_data, + node_datas, + &client_account_id, + &temporary_account_id, + &originator_id, + ); + delete_account_tx_hash.set(Some(tx_hash)); + target_height + .set(Some(latest_height.get() + (gc_num_epochs_to_keep + 1) * epoch_length)); + resharding_height.set(Some(latest_height.get())); + } + + // If an epoch passed since resharding, make sure the delete account transaction finished. + if latest_height.get() == resharding_height.get().unwrap() + epoch_length { + check_txs( + test_loop_data, + node_datas, + &client_account_id, + &[delete_account_tx_hash.get().unwrap()], + ); + checked_deleted_account.set(true); + } + + if latest_height.get() < target_height.get().unwrap() { + return; + } + assert!(checked_deleted_account.get()); + // Since gc window passed after the account was deleted, + // check that it is not accessible through regular node, + // but it is accessible through archival node. + check_deleted_account_availability( + node_datas, + test_loop_data, + &archival_id, + &rpc_id, + &temporary_account_id, + resharding_height.get().unwrap(), + ); + done.set(true); + }, + ); + LoopAction::new(action_fn, succeeded) +} diff --git a/integration-tests/src/test_loop/utils/transactions.rs b/integration-tests/src/test_loop/utils/transactions.rs index 9dda074d95f..a79366a7289 100644 --- a/integration-tests/src/test_loop/utils/transactions.rs +++ b/integration-tests/src/test_loop/utils/transactions.rs @@ -171,7 +171,7 @@ pub fn do_create_account( let nonce = get_next_nonce(&env.test_loop.data, &env.datas, originator); let tx = create_account(env, rpc_id, originator, new_account_id, amount, nonce); env.test_loop.run_for(Duration::seconds(5)); - check_txs(&env.test_loop, &env.datas, rpc_id, &[tx]); + check_txs(&env.test_loop.data, &env.datas, rpc_id, &[tx]); } pub fn do_delete_account( @@ -181,9 +181,9 @@ pub fn do_delete_account( beneficiary_id: &AccountId, ) { tracing::info!(target: "test", "Deleting account."); - let tx = delete_account(env, rpc_id, account_id, beneficiary_id); + let tx = delete_account(&env.test_loop.data, &env.datas, rpc_id, account_id, beneficiary_id); env.test_loop.run_for(Duration::seconds(5)); - check_txs(&env.test_loop, &env.datas, rpc_id, &[tx]); + check_txs(&env.test_loop.data, &env.datas, rpc_id, &[tx]); } pub fn do_deploy_contract( @@ -196,7 +196,7 @@ pub fn do_deploy_contract( let nonce = get_next_nonce(&env.test_loop.data, &env.datas, contract_id); let tx = deploy_contract(&mut env.test_loop, &env.datas, rpc_id, contract_id, code, nonce); env.test_loop.run_for(Duration::seconds(2)); - check_txs(&env.test_loop, &env.datas, rpc_id, &[tx]); + check_txs(&env.test_loop.data, &env.datas, rpc_id, &[tx]); } pub fn do_call_contract( @@ -220,7 +220,7 @@ pub fn do_call_contract( nonce, ); env.test_loop.run_for(Duration::seconds(2)); - check_txs(&env.test_loop, &env.datas, rpc_id, &[tx]); + check_txs(&env.test_loop.data, &env.datas, rpc_id, &[tx]); } pub fn create_account( @@ -231,7 +231,7 @@ pub fn create_account( amount: u128, nonce: u64, ) -> CryptoHash { - let block_hash = get_shared_block_hash(&env.datas, &env.test_loop); + let block_hash = get_shared_block_hash(&env.datas, &env.test_loop.data); let signer = create_user_test_signer(&originator); let new_signer: Signer = create_user_test_signer(&new_account_id); @@ -252,14 +252,15 @@ pub fn create_account( } pub fn delete_account( - env: &mut TestLoopEnv, + test_loop_data: &TestLoopData, + node_datas: &[TestData], rpc_id: &AccountId, account_id: &AccountId, beneficiary_id: &AccountId, ) -> CryptoHash { let signer: Signer = create_user_test_signer(&account_id).into(); - let nonce = get_next_nonce(&env.test_loop.data, &env.datas, account_id); - let block_hash = get_shared_block_hash(&env.datas, &env.test_loop); + let nonce = get_next_nonce(&test_loop_data, node_datas, account_id); + let block_hash = get_shared_block_hash(node_datas, test_loop_data); let tx = SignedTransaction::delete_account( nonce, @@ -271,7 +272,7 @@ pub fn delete_account( ); let tx_hash = tx.get_hash(); - submit_tx(&env.datas, rpc_id, tx); + submit_tx(node_datas, rpc_id, tx); tracing::debug!(target: "test", ?account_id, ?beneficiary_id, ?tx_hash, "deleted account"); tx_hash } @@ -289,7 +290,7 @@ pub fn deploy_contract( code: Vec, nonce: u64, ) -> CryptoHash { - let block_hash = get_shared_block_hash(node_datas, test_loop); + let block_hash = get_shared_block_hash(node_datas, &test_loop.data); let signer = create_user_test_signer(&contract_id); @@ -314,7 +315,7 @@ pub fn call_contract( args: Vec, nonce: u64, ) -> CryptoHash { - let block_hash = get_shared_block_hash(node_datas, test_loop); + let block_hash = get_shared_block_hash(node_datas, &test_loop.data); let signer = create_user_test_signer(sender_id); let attach_gas = 300 * TGAS; let deposit = 0; @@ -355,12 +356,12 @@ pub fn submit_tx(node_datas: &[TestData], rpc_id: &AccountId, tx: SignedTransact /// Please note that it's important to use an rpc node that tracks all shards. /// Otherwise, the transactions may not be found. pub fn check_txs( - test_loop: &TestLoopV2, + test_loop_data: &TestLoopData, node_datas: &[TestData], rpc_id: &AccountId, txs: &[CryptoHash], ) { - let rpc = rpc_client(test_loop, node_datas, rpc_id); + let rpc = rpc_client(test_loop_data, node_datas, rpc_id); for &tx in txs { let tx_outcome = rpc.chain.get_partial_transaction_result(&tx); @@ -373,21 +374,21 @@ pub fn check_txs( /// Get the client for the provided rpd node account id. fn rpc_client<'a>( - test_loop: &'a TestLoopV2, + test_loop_data: &'a TestLoopData, node_datas: &'a [TestData], rpc_id: &AccountId, ) -> &'a Client { let node_data = get_node_data(node_datas, rpc_id); let client_actor_handle = node_data.client_sender.actor_handle(); - let client_actor = test_loop.data.get(&client_actor_handle); + let client_actor = test_loop_data.get(&client_actor_handle); &client_actor.client } /// Finds a block that all clients have on their chain and return its hash. -pub fn get_shared_block_hash(node_datas: &[TestData], test_loop: &TestLoopV2) -> CryptoHash { +pub fn get_shared_block_hash(node_datas: &[TestData], test_loop_data: &TestLoopData) -> CryptoHash { let clients = node_datas .iter() - .map(|data| &test_loop.data.get(&data.client_sender.actor_handle()).client) + .map(|data| &test_loop_data.get(&data.client_sender.actor_handle()).client) .collect_vec(); let (_, block_hash) = clients diff --git a/integration-tests/src/test_loop/utils/trie_sanity.rs b/integration-tests/src/test_loop/utils/trie_sanity.rs index 56e102339bf..ca12fe83b35 100644 --- a/integration-tests/src/test_loop/utils/trie_sanity.rs +++ b/integration-tests/src/test_loop/utils/trie_sanity.rs @@ -338,7 +338,11 @@ fn should_assert_state_sanity( } /// Asserts that all parent shard State is accessible via parent and children shards. -pub fn check_state_shard_uid_mapping_after_resharding(client: &Client, parent_shard_uid: ShardUId) { +pub fn check_state_shard_uid_mapping_after_resharding( + client: &Client, + parent_shard_uid: ShardUId, + allow_negative_refcount: bool, +) { let tip = client.chain.head().unwrap(); let epoch_id = tip.epoch_id; let epoch_config = client.epoch_manager.get_epoch_config(&epoch_id).unwrap(); @@ -356,7 +360,18 @@ pub fn check_state_shard_uid_mapping_after_resharding(client: &Client, parent_sh continue; } let node_hash = CryptoHash::try_from_slice(&key[8..]).unwrap(); - let (value, _) = decode_value_with_rc(&value); + let (value, rc) = decode_value_with_rc(&value); + // It is possible we have delayed receipts leftovers on disk, + // that would result it `MissingTrieValue` if we attempt to read them through the Trie interface. + // TODO(resharding) Remove this when negative refcounts are properly handled. + if rc <= 0 { + assert!(allow_negative_refcount); + // That can only be -1, because delayed receipt can be removed at most twice (by both children). + assert_eq!(rc, -1); + // In case of negative refcount, we only store the refcount, and the value is empty. + assert!(value.is_none()); + continue; + } let parent_value = store.get(parent_shard_uid, &node_hash); // Parent shard data must still be accessible using parent ShardUId. assert_eq!(&parent_value.unwrap()[..], value.unwrap());