diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index fe74ff0dd..89b809f93 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -25,7 +25,7 @@ concurrency: cancel-in-progress: true jobs: build-image: - runs-on: neon-evm-1 + runs-on: ["self-hosted", "k8s-prod"] outputs: evm_tag: ${{ steps.tags.outputs.evm_tag }} evm_sha_tag: ${{ steps.tags.outputs.evm_sha_tag }} @@ -33,9 +33,13 @@ jobs: is_evm_release: ${{ steps.tags.outputs.is_evm_release }} neon_test_tag: ${{ steps.tags.outputs.neon_test_tag }} steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/setup-python@v5 - uses: actions/checkout@v4 with: fetch-depth: 0 + - run: pip install -r ./.github/workflows/requirements.txt - name: Specify image tags run: | python3 ./.github/workflows/deploy.py specify_image_tags \ @@ -60,13 +64,17 @@ jobs: --evm_sha_tag=${{ steps.tags.outputs.evm_sha_tag }} \ --evm_tag=${{ steps.tags.outputs.evm_tag }} run-evm-tests: - runs-on: test-runner + runs-on: ["self-hosted", "k8s-prod"] needs: - build-image steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/setup-python@v5 - uses: actions/checkout@v4 with: fetch-depth: 0 + - run: pip install -r ./.github/workflows/requirements.txt - name: Run tests run: | python3 ./.github/workflows/deploy.py run_tests \ @@ -78,13 +86,17 @@ jobs: if: "failure()" uses: "andymckay/cancel-action@0.4" trigger-proxy-tests: - runs-on: trigger-runner + runs-on: ["self-hosted", "k8s-prod"] needs: - build-image steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/setup-python@v5 - uses: actions/checkout@v4 with: fetch-depth: 0 + - run: pip install -r ./.github/workflows/requirements.txt - name: Trigger proxy build run: | python3 ./.github/workflows/deploy.py trigger_proxy_action \ @@ -100,15 +112,19 @@ jobs: if: "failure()" uses: "andymckay/cancel-action@0.4" finalize-image: - runs-on: neon-evm-1 + runs-on: ["self-hosted", "k8s-prod"] needs: - build-image - trigger-proxy-tests - run-evm-tests steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/setup-python@v5 - uses: actions/checkout@v4 with: fetch-depth: 0 + - run: pip install -r ./.github/workflows/requirements.txt - name: Finalize image run: | python3 ./.github/workflows/deploy.py finalize_image \ diff --git a/evm_loader/Cargo.lock b/evm_loader/Cargo.lock index 0a0544808..29124faad 100644 --- a/evm_loader/Cargo.lock +++ b/evm_loader/Cargo.lock @@ -3609,6 +3609,7 @@ dependencies = [ "enum_dispatch", "ethnum", "evm-loader", + "futures", "goblin 0.8.2", "hex", "hex-literal", diff --git a/evm_loader/lib-interface/src/lib.rs b/evm_loader/lib-interface/src/lib.rs index 0a1781d21..89c07401d 100644 --- a/evm_loader/lib-interface/src/lib.rs +++ b/evm_loader/lib-interface/src/lib.rs @@ -17,7 +17,7 @@ use abi_stable::{ #[repr(C)] #[derive(StableAbi)] -#[sabi(kind(Prefix(prefix_ref = NeonEVMLib_Ref)))] +#[sabi(kind(Prefix(prefix_ref = NeonEVMLibRef)))] #[sabi(missing_field(panic))] pub struct NeonEVMLib { pub hash: extern "C" fn() -> RString, @@ -28,8 +28,8 @@ pub struct NeonEVMLib { } #[allow(clippy::use_self)] -impl RootModule for NeonEVMLib_Ref { - abi_stable::declare_root_module_statics! {NeonEVMLib_Ref} +impl RootModule for NeonEVMLibRef { + abi_stable::declare_root_module_statics! {NeonEVMLibRef} const BASE_NAME: &'static str = "neon-lib-interface"; const NAME: &'static str = "neon-lib-interface"; @@ -46,14 +46,14 @@ pub enum NeonEVMLibLoadError { pub fn load_libraries

( directory: P, -) -> Result, NeonEVMLibLoadError> +) -> Result, NeonEVMLibLoadError> where P: AsRef, { let paths = std::fs::read_dir(directory)?; let mut result = HashMap::new(); for path in paths { - let lib = NeonEVMLib_Ref::load_from_file(&path?.path())?; + let lib = NeonEVMLibRef::load_from_file(&path?.path())?; let hash = lib.hash()(); result.insert(hash.into_string(), lib); diff --git a/evm_loader/lib/Cargo.toml b/evm_loader/lib/Cargo.toml index 7866135ed..175c06b23 100644 --- a/evm_loader/lib/Cargo.toml +++ b/evm_loader/lib/Cargo.toml @@ -52,6 +52,7 @@ clap = "2.34.0" lazy_static = "1.5.0" elsa = "1.10.0" arrayref = "0.3.8" +futures = "0.3.30" [dev-dependencies] hex-literal = "0.4.1" diff --git a/evm_loader/lib/src/account_storage.rs b/evm_loader/lib/src/account_storage.rs index 0a4fda17d..46fa908a8 100644 --- a/evm_loader/lib/src/account_storage.rs +++ b/evm_loader/lib/src/account_storage.rs @@ -918,11 +918,6 @@ impl<'a, T: Rpc> EmulatorAccountStorage<'_, T> { let new_lamports = new_acc.lamports; let new_size = new_acc.get_length(); - if new_acc.is_busy() && new_lamports < self.rent.minimum_balance(new_acc.get_length()) { - info!("Account {pubkey} is not rent exempt"); - return Err(ProgramError::AccountNotRentExempt.into()); - } - let old_lamports = lamports_after_upgrade.unwrap_or(original_lamports); old_lamports_sum += old_lamports; new_lamports_sum += new_lamports; diff --git a/evm_loader/lib/src/account_storage_tests.rs b/evm_loader/lib/src/account_storage_tests.rs index d5f68dc50..5445f3e15 100644 --- a/evm_loader/lib/src/account_storage_tests.rs +++ b/evm_loader/lib/src/account_storage_tests.rs @@ -3,6 +3,7 @@ use crate::rpc; use crate::tracing::AccountOverride; use evm_loader::types::vector::VectorVecExt; use hex_literal::hex; +use solana_account_decoder::UiDataSliceConfig; use std::collections::HashMap; use std::str::FromStr; @@ -19,6 +20,8 @@ mod mock_rpc_client { use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; + use solana_account_decoder::UiDataSliceConfig; + pub struct MockRpcClient { accounts: HashMap, } @@ -33,7 +36,31 @@ mod mock_rpc_client { #[async_trait(?Send)] impl Rpc for MockRpcClient { - async fn get_account(&self, key: &Pubkey) -> ClientResult> { + async fn get_account_slice( + &self, + key: &Pubkey, + slice: Option, + ) -> ClientResult> { + if let Some(data_slice) = slice { + if let Some(orig_acc) = self.accounts.get(key) { + let cut_to = + usize::min(data_slice.offset + data_slice.length, orig_acc.data.len()); + let sliced_data = if data_slice.offset < orig_acc.data.len() { + orig_acc.data[data_slice.offset..cut_to].to_vec() + } else { + vec![] + }; + + return Ok(Some(Account { + lamports: orig_acc.lamports, + data: sliced_data, + owner: orig_acc.owner, + executable: orig_acc.executable, + rent_epoch: orig_acc.rent_epoch, + })); + } + } + let result = self.accounts.get(key).cloned(); Ok(result) } @@ -1790,3 +1817,35 @@ async fn test_storage_new_from_other_and_override() { expected_balance ); } + +#[tokio::test] +async fn test_storage_get_account_slice() { + let slice_from = 2; + let slice_size = 20; + let test_key = Pubkey::new_unique(); + let acc = Account::new(10, 1 * 1024 * 1024, &solana_sdk::sysvar::rent::id()); + + let account_tuple = (test_key, acc); + let accounts_for_rpc = vec![ + (solana_sdk::sysvar::rent::id(), account_tuple.1.clone()), + account_tuple.clone(), + ]; + let rpc_client = mock_rpc_client::MockRpcClient::new(&accounts_for_rpc); + let acc_no_slice = rpc_client + .get_account(&test_key) + .await + .expect("Failed to get account slice"); + + let slice_cfg = UiDataSliceConfig { + offset: slice_from, + length: slice_size, + }; + let sliced_acc = rpc_client + .get_account_slice(&test_key, Some(slice_cfg)) + .await + .expect("Failed to get account slice"); + assert!(acc_no_slice.is_some()); + assert!(sliced_acc.is_some()); + assert!(acc_no_slice.unwrap().data.len() > 2000); + assert_eq!(sliced_acc.unwrap().data.len(), slice_size); +} diff --git a/evm_loader/lib/src/build_info_common.rs b/evm_loader/lib/src/build_info_common.rs index 712145ca4..8ee350cfb 100644 --- a/evm_loader/lib/src/build_info_common.rs +++ b/evm_loader/lib/src/build_info_common.rs @@ -2,10 +2,10 @@ use build_info::chrono::{DateTime, Utc}; use build_info::semver::Version; use build_info::VersionControl::Git; use build_info::{BuildInfo, OptimizationLevel}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct SlimBuildInfo { timestamp: DateTime, profile: String, @@ -15,18 +15,18 @@ pub struct SlimBuildInfo { version_control: GitInfo, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct CrateInfo { name: String, version: Version, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct CompilerInfo { version: Version, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] struct GitInfo { commit_id: String, dirty: bool, diff --git a/evm_loader/lib/src/commands/get_config.rs b/evm_loader/lib/src/commands/get_config.rs index 0cd59b5d5..9fde6a2f6 100644 --- a/evm_loader/lib/src/commands/get_config.rs +++ b/evm_loader/lib/src/commands/get_config.rs @@ -85,7 +85,6 @@ impl BuildConfigSimulator for CallDbClient { async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { let mut simulator = SolanaSimulator::new_without_sync(self).await?; simulator.sync_accounts(self, &[program_id]).await?; - Ok(ConfigSimulator::ProgramTestContext { program_id, simulator, diff --git a/evm_loader/lib/src/lib.rs b/evm_loader/lib/src/lib.rs index 29eadd71d..d2ffa0224 100644 --- a/evm_loader/lib/src/lib.rs +++ b/evm_loader/lib/src/lib.rs @@ -26,15 +26,15 @@ use abi::_MODULE_WM_; use abi_stable::export_root_module; pub use config::Config; pub use errors::NeonError; -use neon_lib_interface::NeonEVMLib_Ref; +use neon_lib_interface::NeonEVMLibRef; pub type NeonResult = Result; -const MODULE: NeonEVMLib_Ref = NeonEVMLib_Ref(_MODULE_WM_.static_as_prefix()); +const MODULE: NeonEVMLibRef = NeonEVMLibRef(_MODULE_WM_.static_as_prefix()); #[export_root_module] #[must_use] -pub const fn get_root_module() -> NeonEVMLib_Ref { +pub const fn get_root_module() -> NeonEVMLibRef { MODULE } diff --git a/evm_loader/lib/src/rpc/db_call_client.rs b/evm_loader/lib/src/rpc/db_call_client.rs index 32e97b9d5..5664b09b3 100644 --- a/evm_loader/lib/src/rpc/db_call_client.rs +++ b/evm_loader/lib/src/rpc/db_call_client.rs @@ -1,4 +1,4 @@ -use super::{e, Rpc}; +use super::{e, Rpc, SliceConfig}; use crate::types::{TracerDb, TracerDbTrait}; use crate::NeonError; use crate::NeonError::RocksDb; @@ -42,9 +42,13 @@ impl CallDbClient { }) } - async fn get_account_at(&self, key: &Pubkey) -> ClientResult> { + async fn get_account_at( + &self, + key: &Pubkey, + slice: Option, + ) -> ClientResult> { self.tracer_db - .get_account_at(key, self.slot, self.tx_index_in_block) + .get_account_at(key, self.slot, self.tx_index_in_block, slice) .await .map_err(|e| e!("load account error", key, e)) } @@ -52,8 +56,12 @@ impl CallDbClient { #[async_trait(?Send)] impl Rpc for CallDbClient { - async fn get_account(&self, key: &Pubkey) -> ClientResult> { - self.get_account_at(key).await + async fn get_account_slice( + &self, + key: &Pubkey, + slice: Option, + ) -> ClientResult> { + self.get_account_at(key, slice).await } async fn get_multiple_accounts( @@ -62,7 +70,7 @@ impl Rpc for CallDbClient { ) -> ClientResult>> { let mut result = Vec::new(); for key in pubkeys { - result.push(self.get_account_at(key).await?); + result.push(self.get_account_at(key, None).await?); } debug!("get_multiple_accounts: pubkeys={pubkeys:?} result={result:?}"); Ok(result) diff --git a/evm_loader/lib/src/rpc/emulator_client.rs b/evm_loader/lib/src/rpc/emulator_client.rs index 5a63af7f6..ae35c25ac 100644 --- a/evm_loader/lib/src/rpc/emulator_client.rs +++ b/evm_loader/lib/src/rpc/emulator_client.rs @@ -10,21 +10,35 @@ use solana_sdk::{ use crate::account_storage::{fake_operator, EmulatorAccountStorage}; -use super::Rpc; +use super::{Rpc, SliceConfig}; #[async_trait(?Send)] impl<'rpc, T: Rpc> Rpc for EmulatorAccountStorage<'rpc, T> { - async fn get_account(&self, key: &Pubkey) -> ClientResult> { - if *key == self.operator() { - return Ok(Some(fake_operator())); - } - - if let Some(account_data) = self.accounts_get(key) { - return Ok(Some(Account::from(&*account_data))); + async fn get_account_slice( + &self, + key: &Pubkey, + slice: Option, + ) -> ClientResult> { + let answer_account = if *key == self.operator() { + Some(fake_operator()) + } else if let Some(account_data) = self.accounts_get(key) { + Some(Account::from(&*account_data)) + } else { + self._get_account_from_rpc(*key).await?.cloned() + }; + + if let Some(data_slice) = slice { + // if only slice is necessary - cut data + if let Some(mut account) = answer_account { + if data_slice.offset != 0 { + account.data.drain(0..data_slice.offset); + } + account.data.truncate(data_slice.length); + return Ok(Some(account)); + } } - let account = self._get_account_from_rpc(*key).await?.cloned(); - Ok(account) + Ok(answer_account) } async fn get_multiple_accounts( diff --git a/evm_loader/lib/src/rpc/mod.rs b/evm_loader/lib/src/rpc/mod.rs index afa09bdd4..d14e1763c 100644 --- a/evm_loader/lib/src/rpc/mod.rs +++ b/evm_loader/lib/src/rpc/mod.rs @@ -9,20 +9,29 @@ use crate::commands::get_config::{BuildConfigSimulator, ConfigSimulator}; use crate::{NeonError, NeonResult}; use async_trait::async_trait; use enum_dispatch::enum_dispatch; +pub use solana_account_decoder::UiDataSliceConfig as SliceConfig; use solana_cli::cli::CliError; use solana_client::client_error::Result as ClientResult; -use solana_sdk::message::Message; -use solana_sdk::native_token::lamports_to_sol; use solana_sdk::{ account::Account, clock::{Slot, UnixTimestamp}, + message::Message, + native_token::lamports_to_sol, pubkey::Pubkey, }; #[async_trait(?Send)] #[enum_dispatch] pub trait Rpc { - async fn get_account(&self, key: &Pubkey) -> ClientResult>; + async fn get_account_slice( + &self, + key: &Pubkey, + slice: Option, + ) -> ClientResult>; + async fn get_account(&self, key: &Pubkey) -> ClientResult> { + self.get_account_slice(key, None).await + } + async fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> ClientResult>>; async fn get_block_time(&self, slot: Slot) -> ClientResult; diff --git a/evm_loader/lib/src/rpc/validator_client.rs b/evm_loader/lib/src/rpc/validator_client.rs index 1c64a8623..e8a0585c9 100644 --- a/evm_loader/lib/src/rpc/validator_client.rs +++ b/evm_loader/lib/src/rpc/validator_client.rs @@ -1,6 +1,6 @@ use crate::{config::APIOptions, Config}; -use super::Rpc; +use super::{Rpc, SliceConfig}; use async_trait::async_trait; use solana_account_decoder::{UiAccount, UiAccountEncoding}; use solana_client::{ @@ -15,6 +15,7 @@ use solana_sdk::{ clock::{Slot, UnixTimestamp}, pubkey::Pubkey, }; + use std::{error::Error, ops::Deref, time::Duration}; use std::{future::Future, sync::Arc}; use tracing::debug; @@ -113,12 +114,16 @@ impl Deref for CloneRpcClient { #[async_trait(?Send)] impl Rpc for CloneRpcClient { - async fn get_account(&self, key: &Pubkey) -> ClientResult> { + async fn get_account_slice( + &self, + key: &Pubkey, + slice: Option, + ) -> ClientResult> { let request = || { let config = RpcAccountInfoConfig { encoding: Some(UiAccountEncoding::Base64Zstd), commitment: Some(self.commitment()), - data_slice: None, + data_slice: slice, min_context_slot: None, }; let params = serde_json::json!([key.to_string(), config]); diff --git a/evm_loader/lib/src/solana_simulator/mod.rs b/evm_loader/lib/src/solana_simulator/mod.rs index 6be80bde4..5c0f4bb36 100644 --- a/evm_loader/lib/src/solana_simulator/mod.rs +++ b/evm_loader/lib/src/solana_simulator/mod.rs @@ -5,6 +5,7 @@ use std::sync::Arc; pub use error::Error; use evm_loader::solana_program::bpf_loader_upgradeable::UpgradeableLoaderState; use evm_loader::solana_program::clock::Slot; + use evm_loader::solana_program::loader_v4; use evm_loader::solana_program::loader_v4::{LoaderV4State, LoaderV4Status}; use evm_loader::solana_program::message::SanitizedMessage; @@ -51,6 +52,7 @@ use solana_sdk::{ pub use utils::SyncState; use crate::rpc::Rpc; +use crate::types::programs_cache::programdata_cache_get_values_by_keys; mod error; mod utils; @@ -113,7 +115,6 @@ impl SolanaSimulator { let Some(account) = account else { continue; }; - if account.executable && bpf_loader_upgradeable::check_id(&account.owner) { let programdata_address = utils::program_data_address(account)?; debug!( @@ -129,7 +130,9 @@ impl SolanaSimulator { storable_accounts.push((key, account)); } - let mut programdata_accounts = rpc.get_multiple_accounts(&programdata_keys).await?; + let mut programdata_accounts = + programdata_cache_get_values_by_keys(&programdata_keys, rpc).await?; + for (key, account) in programdata_keys.iter().zip(&mut programdata_accounts) { let Some(account) = account else { continue; diff --git a/evm_loader/lib/src/types/mod.rs b/evm_loader/lib/src/types/mod.rs index 0993d346b..fc8906043 100644 --- a/evm_loader/lib/src/types/mod.rs +++ b/evm_loader/lib/src/types/mod.rs @@ -1,5 +1,6 @@ pub mod tracer_ch_common; +pub mod programs_cache; pub(crate) mod tracer_ch_db; pub mod tracer_rocks_db; @@ -26,6 +27,8 @@ use evm_loader::{ use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; use serde_with::{hex::Hex, serde_as, DisplayFromStr, OneOrMany}; + +use crate::rpc::SliceConfig; use solana_sdk::signature::Signature; use solana_sdk::{account::Account, pubkey::Pubkey}; use std::collections::HashMap; @@ -79,6 +82,7 @@ pub trait TracerDbTrait { pubkey: &Pubkey, slot: u64, tx_index_in_block: Option, + data_slice: Option, ) -> DbResult>; async fn get_transaction_index(&self, signature: Signature) -> DbResult; diff --git a/evm_loader/lib/src/types/programs_cache.rs b/evm_loader/lib/src/types/programs_cache.rs new file mode 100644 index 000000000..a438b5b60 --- /dev/null +++ b/evm_loader/lib/src/types/programs_cache.rs @@ -0,0 +1,281 @@ +// use crate::tracing::tracers::state_diff::Account; +use crate::rpc::Rpc; +use async_trait::async_trait; + +use bincode::deserialize; +use futures::future::join_all; +use solana_client::client_error::{ClientErrorKind, Result as ClientResult}; +use solana_sdk::{ + account::Account, + bpf_loader_upgradeable::UpgradeableLoaderState, + clock::{Slot, UnixTimestamp}, + pubkey::Pubkey, +}; +use std::collections::HashMap; +use std::hash::Hash; +use std::sync::RwLock; + +use bincode::serialize; +use tokio::sync::OnceCell; +use tracing::info; +#[derive(Debug, Eq, PartialEq, Hash)] +pub struct KeyAccountCache { + addr: Pubkey, + slot: u64, +} + +use crate::rpc::SliceConfig; +type ProgramDataCache = HashMap; +type ThreadSaveProgramDataCache = RwLock; + +static LOCAL_CONFIG: OnceCell = OnceCell::const_new(); + +pub async fn cut_programdata_from_acc(account: &mut Account, data_slice: SliceConfig) { + if data_slice.offset != 0 { + account + .data + .drain(..std::cmp::min(account.data.len(), data_slice.offset)); + } + account.data.truncate(data_slice.length); +} + +async fn programdata_hash_get_instance() -> &'static ThreadSaveProgramDataCache { + LOCAL_CONFIG + .get_or_init(|| async { + let map = HashMap::new(); + + RwLock::new(map) + }) + .await +} + +async fn programdata_hash_get(addr: Pubkey, slot: u64) -> Option { + let val = KeyAccountCache { addr, slot }; + programdata_hash_get_instance() + .await + .read() + .expect("acc_hash_get_instance poisoned") + .get(&val) + .cloned() +} + +async fn programdata_hash_add(addr: Pubkey, slot: u64, acc: Account) { + let val = KeyAccountCache { addr, slot }; + programdata_hash_get_instance() + .await + .write() + .expect("PANIC, no nable") + .insert(val, acc); +} + +fn get_programdata_slot_from_account(acc: &Account) -> ClientResult { + if !bpf_loader_upgradeable::check_id(&acc.owner) { + return Err(ClientErrorKind::Custom("Not upgradeable account".to_string()).into()); + } + + match deserialize::(&acc.data) { + Ok(UpgradeableLoaderState::ProgramData { slot, .. }) => Ok(slot), + Ok(_) => { + panic!("Account is not of type `ProgramData`."); + } + Err(e) => { + eprintln!("Error occurred: {e:?}"); + panic!("Failed to deserialize account data."); + } + } +} + +pub async fn programdata_cache_get_values_by_keys( + programdata_keys: &Vec, + rpc: &impl Rpc, +) -> ClientResult>> { + let mut future_requests = Vec::new(); + let mut answer = Vec::new(); + + for key in programdata_keys { + future_requests.push(rpc.get_account_slice( + key, + Some(SliceConfig { + offset: 0, + length: UpgradeableLoaderState::size_of_programdata_metadata(), + }), + )); + } + + assert_eq!( + programdata_keys.len(), + future_requests.len(), + "programdata_keys.size()!=future_requests.size()" + ); + let results = join_all(future_requests).await; + for (result, key) in results.iter().zip(programdata_keys) { + match result { + Ok(Some(account)) => { + let slot_val = get_programdata_slot_from_account(account)?; + if let Some(acc) = programdata_hash_get(*key, slot_val).await { + answer.push(Some(acc)); + } else if let Ok(Some(tmp_acc)) = rpc.get_account(key).await { + let current_slot = get_programdata_slot_from_account(&tmp_acc)?; + programdata_hash_add(*key, current_slot, tmp_acc.clone()).await; + + answer.push(Some(tmp_acc)); + } else { + answer.push(None); + } + } + Ok(None) => { + info!("Account for key {key:?} is None."); + answer.push(None); + } + Err(e) => { + info!("Error fetching account for key {key:?}: {e:?}"); + } + } + } + Ok(answer) +} + +struct FakeRpc { + accounts: HashMap, +} +#[allow(dead_code)] +impl FakeRpc { + pub fn new() -> Self { + Self { + accounts: HashMap::new(), + } + } + + fn has_account(&self, pubkey: &Pubkey) -> bool { + self.accounts.contains_key(pubkey) + } + + fn make_account(&mut self, pubkey: Pubkey) -> Account { + // Define the slot number you want to test with + let test_slot: u64 = 42; + + // Create mock ProgramData state + let program_data = UpgradeableLoaderState::ProgramData { + slot: test_slot, + upgrade_authority_address: Some(Pubkey::new_unique()), + }; + let mut serialized_data = serialize(&program_data).unwrap(); + serialized_data.resize(4 * 1024 * 1024, 0); + let mut answer = Account::new(0, serialized_data.len(), &bpf_loader_upgradeable::id()); + answer.data = serialized_data; + + self.accounts.insert(pubkey, answer.clone()); + answer + } +} + +#[async_trait(?Send)] + +impl Rpc for FakeRpc { + async fn get_account_slice( + &self, + pubkey: &Pubkey, + slice: Option, + ) -> ClientResult> { + assert!(self.accounts.contains_key(pubkey), " "); + + let mut answer = self.accounts.get(pubkey).unwrap().clone(); + if let Some(data_slice) = slice { + if data_slice.offset != 0 { + answer + .data + .drain(..std::cmp::min(answer.data.len(), data_slice.offset)); + } + answer.data.truncate(data_slice.length); + } + Ok(Some(answer)) + } + + async fn get_multiple_accounts( + &self, + pubkeys: &[Pubkey], + ) -> ClientResult>> { + let mut futures = Vec::new(); + for pubkey in pubkeys { + futures.push(self.get_account(pubkey).await?); + } + + Ok(futures) + } + + async fn get_block_time(&self, _slot: Slot) -> ClientResult { + Ok(9999) + } + async fn get_slot(&self) -> ClientResult { + Ok(1212) + } + + async fn get_deactivated_solana_features(&self) -> ClientResult> { + Ok(Vec::new()) + } +} +use evm_loader::solana_program::bpf_loader_upgradeable; +use tokio; + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_acc_slice() { + let mut rpc = FakeRpc::new(); + let test_key = Pubkey::new_unique(); + + let test_acc = rpc.make_account(test_key); + if test_acc.data.len() >= 4000000 { + println!("Account data len: {}", test_acc.data.len()); + } else { + panic!("test stop"); + } + if let Ok(test2_acc) = rpc.get_account(&test_key).await { + assert_eq!( + test_acc.data.len(), + test2_acc.expect("test fail").data.len() + ); + } else { + panic!("fake rpc returned error"); + } + + let test3_acc = rpc + .get_account_slice( + &test_key, + Some(SliceConfig { + offset: 0, + length: 1024, + }), + ) + .await; + assert_eq!(1024, test3_acc.unwrap().expect("test fail").data.len()); + } + #[tokio::test] + async fn test_acc_request() { + const TEST_KEYS_COUNT: usize = 10; + let mut rpc = FakeRpc::new(); + let mut test_keys = Vec::new(); //Pubkey::new_unique(); + + for _i in 0..TEST_KEYS_COUNT { + let curr_key = Pubkey::new_unique(); + rpc.make_account(curr_key); + test_keys.push(curr_key); + } + + let multiple_accounts = rpc + .get_multiple_accounts(&test_keys) + .await + .expect("ERR DURING ACC REQUESTS"); + + let hashed_accounts = programdata_cache_get_values_by_keys(&test_keys, &rpc) + .await + .expect("ERR DURING ACC REQUESTS WITH HASH"); + assert_eq!(hashed_accounts.len(), multiple_accounts.len()); + for i in 0..TEST_KEYS_COUNT { + assert!(hashed_accounts[i].is_some(), "BAD ACC"); + assert!(multiple_accounts[i].is_some(), "BAD ACC"); + } + } +} diff --git a/evm_loader/lib/src/types/tracer_ch_db.rs b/evm_loader/lib/src/types/tracer_ch_db.rs index 15dddd982..818daf32d 100644 --- a/evm_loader/lib/src/types/tracer_ch_db.rs +++ b/evm_loader/lib/src/types/tracer_ch_db.rs @@ -5,14 +5,15 @@ use crate::{ }; use super::tracer_ch_common::{ChResult, EthSyncStatus, EthSyncing, RevisionMap, SlotParentRooted}; - use crate::account_data::AccountData; use crate::config::ChDbConfig; +use crate::types::programs_cache::cut_programdata_from_acc; use anyhow::anyhow; use async_trait::async_trait; use clickhouse::Client; use log::{debug, error, info}; use rand::Rng; +pub use solana_account_decoder::UiDataSliceConfig as SliceConfig; use solana_sdk::signature::Signature; use solana_sdk::{ account::Account, @@ -93,23 +94,19 @@ impl TracerDbTrait for ClickHouseDb { pubkey: &Pubkey, slot: u64, tx_index_in_block: Option, + data_slice: Option, ) -> DbResult> { - if let Some(tx_index_in_block) = tx_index_in_block { - return if let Some(account) = self - .get_account_at_index_in_block(pubkey, slot, tx_index_in_block) - .await? - { - Ok(Some(account)) - } else { - self.get_account_at_slot(pubkey, slot - 1) - .await - .map_err(|e| anyhow!("Failed to get NEON_REVISION, error: {e}")) - }; + let result = self + .get_full_account_at(pubkey, slot, tx_index_in_block) + .await; + if let Ok(Some(mut account)) = result { + if let Some(slice) = data_slice { + cut_programdata_from_acc(&mut account, slice).await; + } + Ok(Some(account)) + } else { + result } - - self.get_account_at_slot(pubkey, slot) - .await - .map_err(|e| anyhow!("Failed to get NEON_REVISION, error: {e}")) } async fn get_transaction_index(&self, signature: Signature) -> DbResult { @@ -321,7 +318,29 @@ impl ClickHouseDb { Self { client } } + async fn get_full_account_at( + &self, + pubkey: &Pubkey, + slot: u64, + tx_index_in_block: Option, + ) -> DbResult> { + if let Some(tx_index_in_block) = tx_index_in_block { + return if let Some(account) = self + .get_account_at_index_in_block(pubkey, slot, tx_index_in_block) + .await? + { + Ok(Some(account)) + } else { + self.get_account_at_slot(pubkey, slot - 1) + .await + .map_err(|e| anyhow!("Failed to get NEON_REVISION, error: {e}")) + }; + } + self.get_account_at_slot(pubkey, slot) + .await + .map_err(|e| anyhow!("Failed to get NEON_REVISION, error: {e}")) + } async fn get_branch_slots(&self, slot: Option) -> ChResult<(u64, Vec)> { fn branch_from( rows: Vec, @@ -653,6 +672,7 @@ impl ClickHouseDb { &self, pubkey: &Pubkey, sol_sig: &[u8; 64], + bin_slice: Option, ) -> DbResult> { let sol_sig_str = bs58::encode(sol_sig).into_string(); info!("get_account_by_sol_sig {{ pubkey: {pubkey}, sol_sig: {sol_sig_str} }}"); @@ -734,7 +754,7 @@ impl ClickHouseDb { // If not found, get closest account state in one of previous slots if let Some(parent) = slot.parent { - self.get_account_at(pubkey, parent, None).await + self.get_account_at(pubkey, parent, None, bin_slice).await } else { Ok(None) } diff --git a/evm_loader/lib/src/types/tracer_rocks_db.rs b/evm_loader/lib/src/types/tracer_rocks_db.rs index e4d5191d6..340f5636b 100644 --- a/evm_loader/lib/src/types/tracer_rocks_db.rs +++ b/evm_loader/lib/src/types/tracer_rocks_db.rs @@ -6,6 +6,7 @@ use jsonrpsee::core::Serialize; use jsonrpsee::rpc_params; use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; use serde_json::from_str; +use solana_account_decoder::UiDataSliceConfig; use solana_sdk::signature::Signature; use solana_sdk::{ account::Account, @@ -94,14 +95,15 @@ impl TracerDbTrait for RocksDb { pubkey: &Pubkey, slot: u64, tx_index_in_block: Option, + maybe_bin_slice: Option, ) -> DbResult> { - info!("get_account_at {pubkey:?}, slot: {slot:?}, tx_index: {tx_index_in_block:?}"); + info!("get_account_at {pubkey:?}, slot: {slot:?}, tx_index: {tx_index_in_block:?}, bin_slice: {maybe_bin_slice:?}"); let response: String = self .client .request( "get_account", - rpc_params![pubkey.to_string(), slot, tx_index_in_block], + rpc_params![pubkey.to_string(), slot, tx_index_in_block, maybe_bin_slice], ) .await?; diff --git a/evm_loader/program/src/account/state.rs b/evm_loader/program/src/account/state.rs index 4f00f8df1..50ca97073 100644 --- a/evm_loader/program/src/account/state.rs +++ b/evm_loader/program/src/account/state.rs @@ -327,7 +327,7 @@ impl<'a> StateAccount<'a> { } #[must_use] - fn gas_available(&self) -> U256 { + pub fn gas_available(&self) -> U256 { self.trx().gas_limit().saturating_sub(self.gas_used()) } diff --git a/evm_loader/program/src/instruction/transaction_cancel.rs b/evm_loader/program/src/instruction/transaction_cancel.rs index 86e30e046..1f3af0d6b 100644 --- a/evm_loader/program/src/instruction/transaction_cancel.rs +++ b/evm_loader/program/src/instruction/transaction_cancel.rs @@ -1,3 +1,5 @@ +use std::cmp::min; + use crate::account::{AccountsDB, BalanceAccount, Operator, OperatorBalanceAccount, StateAccount}; use crate::config::DEFAULT_CHAIN_ID; use crate::debug::log_data; @@ -53,17 +55,20 @@ fn execute<'a>( ) -> Result<()> { let trx_chain_id = storage.trx().chain_id().unwrap_or(DEFAULT_CHAIN_ID); - let used_gas = U256::ZERO; - let total_used_gas = storage.gas_used(); + let used_gas = min( + storage.gas_available(), + U256::from(CANCEL_TRX_COST + LAST_ITERATION_COST), + ); + let total_used_gas = storage.gas_used() + used_gas; + log_data(&[ b"GAS", &used_gas.to_le_bytes(), &total_used_gas.to_le_bytes(), ]); - let gas = U256::from(CANCEL_TRX_COST + LAST_ITERATION_COST); - let priority_fee = priority_fee_txn_calculator::handle_priority_fee(storage.trx(), gas)?; - let _ = storage.consume_gas(gas, priority_fee, accounts.try_operator_balance()); // ignore error + let priority_fee = priority_fee_txn_calculator::handle_priority_fee(storage.trx(), used_gas)?; + let _ = storage.consume_gas(used_gas, priority_fee, accounts.try_operator_balance()); // ignore error let origin = storage.trx_origin(); let (origin_pubkey, _) = origin.find_balance_address(program_id, trx_chain_id); diff --git a/evm_loader/rpc-client/src/http.rs b/evm_loader/rpc-client/src/http.rs index d81134619..36e84a5c3 100644 --- a/evm_loader/rpc-client/src/http.rs +++ b/evm_loader/rpc-client/src/http.rs @@ -3,6 +3,9 @@ use async_trait::async_trait; use jsonrpsee_core::{client::ClientT, rpc_params}; use jsonrpsee_http_client::{HttpClient, HttpClientBuilder}; +use neon_lib::build_info_common::SlimBuildInfo; +use neon_lib::commands::simulate_solana::SimulateSolanaResponse; +use neon_lib::types::SimulateSolanaRequest; use neon_lib::LibMethod; use neon_lib::{ commands::{ @@ -52,7 +55,7 @@ impl Default for NeonRpcHttpClientBuilder { } } -#[async_trait(?Send)] +#[async_trait] impl NeonRpcClient for NeonRpcHttpClient { async fn emulate(&self, params: EmulateApiRequest) -> NeonRpcClientResult { self.request(LibMethod::Emulate, params).await @@ -90,6 +93,21 @@ impl NeonRpcClient for NeonRpcHttpClient { async fn trace(&self, params: EmulateApiRequest) -> NeonRpcClientResult { self.request(LibMethod::Trace, params).await } + + async fn simulate_solana( + &self, + params: SimulateSolanaRequest, + ) -> NeonRpcClientResult { + self.request(LibMethod::SimulateSolana, params).await + } + + async fn build_info(&self) -> NeonRpcClientResult { + self.custom_request_without_params("build_info").await + } + + async fn lib_build_info(&self) -> NeonRpcClientResult { + self.custom_request_without_params("lib_build_info").await + } } impl NeonRpcHttpClient { @@ -110,4 +128,11 @@ impl NeonRpcHttpClient { { Ok(self.client.request(method.into(), rpc_params![]).await?) } + + async fn custom_request_without_params(&self, method: &str) -> NeonRpcClientResult + where + R: DeserializeOwned, + { + Ok(self.client.request(method, rpc_params![]).await?) + } } diff --git a/evm_loader/rpc-client/src/lib.rs b/evm_loader/rpc-client/src/lib.rs index 36233df2f..360ac2ec5 100644 --- a/evm_loader/rpc-client/src/lib.rs +++ b/evm_loader/rpc-client/src/lib.rs @@ -10,21 +10,22 @@ pub use error::NeonRpcClientError; use async_trait::async_trait; use neon_lib::{ + build_info_common::SlimBuildInfo, commands::{ emulate::EmulateResponse, get_balance::GetBalanceResponse, get_config::GetConfigResponse, get_contract::GetContractResponse, get_holder::GetHolderResponse, - get_storage_at::GetStorageAtReturn, + get_storage_at::GetStorageAtReturn, simulate_solana::SimulateSolanaResponse, }, types::{ EmulateApiRequest, GetBalanceRequest, GetContractRequest, GetHolderRequest, - GetStorageAtRequest, + GetStorageAtRequest, SimulateSolanaRequest, }, }; type NeonRpcClientResult = Result; -#[async_trait(?Send)] -pub trait NeonRpcClient { +#[async_trait] +pub trait NeonRpcClient: Sync + Send + 'static { async fn emulate(&self, params: EmulateApiRequest) -> NeonRpcClientResult; async fn balance( &self, @@ -41,4 +42,10 @@ pub trait NeonRpcClient { params: GetStorageAtRequest, ) -> NeonRpcClientResult; async fn trace(&self, params: EmulateApiRequest) -> NeonRpcClientResult; + async fn simulate_solana( + &self, + params: SimulateSolanaRequest, + ) -> NeonRpcClientResult; + async fn build_info(&self) -> NeonRpcClientResult; + async fn lib_build_info(&self) -> NeonRpcClientResult; } diff --git a/evm_loader/rpc/src/context.rs b/evm_loader/rpc/src/context.rs index 614b629a5..61c1b67ca 100644 --- a/evm_loader/rpc/src/context.rs +++ b/evm_loader/rpc/src/context.rs @@ -1,6 +1,6 @@ -use neon_lib_interface::NeonEVMLib_Ref; +use neon_lib_interface::NeonEVMLibRef; use std::collections::HashMap; pub struct Context { - pub libraries: HashMap, + pub libraries: HashMap, } diff --git a/evm_loader/rpc/src/handlers/mod.rs b/evm_loader/rpc/src/handlers/mod.rs index ea541301a..dc6dd95cd 100644 --- a/evm_loader/rpc/src/handlers/mod.rs +++ b/evm_loader/rpc/src/handlers/mod.rs @@ -13,11 +13,11 @@ pub mod trace; use crate::context::Context; use jsonrpc_v2::Data; use neon_lib::LibMethod; -use neon_lib_interface::{types::NeonEVMLibError, NeonEVMLib_Ref}; +use neon_lib_interface::{types::NeonEVMLibError, NeonEVMLibRef}; use serde::Serialize; use serde_json::Value; -fn get_library(context: &Data) -> Result<&NeonEVMLib_Ref, jsonrpc_v2::Error> { +fn get_library(context: &Data) -> Result<&NeonEVMLibRef, jsonrpc_v2::Error> { // just for testing let hash = context .libraries