From b12a03cd8272e05d1660164f6d1383729b552d7a Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Wed, 6 Nov 2024 16:48:17 +0400 Subject: [PATCH 01/13] task --- evm_loader/api/src/api_server/handlers/get_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_loader/api/src/api_server/handlers/get_config.rs b/evm_loader/api/src/api_server/handlers/get_config.rs index 4de5c1bf5..0d6584bed 100644 --- a/evm_loader/api/src/api_server/handlers/get_config.rs +++ b/evm_loader/api/src/api_server/handlers/get_config.rs @@ -20,7 +20,7 @@ pub async fn get_config(state: NeonApiState, request_id: RequestId) -> impl Resp Ok(rpc) => rpc, Err(e) => return process_error(StatusCode::BAD_REQUEST, &e), }; - + // Entry GetConfigResponse process_result( &GetConfigCommand::execute(&rpc, state.config.evm_loader) .await From 73d87a82afa2ac492d323956c1c212ccf440f7e1 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Wed, 13 Nov 2024 15:50:32 +0400 Subject: [PATCH 02/13] GetConfigResponce struct caching --- evm_loader/lib/src/account_storage_tests.rs | 21 +++-- evm_loader/lib/src/commands/get_config.rs | 96 ++++++++++++--------- evm_loader/lib/src/rpc/mod.rs | 6 +- evm_loader/lib/src/types/programs_cache.rs | 81 +++++++++++------ 4 files changed, 127 insertions(+), 77 deletions(-) diff --git a/evm_loader/lib/src/account_storage_tests.rs b/evm_loader/lib/src/account_storage_tests.rs index 5445f3e15..5fceed714 100644 --- a/evm_loader/lib/src/account_storage_tests.rs +++ b/evm_loader/lib/src/account_storage_tests.rs @@ -4,24 +4,26 @@ use crate::tracing::AccountOverride; use evm_loader::types::vector::VectorVecExt; use hex_literal::hex; use solana_account_decoder::UiDataSliceConfig; + use std::collections::HashMap; use std::str::FromStr; - const STORAGE_LENGTH: usize = 32 * STORAGE_ENTRIES_IN_CONTRACT_ACCOUNT; mod mock_rpc_client { + use crate::types::programs_cache::get_programdata_slot_from_account; + use solana_sdk::bpf_loader_upgradeable::UpgradeableLoaderState; + use crate::commands::get_config::BuildConfigSimulator; use crate::NeonResult; use crate::{commands::get_config::ConfigSimulator, rpc::Rpc}; use async_trait::async_trait; + use solana_account_decoder::UiDataSliceConfig; use solana_client::client_error::Result as ClientResult; use solana_sdk::account::Account; use solana_sdk::clock::{Slot, UnixTimestamp}; use solana_sdk::pubkey::Pubkey; use std::collections::HashMap; - use solana_account_decoder::UiDataSliceConfig; - pub struct MockRpcClient { accounts: HashMap, } @@ -91,8 +93,17 @@ mod mock_rpc_client { #[async_trait(?Send)] impl BuildConfigSimulator for MockRpcClient { - fn use_cache(&self) -> bool { - false + async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> u64 { + let slice = UiDataSliceConfig { + offset: 0, + length: UpgradeableLoaderState::size_of_programdata_metadata(), + }; + let result = self.get_account_slice(program_id, Some(slice)).await; + if let Ok(Some(acc)) = result { + get_programdata_slot_from_account(&acc).expect("NO slot value for acc") + } else { + panic!("get_account_slice return an Error "); + } } async fn build_config_simulator(&self, _program_id: Pubkey) -> NeonResult { unimplemented!(); diff --git a/evm_loader/lib/src/commands/get_config.rs b/evm_loader/lib/src/commands/get_config.rs index f67108b7d..95434c4e3 100644 --- a/evm_loader/lib/src/commands/get_config.rs +++ b/evm_loader/lib/src/commands/get_config.rs @@ -2,21 +2,24 @@ use std::collections::BTreeMap; +use crate::rpc::{CallDbClient, CloneRpcClient, Rpc}; +use crate::solana_simulator::SolanaSimulator; +use crate::types::programs_cache::{get_programdata_slot_from_account, KeyAccountCache}; +use crate::types::programs_cache::{program_config_cache_add, program_config_cache_get}; use async_trait::async_trait; use base64::Engine; use enum_dispatch::enum_dispatch; +use evm_loader::solana_program::bpf_loader_upgradeable::UpgradeableLoaderState; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; +pub use solana_account_decoder::UiDataSliceConfig as SliceConfig; use solana_client::rpc_config::RpcSimulateTransactionConfig; use solana_sdk::signer::Signer; use solana_sdk::{instruction::Instruction, pubkey::Pubkey, transaction::Transaction}; -use tokio::sync::OnceCell; -use crate::rpc::{CallDbClient, CloneRpcClient}; -use crate::solana_simulator::SolanaSimulator; use crate::NeonResult; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum Status { Ok, Emergency, @@ -33,7 +36,7 @@ pub struct ChainInfo { } #[serde_as] -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct GetConfigResponse { pub version: String, pub revision: String, @@ -58,16 +61,51 @@ pub enum ConfigSimulator<'r> { #[async_trait(?Send)] #[enum_dispatch] pub trait BuildConfigSimulator { - fn use_cache(&self) -> bool; + async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> u64; + + async fn get_config_simulator(&self, program_id: Pubkey) -> NeonResult { + let key = KeyAccountCache { + addr: program_id, + slot: self.get_last_deployed_slot(&program_id).await, + }; + + let rz = program_config_cache_get(&key).await; + if rz.is_some() { + return Ok(rz.unwrap()); + }; + let mut simulator = self.build_config_simulator(program_id).await?; + + let (version, revision) = simulator.get_version().await?; + + let result = GetConfigResponse { + version, + revision, + status: simulator.get_status().await?, + environment: simulator.get_environment().await?, + chains: simulator.get_chains().await?, + config: simulator.get_properties().await?, + }; + program_config_cache_add(key, result.clone()).await; + Ok(result) + } + async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult; } #[async_trait(?Send)] impl BuildConfigSimulator for CloneRpcClient { - fn use_cache(&self) -> bool { - true + async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> u64 { + let slice = SliceConfig { + offset: 0, + length: UpgradeableLoaderState::size_of_programdata_metadata(), + }; + let result = self.get_account_slice(program_id, Some(slice)).await; + if let Ok(Some(acc)) = result { + get_programdata_slot_from_account(&acc).expect("NO slot value for acc") + } else { + panic!("get_account_slice return an Error "); + } } - async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { Ok(ConfigSimulator::CloneRpcClient { program_id, @@ -78,13 +116,13 @@ impl BuildConfigSimulator for CloneRpcClient { #[async_trait(?Send)] impl BuildConfigSimulator for CallDbClient { - fn use_cache(&self) -> bool { - false + async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> u64 { + 0 } - async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { let mut simulator = SolanaSimulator::new_without_sync(self).await?; simulator.sync_accounts(self, &[program_id]).await?; + Ok(ConfigSimulator::ProgramTestContext { program_id, simulator, @@ -264,44 +302,16 @@ pub async fn execute( rpc: &impl BuildConfigSimulator, program_id: Pubkey, ) -> NeonResult { - let mut simulator = rpc.build_config_simulator(program_id).await?; - - let (version, revision) = simulator.get_version().await?; - - Ok(GetConfigResponse { - version, - revision, - status: simulator.get_status().await?, - environment: simulator.get_environment().await?, - chains: simulator.get_chains().await?, - config: simulator.get_properties().await?, - }) + rpc.get_config_simulator(program_id).await } -static CHAINS_CACHE: OnceCell> = OnceCell::const_new(); +// static CHAINS_CACHE: OnceCell> = OnceCell::const_new(); pub async fn read_chains( rpc: &impl BuildConfigSimulator, program_id: Pubkey, ) -> NeonResult> { - if rpc.use_cache() { - return CHAINS_CACHE - .get_or_try_init(|| get_chains(rpc, program_id)) - .await - .cloned(); - } - - get_chains(rpc, program_id).await -} - -async fn get_chains( - rpc: &(impl BuildConfigSimulator + Sized), - program_id: Pubkey, -) -> NeonResult> { - rpc.build_config_simulator(program_id) - .await? - .get_chains() - .await + Ok(rpc.get_config_simulator(program_id).await?.chains) } async fn read_chain_id( diff --git a/evm_loader/lib/src/rpc/mod.rs b/evm_loader/lib/src/rpc/mod.rs index d14e1763c..3f2696b3b 100644 --- a/evm_loader/lib/src/rpc/mod.rs +++ b/evm_loader/lib/src/rpc/mod.rs @@ -2,12 +2,11 @@ mod db_call_client; mod emulator_client; mod validator_client; -pub use db_call_client::CallDbClient; -pub use validator_client::CloneRpcClient; - +use crate::commands::get_config::GetConfigResponse; use crate::commands::get_config::{BuildConfigSimulator, ConfigSimulator}; use crate::{NeonError, NeonResult}; use async_trait::async_trait; +pub use db_call_client::CallDbClient; use enum_dispatch::enum_dispatch; pub use solana_account_decoder::UiDataSliceConfig as SliceConfig; use solana_cli::cli::CliError; @@ -19,6 +18,7 @@ use solana_sdk::{ native_token::lamports_to_sol, pubkey::Pubkey, }; +pub use validator_client::CloneRpcClient; #[async_trait(?Send)] #[enum_dispatch] diff --git a/evm_loader/lib/src/types/programs_cache.rs b/evm_loader/lib/src/types/programs_cache.rs index a438b5b60..b80ebe7b9 100644 --- a/evm_loader/lib/src/types/programs_cache.rs +++ b/evm_loader/lib/src/types/programs_cache.rs @@ -2,6 +2,7 @@ use crate::rpc::Rpc; use async_trait::async_trait; +use crate::commands::get_config::GetConfigResponse; use bincode::deserialize; use futures::future::join_all; use solana_client::client_error::{ClientErrorKind, Result as ClientResult}; @@ -20,15 +21,18 @@ use tokio::sync::OnceCell; use tracing::info; #[derive(Debug, Eq, PartialEq, Hash)] pub struct KeyAccountCache { - addr: Pubkey, - slot: u64, + pub addr: Pubkey, + pub slot: u64, } - use crate::rpc::SliceConfig; -type ProgramDataCache = HashMap; -type ThreadSaveProgramDataCache = RwLock; +type ProgramDataCache = HashMap; +type ThreadSaveCache = RwLock>; + +type ThreadSaveProgramDataCache = ThreadSaveCache; +type ThreadSaveConfigCache<'a> = ThreadSaveCache; -static LOCAL_CONFIG: OnceCell = OnceCell::const_new(); +static ACCOUNT_CACHE_TABLE: OnceCell = OnceCell::const_new(); +static CONFIG_CACHE_TABLE: OnceCell = OnceCell::const_new(); pub async fn cut_programdata_from_acc(account: &mut Account, data_slice: SliceConfig) { if data_slice.offset != 0 { @@ -38,9 +42,25 @@ pub async fn cut_programdata_from_acc(account: &mut Account, data_slice: SliceCo } account.data.truncate(data_slice.length); } - -async fn programdata_hash_get_instance() -> &'static ThreadSaveProgramDataCache { - LOCAL_CONFIG +fn cache_get( + key: &KeyAccountCache, + table: &ThreadSaveCache, +) -> Option { + table + .read() + .expect("acc_hash_get_instance poisoned") + .get(key) + .cloned() +} +fn cache_add( + key: KeyAccountCache, + value: Value, + table: &ThreadSaveCache, +) { + table.write().expect("PANIC, no space ").insert(key, value); +} +async fn programdata_account_cache_get_instance() -> &'static ThreadSaveProgramDataCache { + ACCOUNT_CACHE_TABLE .get_or_init(|| async { let map = HashMap::new(); @@ -49,26 +69,17 @@ async fn programdata_hash_get_instance() -> &'static ThreadSaveProgramDataCache .await } -async fn programdata_hash_get(addr: Pubkey, slot: u64) -> Option { +async fn programdata_account_cache_get(addr: Pubkey, slot: u64) -> Option { let val = KeyAccountCache { addr, slot }; - programdata_hash_get_instance() - .await - .read() - .expect("acc_hash_get_instance poisoned") - .get(&val) - .cloned() + cache_get(&val, programdata_account_cache_get_instance().await) } -async fn programdata_hash_add(addr: Pubkey, slot: u64, acc: Account) { - let val = KeyAccountCache { addr, slot }; - programdata_hash_get_instance() - .await - .write() - .expect("PANIC, no nable") - .insert(val, acc); +async fn programdata_account_cache_add(addr: Pubkey, slot: u64, acc: Account) { + let key = KeyAccountCache { addr, slot }; + cache_add(key, acc, programdata_account_cache_get_instance().await); } -fn get_programdata_slot_from_account(acc: &Account) -> ClientResult { +pub fn get_programdata_slot_from_account(acc: &Account) -> ClientResult { if !bpf_loader_upgradeable::check_id(&acc.owner) { return Err(ClientErrorKind::Custom("Not upgradeable account".to_string()).into()); } @@ -112,11 +123,11 @@ pub async fn programdata_cache_get_values_by_keys( match result { Ok(Some(account)) => { let slot_val = get_programdata_slot_from_account(account)?; - if let Some(acc) = programdata_hash_get(*key, slot_val).await { + if let Some(acc) = programdata_account_cache_get(*key, slot_val).await { answer.push(Some(acc)); } else if let Ok(Some(tmp_acc)) = rpc.get_account(key).await { let current_slot = get_programdata_slot_from_account(&tmp_acc)?; - programdata_hash_add(*key, current_slot, tmp_acc.clone()).await; + programdata_account_cache_add(*key, current_slot, tmp_acc.clone()).await; answer.push(Some(tmp_acc)); } else { @@ -169,6 +180,24 @@ impl FakeRpc { } } +async fn program_config_cache_get_instance() -> &'static ThreadSaveConfigCache<'static> { + CONFIG_CACHE_TABLE + .get_or_init(|| async { + let map = HashMap::new(); + + RwLock::new(map) + }) + .await +} + +pub async fn program_config_cache_get(key: &KeyAccountCache) -> Option { + cache_get(key, program_config_cache_get_instance().await) +} + +pub async fn program_config_cache_add(key: KeyAccountCache, val: GetConfigResponse) { + cache_add(key, val, program_config_cache_get_instance().await); +} + #[async_trait(?Send)] impl Rpc for FakeRpc { From 317355a62a69adc1f42b96f65ade25fd917eebad Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Mon, 2 Dec 2024 14:18:47 +0400 Subject: [PATCH 03/13] Move get_last_deployed_slot --- evm_loader/lib/src/account_storage_tests.rs | 14 ---------- evm_loader/lib/src/commands/get_config.rs | 31 ++++++--------------- evm_loader/lib/src/rpc/mod.rs | 19 +++++++++++-- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/evm_loader/lib/src/account_storage_tests.rs b/evm_loader/lib/src/account_storage_tests.rs index 5fceed714..fac8c7997 100644 --- a/evm_loader/lib/src/account_storage_tests.rs +++ b/evm_loader/lib/src/account_storage_tests.rs @@ -10,8 +10,6 @@ use std::str::FromStr; const STORAGE_LENGTH: usize = 32 * STORAGE_ENTRIES_IN_CONTRACT_ACCOUNT; mod mock_rpc_client { - use crate::types::programs_cache::get_programdata_slot_from_account; - use solana_sdk::bpf_loader_upgradeable::UpgradeableLoaderState; use crate::commands::get_config::BuildConfigSimulator; use crate::NeonResult; @@ -93,18 +91,6 @@ mod mock_rpc_client { #[async_trait(?Send)] impl BuildConfigSimulator for MockRpcClient { - async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> u64 { - let slice = UiDataSliceConfig { - offset: 0, - length: UpgradeableLoaderState::size_of_programdata_metadata(), - }; - let result = self.get_account_slice(program_id, Some(slice)).await; - if let Ok(Some(acc)) = result { - get_programdata_slot_from_account(&acc).expect("NO slot value for acc") - } else { - panic!("get_account_slice return an Error "); - } - } async fn build_config_simulator(&self, _program_id: Pubkey) -> NeonResult { unimplemented!(); } diff --git a/evm_loader/lib/src/commands/get_config.rs b/evm_loader/lib/src/commands/get_config.rs index 95434c4e3..52c83c39f 100644 --- a/evm_loader/lib/src/commands/get_config.rs +++ b/evm_loader/lib/src/commands/get_config.rs @@ -4,12 +4,11 @@ use std::collections::BTreeMap; use crate::rpc::{CallDbClient, CloneRpcClient, Rpc}; use crate::solana_simulator::SolanaSimulator; -use crate::types::programs_cache::{get_programdata_slot_from_account, KeyAccountCache}; +use crate::types::programs_cache::KeyAccountCache; use crate::types::programs_cache::{program_config_cache_add, program_config_cache_get}; use async_trait::async_trait; use base64::Engine; use enum_dispatch::enum_dispatch; -use evm_loader::solana_program::bpf_loader_upgradeable::UpgradeableLoaderState; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; pub use solana_account_decoder::UiDataSliceConfig as SliceConfig; @@ -60,13 +59,14 @@ pub enum ConfigSimulator<'r> { #[async_trait(?Send)] #[enum_dispatch] -pub trait BuildConfigSimulator { - async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> u64; +pub trait BuildConfigSimulator: Rpc { + async fn get_config(&self, program_id: Pubkey) -> NeonResult { + let maybe_slot = self.get_last_deployed_slot(&program_id).await?; + let slot = maybe_slot.expect("Account should have slot "); - async fn get_config_simulator(&self, program_id: Pubkey) -> NeonResult { let key = KeyAccountCache { addr: program_id, - slot: self.get_last_deployed_slot(&program_id).await, + slot, }; let rz = program_config_cache_get(&key).await; @@ -94,18 +94,6 @@ pub trait BuildConfigSimulator { #[async_trait(?Send)] impl BuildConfigSimulator for CloneRpcClient { - async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> u64 { - let slice = SliceConfig { - offset: 0, - length: UpgradeableLoaderState::size_of_programdata_metadata(), - }; - let result = self.get_account_slice(program_id, Some(slice)).await; - if let Ok(Some(acc)) = result { - get_programdata_slot_from_account(&acc).expect("NO slot value for acc") - } else { - panic!("get_account_slice return an Error "); - } - } async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { Ok(ConfigSimulator::CloneRpcClient { program_id, @@ -116,9 +104,6 @@ impl BuildConfigSimulator for CloneRpcClient { #[async_trait(?Send)] impl BuildConfigSimulator for CallDbClient { - async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> u64 { - 0 - } async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { let mut simulator = SolanaSimulator::new_without_sync(self).await?; simulator.sync_accounts(self, &[program_id]).await?; @@ -302,7 +287,7 @@ pub async fn execute( rpc: &impl BuildConfigSimulator, program_id: Pubkey, ) -> NeonResult { - rpc.get_config_simulator(program_id).await + rpc.get_config(program_id).await } // static CHAINS_CACHE: OnceCell> = OnceCell::const_new(); @@ -311,7 +296,7 @@ pub async fn read_chains( rpc: &impl BuildConfigSimulator, program_id: Pubkey, ) -> NeonResult> { - Ok(rpc.get_config_simulator(program_id).await?.chains) + Ok(rpc.get_config(program_id).await?.chains) } async fn read_chain_id( diff --git a/evm_loader/lib/src/rpc/mod.rs b/evm_loader/lib/src/rpc/mod.rs index 3f2696b3b..9c1d53687 100644 --- a/evm_loader/lib/src/rpc/mod.rs +++ b/evm_loader/lib/src/rpc/mod.rs @@ -1,16 +1,16 @@ mod db_call_client; mod emulator_client; mod validator_client; - use crate::commands::get_config::GetConfigResponse; use crate::commands::get_config::{BuildConfigSimulator, ConfigSimulator}; use crate::{NeonError, NeonResult}; use async_trait::async_trait; pub use db_call_client::CallDbClient; use enum_dispatch::enum_dispatch; +use evm_loader::solana_program::bpf_loader_upgradeable::UpgradeableLoaderState; pub use solana_account_decoder::UiDataSliceConfig as SliceConfig; use solana_cli::cli::CliError; -use solana_client::client_error::Result as ClientResult; +use solana_client::client_error::{ClientErrorKind, Result as ClientResult}; use solana_sdk::{ account::Account, clock::{Slot, UnixTimestamp}, @@ -28,6 +28,20 @@ pub trait Rpc { key: &Pubkey, slice: Option, ) -> ClientResult>; + + async fn get_last_deployed_slot(&self, program_id: &Pubkey) -> ClientResult> { + let slice = SliceConfig { + offset: 0, + length: UpgradeableLoaderState::size_of_programdata_metadata(), + }; + let result = self.get_account_slice(program_id, Some(slice)).await; + if let Ok(Some(acc)) = result { + let slot = get_programdata_slot_from_account(&acc)?; + return Ok(Some(slot)); + } + Err(ClientErrorKind::Custom("Not account on slot ".to_string()).into()) + } + async fn get_account(&self, key: &Pubkey) -> ClientResult> { self.get_account_slice(key, None).await } @@ -61,6 +75,7 @@ macro_rules! e { }; } +use crate::types::programs_cache::get_programdata_slot_from_account; pub(crate) use e; pub(crate) async fn check_account_for_fee( From 9540f4608a6adca85dd0d1ab38ffbbcba537fd8b Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Mon, 2 Dec 2024 14:53:23 +0400 Subject: [PATCH 04/13] template struct instead 2 template methods --- evm_loader/lib/src/types/programs_cache.rs | 76 ++++++++++++---------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/evm_loader/lib/src/types/programs_cache.rs b/evm_loader/lib/src/types/programs_cache.rs index b80ebe7b9..2e1791ab8 100644 --- a/evm_loader/lib/src/types/programs_cache.rs +++ b/evm_loader/lib/src/types/programs_cache.rs @@ -26,10 +26,40 @@ pub struct KeyAccountCache { } use crate::rpc::SliceConfig; type ProgramDataCache = HashMap; -type ThreadSaveCache = RwLock>; + +struct ThreadSaveCache +where + Value: Clone, +{ + table: RwLock>, +} +impl ThreadSaveCache +where + Value: Clone, +{ + pub fn new() -> Self { + Self { + table: RwLock::new(HashMap::new()), + } + } + + fn get(&self, key: &KeyAccountCache) -> Option { + self.table + .read() + .expect("acc_hash_get_instance poisoned") + .get(key) + .cloned() + } + fn add(&self, key: KeyAccountCache, value: Value) { + self.table + .write() + .expect("PANIC, no space ") + .insert(key, value); + } +} type ThreadSaveProgramDataCache = ThreadSaveCache; -type ThreadSaveConfigCache<'a> = ThreadSaveCache; +type ThreadSaveConfigCache = ThreadSaveCache; static ACCOUNT_CACHE_TABLE: OnceCell = OnceCell::const_new(); static CONFIG_CACHE_TABLE: OnceCell = OnceCell::const_new(); @@ -42,41 +72,21 @@ pub async fn cut_programdata_from_acc(account: &mut Account, data_slice: SliceCo } account.data.truncate(data_slice.length); } -fn cache_get( - key: &KeyAccountCache, - table: &ThreadSaveCache, -) -> Option { - table - .read() - .expect("acc_hash_get_instance poisoned") - .get(key) - .cloned() -} -fn cache_add( - key: KeyAccountCache, - value: Value, - table: &ThreadSaveCache, -) { - table.write().expect("PANIC, no space ").insert(key, value); -} + async fn programdata_account_cache_get_instance() -> &'static ThreadSaveProgramDataCache { ACCOUNT_CACHE_TABLE - .get_or_init(|| async { - let map = HashMap::new(); - - RwLock::new(map) - }) + .get_or_init(|| async { ThreadSaveProgramDataCache::new() }) .await } async fn programdata_account_cache_get(addr: Pubkey, slot: u64) -> Option { - let val = KeyAccountCache { addr, slot }; - cache_get(&val, programdata_account_cache_get_instance().await) + let key = KeyAccountCache { addr, slot }; + programdata_account_cache_get_instance().await.get(&key) } async fn programdata_account_cache_add(addr: Pubkey, slot: u64, acc: Account) { let key = KeyAccountCache { addr, slot }; - cache_add(key, acc, programdata_account_cache_get_instance().await); + programdata_account_cache_get_instance().await.add(key, acc); } pub fn get_programdata_slot_from_account(acc: &Account) -> ClientResult { @@ -180,22 +190,18 @@ impl FakeRpc { } } -async fn program_config_cache_get_instance() -> &'static ThreadSaveConfigCache<'static> { +async fn program_config_cache_get_instance() -> &'static ThreadSaveConfigCache { CONFIG_CACHE_TABLE - .get_or_init(|| async { - let map = HashMap::new(); - - RwLock::new(map) - }) + .get_or_init(|| async { ThreadSaveConfigCache::new() }) .await } pub async fn program_config_cache_get(key: &KeyAccountCache) -> Option { - cache_get(key, program_config_cache_get_instance().await) + program_config_cache_get_instance().await.get(key) } pub async fn program_config_cache_add(key: KeyAccountCache, val: GetConfigResponse) { - cache_add(key, val, program_config_cache_get_instance().await); + program_config_cache_get_instance().await.add(key, val); } #[async_trait(?Send)] From 78f8e0fdc9f1ae594fd249f5d4f55d85a791b690 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Mon, 2 Dec 2024 15:17:47 +0400 Subject: [PATCH 05/13] CloneRpcClient last deployed slot always 0 --- evm_loader/lib/src/rpc/validator_client.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/evm_loader/lib/src/rpc/validator_client.rs b/evm_loader/lib/src/rpc/validator_client.rs index e8a0585c9..e62860be1 100644 --- a/evm_loader/lib/src/rpc/validator_client.rs +++ b/evm_loader/lib/src/rpc/validator_client.rs @@ -114,6 +114,9 @@ impl Deref for CloneRpcClient { #[async_trait(?Send)] impl Rpc for CloneRpcClient { + async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { + Ok(Some(0)) + } async fn get_account_slice( &self, key: &Pubkey, From 5b345efa6598de5b6228dfbbfa1b4cb874f3c89c Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Mon, 2 Dec 2024 16:05:48 +0400 Subject: [PATCH 06/13] try no CloneRpcClient depl slot 0 --- evm_loader/lib/src/rpc/validator_client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evm_loader/lib/src/rpc/validator_client.rs b/evm_loader/lib/src/rpc/validator_client.rs index e62860be1..edb2b7622 100644 --- a/evm_loader/lib/src/rpc/validator_client.rs +++ b/evm_loader/lib/src/rpc/validator_client.rs @@ -114,9 +114,9 @@ impl Deref for CloneRpcClient { #[async_trait(?Send)] impl Rpc for CloneRpcClient { - async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { - Ok(Some(0)) - } + // async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { + // Ok(Some(0)) + // } async fn get_account_slice( &self, key: &Pubkey, From 0e6a150c3acfee15edda88d1cc0f25ea27b21d77 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Tue, 3 Dec 2024 13:04:29 +0400 Subject: [PATCH 07/13] tests --- evm_loader/lib/src/commands/get_config.rs | 2 - evm_loader/lib/src/types/programs_cache.rs | 89 +++++++++++++++++++++- 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/evm_loader/lib/src/commands/get_config.rs b/evm_loader/lib/src/commands/get_config.rs index 52c83c39f..f0e3a3f0d 100644 --- a/evm_loader/lib/src/commands/get_config.rs +++ b/evm_loader/lib/src/commands/get_config.rs @@ -290,8 +290,6 @@ pub async fn execute( rpc.get_config(program_id).await } -// static CHAINS_CACHE: OnceCell> = OnceCell::const_new(); - pub async fn read_chains( rpc: &impl BuildConfigSimulator, program_id: Pubkey, diff --git a/evm_loader/lib/src/types/programs_cache.rs b/evm_loader/lib/src/types/programs_cache.rs index 2e1791ab8..835606acc 100644 --- a/evm_loader/lib/src/types/programs_cache.rs +++ b/evm_loader/lib/src/types/programs_cache.rs @@ -19,12 +19,14 @@ use std::sync::RwLock; use bincode::serialize; use tokio::sync::OnceCell; use tracing::info; -#[derive(Debug, Eq, PartialEq, Hash)] + +use crate::rpc::SliceConfig; +#[derive(Debug, Eq, PartialEq, Hash, Clone)] pub struct KeyAccountCache { pub addr: Pubkey, pub slot: u64, } -use crate::rpc::SliceConfig; + type ProgramDataCache = HashMap; struct ThreadSaveCache @@ -313,4 +315,87 @@ mod tests { assert!(multiple_accounts[i].is_some(), "BAD ACC"); } } + + #[test] + fn test_create_new_cache() { + let cache: ThreadSaveCache = ThreadSaveCache::new(); + assert!(cache + .get(&KeyAccountCache { + slot: 0, + addr: Pubkey::new_unique(), + }) + .is_none()); + } + + #[test] + fn test_add_and_get_value() { + let cache: ThreadSaveCache = ThreadSaveCache::new(); + let key = KeyAccountCache { + slot: 0, + addr: Pubkey::new_unique(), + }; + let value = "test_value".to_string(); + + // Add the value to the cache + cache.add(key.clone(), value.clone()); + + // Retrieve the value from the cache + let result = cache.get(&key); + assert!(result.is_some()); + assert_eq!(result.unwrap(), value); + } + + #[test] + fn test_get_nonexistent_key() { + let cache: ThreadSaveCache = ThreadSaveCache::new(); + let key = KeyAccountCache { + slot: 0, + addr: Pubkey::new_unique(), + }; + + // Attempt to get a value for a key that doesn't exist + assert!(cache.get(&key).is_none()); + } + + #[test] + fn test_overwrite_existing_key() { + let cache: ThreadSaveCache = ThreadSaveCache::new(); + let key = KeyAccountCache { + slot: 0, + addr: Pubkey::new_unique(), + }; + let value1 = "value1".to_string(); + let value2 = "value2".to_string(); + + // Add the first value + cache.add(key.clone(), value1.clone()); + assert_eq!(cache.get(&key).unwrap(), value1); + + // Overwrite with the second value + cache.add(key.clone(), value2.clone()); + assert_eq!(cache.get(&key).unwrap(), value2); + } + + #[test] + fn test_multiple_keys() { + let cache: ThreadSaveCache = ThreadSaveCache::new(); + let key1 = KeyAccountCache { + slot: 0, + addr: Pubkey::new_unique(), + }; + let value1 = "value1".to_string(); + let key2 = KeyAccountCache { + slot: 0, + addr: Pubkey::new_unique(), + }; + let value2 = "value2".to_string(); + + // Add multiple key-value pairs + cache.add(key1.clone(), value1.clone()); + cache.add(key2.clone(), value2.clone()); + + // Check values for both keys + assert_eq!(cache.get(&key1).unwrap(), value1); + assert_eq!(cache.get(&key2).unwrap(), value2); + } } From f06628b2ee3efc54286666c128c7f8b017584cb5 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Wed, 4 Dec 2024 15:54:37 +0400 Subject: [PATCH 08/13] get_last_deployed slot return None in case of not upgradeable acc --- .../api/src/api_server/handlers/get_config.rs | 2 +- evm_loader/lib/src/rpc/mod.rs | 2 +- evm_loader/lib/src/types/programs_cache.rs | 33 ++++++++++--------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/evm_loader/api/src/api_server/handlers/get_config.rs b/evm_loader/api/src/api_server/handlers/get_config.rs index 0d6584bed..4de5c1bf5 100644 --- a/evm_loader/api/src/api_server/handlers/get_config.rs +++ b/evm_loader/api/src/api_server/handlers/get_config.rs @@ -20,7 +20,7 @@ pub async fn get_config(state: NeonApiState, request_id: RequestId) -> impl Resp Ok(rpc) => rpc, Err(e) => return process_error(StatusCode::BAD_REQUEST, &e), }; - // Entry GetConfigResponse + process_result( &GetConfigCommand::execute(&rpc, state.config.evm_loader) .await diff --git a/evm_loader/lib/src/rpc/mod.rs b/evm_loader/lib/src/rpc/mod.rs index 9c1d53687..a909a5692 100644 --- a/evm_loader/lib/src/rpc/mod.rs +++ b/evm_loader/lib/src/rpc/mod.rs @@ -37,7 +37,7 @@ pub trait Rpc { let result = self.get_account_slice(program_id, Some(slice)).await; if let Ok(Some(acc)) = result { let slot = get_programdata_slot_from_account(&acc)?; - return Ok(Some(slot)); + return Ok(slot); } Err(ClientErrorKind::Custom("Not account on slot ".to_string()).into()) } diff --git a/evm_loader/lib/src/types/programs_cache.rs b/evm_loader/lib/src/types/programs_cache.rs index 835606acc..d99c57450 100644 --- a/evm_loader/lib/src/types/programs_cache.rs +++ b/evm_loader/lib/src/types/programs_cache.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use crate::commands::get_config::GetConfigResponse; use bincode::deserialize; use futures::future::join_all; -use solana_client::client_error::{ClientErrorKind, Result as ClientResult}; +use solana_client::client_error::Result as ClientResult; use solana_sdk::{ account::Account, bpf_loader_upgradeable::UpgradeableLoaderState, @@ -91,16 +91,15 @@ async fn programdata_account_cache_add(addr: Pubkey, slot: u64, acc: Account) { programdata_account_cache_get_instance().await.add(key, acc); } -pub fn get_programdata_slot_from_account(acc: &Account) -> ClientResult { +/// in case of Not upgradeable account - return option None +pub fn get_programdata_slot_from_account(acc: &Account) -> ClientResult> { if !bpf_loader_upgradeable::check_id(&acc.owner) { - return Err(ClientErrorKind::Custom("Not upgradeable account".to_string()).into()); + return Ok(None); } match deserialize::(&acc.data) { - Ok(UpgradeableLoaderState::ProgramData { slot, .. }) => Ok(slot), - Ok(_) => { - panic!("Account is not of type `ProgramData`."); - } + Ok(UpgradeableLoaderState::ProgramData { slot, .. }) => Ok(Some(slot)), + Ok(_) => Ok(None), Err(e) => { eprintln!("Error occurred: {e:?}"); panic!("Failed to deserialize account data."); @@ -134,14 +133,18 @@ pub async fn programdata_cache_get_values_by_keys( for (result, key) in results.iter().zip(programdata_keys) { match result { Ok(Some(account)) => { - let slot_val = get_programdata_slot_from_account(account)?; - if let Some(acc) = programdata_account_cache_get(*key, slot_val).await { - answer.push(Some(acc)); - } else if let Ok(Some(tmp_acc)) = rpc.get_account(key).await { - let current_slot = get_programdata_slot_from_account(&tmp_acc)?; - programdata_account_cache_add(*key, current_slot, tmp_acc.clone()).await; - - answer.push(Some(tmp_acc)); + if let Some(slot_val) = get_programdata_slot_from_account(account)? { + if let Some(acc) = programdata_account_cache_get(*key, slot_val).await { + answer.push(Some(acc)); + } else if let Ok(Some(tmp_acc)) = rpc.get_account(key).await { + let current_slot = + get_programdata_slot_from_account(&tmp_acc)?.expect("No current slot "); + programdata_account_cache_add(*key, current_slot, tmp_acc.clone()).await; + + answer.push(Some(tmp_acc)); + } else { + answer.push(None); + } } else { answer.push(None); } From 6b5bd8cd9d7fa042212717e85e0836d6ea118829 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Wed, 4 Dec 2024 17:00:47 +0400 Subject: [PATCH 09/13] handle option error --- evm_loader/lib/src/commands/get_config.rs | 31 ++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/evm_loader/lib/src/commands/get_config.rs b/evm_loader/lib/src/commands/get_config.rs index f0e3a3f0d..e5846c5e4 100644 --- a/evm_loader/lib/src/commands/get_config.rs +++ b/evm_loader/lib/src/commands/get_config.rs @@ -62,17 +62,17 @@ pub enum ConfigSimulator<'r> { pub trait BuildConfigSimulator: Rpc { async fn get_config(&self, program_id: Pubkey) -> NeonResult { let maybe_slot = self.get_last_deployed_slot(&program_id).await?; - let slot = maybe_slot.expect("Account should have slot "); - - let key = KeyAccountCache { - addr: program_id, - slot, - }; - - let rz = program_config_cache_get(&key).await; - if rz.is_some() { - return Ok(rz.unwrap()); - }; + if let Some(slot) = maybe_slot { + let key = KeyAccountCache { + addr: program_id, + slot, + }; + + let rz = program_config_cache_get(&key).await; + if rz.is_some() { + return Ok(rz.unwrap()); + }; + } let mut simulator = self.build_config_simulator(program_id).await?; let (version, revision) = simulator.get_version().await?; @@ -85,7 +85,14 @@ pub trait BuildConfigSimulator: Rpc { chains: simulator.get_chains().await?, config: simulator.get_properties().await?, }; - program_config_cache_add(key, result.clone()).await; + if let Some(slot) = maybe_slot { + let key = KeyAccountCache { + addr: program_id, + slot, + }; + program_config_cache_add(key, result.clone()).await; + } + Ok(result) } From 63d3c490776f1f5ea807d052bd5d8c666352f836 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Wed, 4 Dec 2024 19:02:04 +0400 Subject: [PATCH 10/13] Revert "try no CloneRpcClient depl slot 0" This reverts commit 11e059ddce6a5dec9e2fb924cd3baafd2a83e61e. --- evm_loader/lib/src/rpc/validator_client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evm_loader/lib/src/rpc/validator_client.rs b/evm_loader/lib/src/rpc/validator_client.rs index edb2b7622..e62860be1 100644 --- a/evm_loader/lib/src/rpc/validator_client.rs +++ b/evm_loader/lib/src/rpc/validator_client.rs @@ -114,9 +114,9 @@ impl Deref for CloneRpcClient { #[async_trait(?Send)] impl Rpc for CloneRpcClient { - // async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { - // Ok(Some(0)) - // } + async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { + Ok(Some(0)) + } async fn get_account_slice( &self, key: &Pubkey, From 88a1e2a883318ce0fbbc9b1e2f984b63ddacf7f1 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Thu, 5 Dec 2024 09:38:53 +0400 Subject: [PATCH 11/13] fix review feedback --- evm_loader/lib/src/rpc/validator_client.rs | 1 + evm_loader/lib/src/types/programs_cache.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/evm_loader/lib/src/rpc/validator_client.rs b/evm_loader/lib/src/rpc/validator_client.rs index e62860be1..d912a472f 100644 --- a/evm_loader/lib/src/rpc/validator_client.rs +++ b/evm_loader/lib/src/rpc/validator_client.rs @@ -115,6 +115,7 @@ impl Deref for CloneRpcClient { #[async_trait(?Send)] impl Rpc for CloneRpcClient { async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { + // Update event impossible during CloneRpcClient work. Returns always 0 for caching. Ok(Some(0)) } async fn get_account_slice( diff --git a/evm_loader/lib/src/types/programs_cache.rs b/evm_loader/lib/src/types/programs_cache.rs index d99c57450..a72a9fbef 100644 --- a/evm_loader/lib/src/types/programs_cache.rs +++ b/evm_loader/lib/src/types/programs_cache.rs @@ -48,14 +48,14 @@ where fn get(&self, key: &KeyAccountCache) -> Option { self.table .read() - .expect("acc_hash_get_instance poisoned") + .expect("lock on read error ") .get(key) .cloned() } fn add(&self, key: KeyAccountCache, value: Value) { self.table .write() - .expect("PANIC, no space ") + .expect("lock on write error ") .insert(key, value); } } From 823a92a69dec645a07759375aa4e18ed0f6a8e29 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Thu, 5 Dec 2024 10:10:55 +0400 Subject: [PATCH 12/13] remove get_last_depl slot for Rpc --- evm_loader/lib/src/rpc/validator_client.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/evm_loader/lib/src/rpc/validator_client.rs b/evm_loader/lib/src/rpc/validator_client.rs index d912a472f..e8a0585c9 100644 --- a/evm_loader/lib/src/rpc/validator_client.rs +++ b/evm_loader/lib/src/rpc/validator_client.rs @@ -114,10 +114,6 @@ impl Deref for CloneRpcClient { #[async_trait(?Send)] impl Rpc for CloneRpcClient { - async fn get_last_deployed_slot(&self, _program_id: &Pubkey) -> ClientResult> { - // Update event impossible during CloneRpcClient work. Returns always 0 for caching. - Ok(Some(0)) - } async fn get_account_slice( &self, key: &Pubkey, From 4161c2ee74632d94e8c76c22436c322ae38e5484 Mon Sep 17 00:00:00 2001 From: Novikov Roman Date: Thu, 5 Dec 2024 11:27:00 +0400 Subject: [PATCH 13/13] revert get_chains logic --- evm_loader/lib/src/account_storage_tests.rs | 3 +++ evm_loader/lib/src/commands/get_config.rs | 23 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/evm_loader/lib/src/account_storage_tests.rs b/evm_loader/lib/src/account_storage_tests.rs index fac8c7997..98a31fc4f 100644 --- a/evm_loader/lib/src/account_storage_tests.rs +++ b/evm_loader/lib/src/account_storage_tests.rs @@ -91,6 +91,9 @@ mod mock_rpc_client { #[async_trait(?Send)] impl BuildConfigSimulator for MockRpcClient { + fn use_cache_for_chains(&self) -> bool { + false + } async fn build_config_simulator(&self, _program_id: Pubkey) -> NeonResult { unimplemented!(); } diff --git a/evm_loader/lib/src/commands/get_config.rs b/evm_loader/lib/src/commands/get_config.rs index e5846c5e4..35097ba49 100644 --- a/evm_loader/lib/src/commands/get_config.rs +++ b/evm_loader/lib/src/commands/get_config.rs @@ -17,7 +17,7 @@ use solana_sdk::signer::Signer; use solana_sdk::{instruction::Instruction, pubkey::Pubkey, transaction::Transaction}; use crate::NeonResult; - +use tokio::sync::OnceCell; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Status { Ok, @@ -60,6 +60,7 @@ pub enum ConfigSimulator<'r> { #[async_trait(?Send)] #[enum_dispatch] pub trait BuildConfigSimulator: Rpc { + fn use_cache_for_chains(&self) -> bool; async fn get_config(&self, program_id: Pubkey) -> NeonResult { let maybe_slot = self.get_last_deployed_slot(&program_id).await?; if let Some(slot) = maybe_slot { @@ -101,6 +102,9 @@ pub trait BuildConfigSimulator: Rpc { #[async_trait(?Send)] impl BuildConfigSimulator for CloneRpcClient { + fn use_cache_for_chains(&self) -> bool { + true + } async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { Ok(ConfigSimulator::CloneRpcClient { program_id, @@ -111,6 +115,9 @@ impl BuildConfigSimulator for CloneRpcClient { #[async_trait(?Send)] impl BuildConfigSimulator for CallDbClient { + fn use_cache_for_chains(&self) -> bool { + false + } async fn build_config_simulator(&self, program_id: Pubkey) -> NeonResult { let mut simulator = SolanaSimulator::new_without_sync(self).await?; simulator.sync_accounts(self, &[program_id]).await?; @@ -289,7 +296,7 @@ impl ConfigSimulator<'_> { Ok(result) } } - +static CHAINS_CACHE: OnceCell> = OnceCell::const_new(); pub async fn execute( rpc: &impl BuildConfigSimulator, program_id: Pubkey, @@ -301,6 +308,18 @@ pub async fn read_chains( rpc: &impl BuildConfigSimulator, program_id: Pubkey, ) -> NeonResult> { + if rpc.use_cache_for_chains() { + let result = CHAINS_CACHE + .get_or_init(|| async { + rpc.get_config(program_id) + .await + .expect(" get config error for chain info") + .chains + }) + .await + .clone(); + return Ok(result); + } Ok(rpc.get_config(program_id).await?.chains) }