Ndev 3050 rocks db (#525)

* Add working with TracerDB based on RocksDB * Add TRACER_DB_TYPE to select ClickHouse or RocksDB implementation --------- Co-authored-by: Semen Medvedev <[email protected]>
neonlabsorg · Sep 4, 2024 · d7ae8d2 · d7ae8d2
1 parent ae50b87
commit d7ae8d2
Show file tree

Hide file tree

Showing 15 changed files with 732 additions and 256 deletions.
diff --git a/ci/docker-compose-ci.yml b/ci/docker-compose-ci.yml
@@ -35,6 +35,7 @@ services:
       SOLANA_KEY_FOR_CONFIG: BMp6gEnveANdvSvspESJUrNczuHz1GF5UQKjVLCkAZih
       COMMITMENT: confirmed
       NEON_DB_CLICKHOUSE_URLS: "http://45.250.253.36:8123;http://45.250.253.38:8123"
+      TRACER_DB_TYPE: clickhouse
     image: ${EVM_LOADER_IMAGE}
     ports:
     - "8085"
@@ -57,6 +58,7 @@ services:
       NEON_CHAIN_ID: 111
       COMMITMENT: confirmed
       NEON_DB_CLICKHOUSE_URLS: "http://45.250.253.36:8123;http://45.250.253.38:8123"
+      TRACER_DB_TYPE: clickhouse
       NEON_DB_INDEXER_HOST: 45.250.253.32
       NEON_DB_INDEXER_PORT: 5432
       NEON_DB_INDEXER_DATABASE: indexer
@@ -65,6 +67,7 @@ services:
       KEYPAIR: /opt/operator-keypairs/id.json
       FEEPAIR: /opt/operator-keypairs/id.json
       SOLANA_KEY_FOR_CONFIG: BMp6gEnveANdvSvspESJUrNczuHz1GF5UQKjVLCkAZih
+
     image: ${EVM_LOADER_IMAGE}
     ports:
       - "3100"

diff --git a/ci/docker-compose-test.yml b/ci/docker-compose-test.yml
@@ -38,6 +38,7 @@ services:
       SOLANA_KEY_FOR_CONFIG: BMp6gEnveANdvSvspESJUrNczuHz1GF5UQKjVLCkAZih
       COMMITMENT: confirmed
       NEON_DB_CLICKHOUSE_URLS: "http://45.250.253.36:8123;http://45.250.253.38:8123"
+      TRACER_DB_TYPE: clickhouse
     image: ${EVM_LOADER_IMAGE}
     ports:
     - "8085:8085"
@@ -61,6 +62,7 @@ services:
       NEON_CHAIN_ID: 111
       COMMITMENT: confirmed
       NEON_DB_CLICKHOUSE_URLS: "http://45.250.253.36:8123;http://45.250.253.38:8123"
+      TRACER_DB_TYPE: clickhouse
       NEON_DB_INDEXER_HOST: 45.250.253.32
       NEON_DB_INDEXER_PORT: 5432
       NEON_DB_INDEXER_DATABASE: indexer

diff --git a/evm_loader/Cargo.lock b/evm_loader/Cargo.lock
diff --git a/evm_loader/api/src/main.rs b/evm_loader/api/src/main.rs
@@ -54,7 +54,7 @@ async fn main() -> NeonApiResult<()> {
     info!("{}", get_build_info());
 
     let api_config = config::load_api_config_from_environment();
-    let state: NeonApiState = Data::new(State::new(api_config));
+    let state: NeonApiState = Data::new(State::new(api_config).await);
 
     let listener_addr = options
         .value_of("host")

diff --git a/evm_loader/cli/src/main.rs b/evm_loader/cli/src/main.rs
@@ -155,14 +155,12 @@ async fn build_rpc(options: &ArgMatches<'_>, config: &Config) -> Result<RpcEnum,
         .map(|slot_str| slot_str.parse().expect("slot parse error"));
 
     Ok(if let Some(slot) = slot {
-        RpcEnum::CallDbClient(
-            CallDbClient::new(
-                TracerDb::new(config.db_config.as_ref().expect("db-config not found")),
-                slot,
-                None,
-            )
-            .await?,
-        )
+        let db_config = config
+            .db_config
+            .clone()
+            .ok_or(NeonError::LoadingDBConfigError)?;
+        let tracer_db = TracerDb::from_config(&db_config).await;
+        RpcEnum::CallDbClient(CallDbClient::new(tracer_db, slot, None).await?)
     } else {
         RpcEnum::CloneRpcClient(CloneRpcClient::new_from_config(config))
     })

diff --git a/evm_loader/lib/Cargo.toml b/evm_loader/lib/Cargo.toml
@@ -11,6 +11,7 @@ anyhow = "1.0"
 bincode = "1.3.3"
 hyper = "0.14"
 evm-loader = { path = "../program", default-features = false, features = ["log", "async-trait"] }
+jsonrpsee = { version = "0.20", features = ["server", "ws-client", "macros"] }
 solana-sdk.workspace = true
 solana-client.workspace = true
 solana-account-decoder.workspace = true

diff --git a/evm_loader/lib/src/abi/mod.rs b/evm_loader/lib/src/abi/mod.rs
@@ -29,7 +29,14 @@ use serde_json::json;
 
 lazy_static! {
     static ref RUNTIME: tokio::runtime::Runtime = tokio::runtime::Runtime::new().unwrap();
-    static ref STATE: State = State::new(load_config());
+    static ref STATE: State = init_state_sync();
+}
+
+#[must_use]
+pub fn init_state_sync() -> State {
+    tokio::runtime::Runtime::new()
+        .unwrap()
+        .block_on(async { State::new(load_config()).await })
 }
 
 pub const _MODULE_WM_: &WithMetadata<NeonEVMLib> = &WithMetadata::new(NeonEVMLib {

diff --git a/evm_loader/lib/src/abi/state.rs b/evm_loader/lib/src/abi/state.rs
@@ -4,16 +4,16 @@ use crate::types::TracerDb;
 use crate::NeonError;
 
 pub struct State {
-    pub tracer_db: TracerDb,
+    pub tracer_db: Option<TracerDb>,
     pub rpc_client: CloneRpcClient,
     pub config: APIOptions,
 }
 
 impl State {
     #[must_use]
-    pub fn new(config: APIOptions) -> Self {
+    pub async fn new(config: APIOptions) -> Self {
         Self {
-            tracer_db: TracerDb::new(&config.db_config),
+            tracer_db: TracerDb::maybe_from_config(&config.db_config).await,
             rpc_client: CloneRpcClient::new_from_api_config(&config),
             config,
         }
@@ -26,7 +26,14 @@ impl State {
     ) -> Result<RpcEnum, NeonError> {
         Ok(if let Some(slot) = slot {
             RpcEnum::CallDbClient(
-                CallDbClient::new(self.tracer_db.clone(), slot, tx_index_in_block).await?,
+                CallDbClient::new(
+                    self.tracer_db
+                        .clone()
+                        .expect("TracerDB must be configured for CallDbClient"),
+                    slot,
+                    tx_index_in_block,
+                )
+                .await?,
             )
         } else {
             RpcEnum::CloneRpcClient(self.rpc_client.clone())

diff --git a/evm_loader/lib/src/commands/init_environment.rs b/evm_loader/lib/src/commands/init_environment.rs
@@ -135,14 +135,15 @@ pub async fn execute(
     let program_parameters = Parameters::new(read_elf_parameters(config, &data));
 
     let neon_revision = program_parameters.get::<String>("NEON_REVISION")?;
-    if neon_revision != env!("NEON_REVISION") {
+    let env_neon_revision = env!("NEON_REVISION");
+    if neon_revision != env_neon_revision {
         if force {
             warn!("NeonEVM revision doesn't match CLI revision. This check has been disabled with `--force` flag");
         } else {
             error!("NeonEVM revision doesn't match CLI revision. Use appropriate neon-cli version or add `--force` flag");
             return Err(EnvironmentError::RevisionMismatch(
                 neon_revision,
-                env!("NEON_REVISION").to_string(),
+                env_neon_revision.to_string(),
             )
             .into());
         }