From 111a367c5a41a03e48e1c0e8db0b5eb8abf347aa Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 21 Jul 2023 15:57:12 -0400 Subject: [PATCH 01/12] feat: parser optimizations --- .../chainhook-sdk/src/chainhooks/tests/mod.rs | 2 +- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 63 ++++++++----------- components/chainhook-sdk/src/lib.rs | 1 + 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/components/chainhook-sdk/src/chainhooks/tests/mod.rs b/components/chainhook-sdk/src/chainhooks/tests/mod.rs index 389f71714..4c823931f 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/tests/mod.rs @@ -580,4 +580,4 @@ fn test_stacks_hook_action_file_append() { else { panic!("wrong occurence type"); } -} \ No newline at end of file +} diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index e6ec9c886..39a4f212b 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -8,12 +8,9 @@ use crate::chainhooks::types::{ use crate::observer::BitcoinConfig; use crate::utils::Context; -use bitcoincore_rpc::bitcoin::hashes::hex::FromHex; use bitcoincore_rpc::bitcoin::hashes::Hash; use bitcoincore_rpc::bitcoin::{self, Address, Amount, BlockHash}; -use bitcoincore_rpc_json::{ - GetRawTransactionResultVinScriptSig, GetRawTransactionResultVoutScriptPubKey, -}; +use bitcoincore_rpc_json::GetRawTransactionResultVoutScriptPubKey; pub use blocks_pool::BitcoinBlockPool; use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut}; use chainhook_types::{ @@ -29,13 +26,12 @@ use serde::Deserialize; #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct BitcoinBlockFullBreakdown { - pub hash: bitcoin::BlockHash, + pub hash: String, pub height: usize, - pub merkleroot: bitcoin::TxMerkleNode, pub tx: Vec, pub time: usize, pub nonce: u32, - pub previousblockhash: Option, + pub previousblockhash: Option, } impl BitcoinBlockFullBreakdown { @@ -47,10 +43,13 @@ impl BitcoinBlockFullBreakdown { hash: hash, }; // Parent block id - let hash = format!("0x{}", self.previousblockhash.unwrap().to_string()); + let parent_block_hash = match self.previousblockhash { + Some(ref value) => format!("0x{}", value), + None => format!("0x{}", BlockHash::all_zeros().to_string()), + }; let parent_block_identifier = BlockIdentifier { index: (self.height - 1) as u64, - hash, + hash: parent_block_hash, }; BlockHeader { block_identifier, @@ -62,7 +61,7 @@ impl BitcoinBlockFullBreakdown { #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct BitcoinTransactionFullBreakdown { - pub txid: bitcoin::Txid, + pub txid: String, pub vin: Vec, pub vout: Vec, } @@ -72,45 +71,34 @@ pub struct BitcoinTransactionFullBreakdown { pub struct BitcoinTransactionInputFullBreakdown { pub sequence: u32, /// The raw scriptSig in case of a coinbase tx. - #[serde(default, with = "bitcoincore_rpc_json::serde_hex::opt")] - pub coinbase: Option>, + // #[serde(default, with = "bitcoincore_rpc_json::serde_hex::opt")] + // pub coinbase: Option>, /// Not provided for coinbase txs. - pub txid: Option, + pub txid: Option, /// Not provided for coinbase txs. pub vout: Option, /// The scriptSig in case of a non-coinbase tx. pub script_sig: Option, /// Not provided for coinbase txs. - #[serde(default, deserialize_with = "deserialize_hex_array_opt")] - pub txinwitness: Option>>, + pub txinwitness: Option>, pub prevout: Option, } +#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GetRawTransactionResultVinScriptSig { + pub hex: String, +} + impl BitcoinTransactionInputFullBreakdown { /// Whether this input is from a coinbase tx. /// The [txid], [vout] and [script_sig] fields are not provided /// for coinbase transactions. pub fn is_coinbase(&self) -> bool { - self.coinbase.is_some() + self.txid.is_none() } } -fn deserialize_hex_array_opt<'de, D>(deserializer: D) -> Result>>, D::Error> -where - D: serde::Deserializer<'de>, -{ - use crate::serde::de::Error; - - //TODO(stevenroose) Revisit when issue is fixed: - // https://github.com/serde-rs/serde/issues/723 - let v: Vec = Vec::deserialize(deserializer)?; - let mut res = Vec::new(); - for h in v.into_iter() { - res.push(FromHex::from_hex(&h).map_err(D::Error::custom)?); - } - Ok(Some(res)) -} - #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct BitcoinTransactionInputPrevoutFullBreakdown { @@ -395,6 +383,7 @@ pub fn standardize_bitcoin_block( ))?; sats_in += prevout.value.to_sat(); + inputs.push(TxIn { previous_output: OutPoint { txid: TransactionIdentifier::new(&txid.to_string()), @@ -402,16 +391,16 @@ pub fn standardize_bitcoin_block( block_height: prevout.height, value: prevout.value.to_sat(), }, - script_sig: format!("0x{}", hex::encode(&script_sig.hex)), + script_sig: format!("0x{}", script_sig.hex), sequence: input.sequence, witness: input .txinwitness .unwrap_or(vec![]) .to_vec() .iter() - .map(|w| format!("0x{}", hex::encode(w))) + .map(|w| format!("0x{}", w)) .collect::>(), - }) + }); } let mut outputs = vec![]; @@ -450,7 +439,9 @@ pub fn standardize_bitcoin_block( parent_block_identifier: BlockIdentifier { hash: format!( "0x{}", - block.previousblockhash.unwrap_or(BlockHash::all_zeros()) + block + .previousblockhash + .unwrap_or(BlockHash::all_zeros().to_string()) ), index: block_height - 1, }, diff --git a/components/chainhook-sdk/src/lib.rs b/components/chainhook-sdk/src/lib.rs index 144ad8875..4cb4bd984 100644 --- a/components/chainhook-sdk/src/lib.rs +++ b/components/chainhook-sdk/src/lib.rs @@ -9,6 +9,7 @@ extern crate serde_derive; #[macro_use] extern crate serde_json; +#[cfg(test)] #[macro_use] extern crate lazy_static; From 58c157f627c76223496c63dd3a152f67f27efe91 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 21 Jul 2023 20:51:35 -0400 Subject: [PATCH 02/12] fix: add exponential backoff --- Cargo.lock | 2 +- components/chainhook-sdk/Cargo.toml | 2 +- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 22 +++++++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eaa0715ff..823d012f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,7 +457,7 @@ dependencies = [ [[package]] name = "chainhook-sdk" -version = "0.7.5" +version = "0.7.7" dependencies = [ "base58 0.2.0", "base64", diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index fb8f6236b..f06ff964b 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook-sdk" -version = "0.7.5" +version = "0.7.7" description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin" license = "GPL-3.0" edition = "2021" diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index 39a4f212b..33b92c3d5 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -20,6 +20,7 @@ use chainhook_types::{ StacksBlockCommitmentData, TransactionIdentifier, TransferSTXData, }; use hiro_system_kit::slog; +use rand::{thread_rng, Rng}; use reqwest::Client as HttpClient; use serde::Deserialize; @@ -231,19 +232,26 @@ pub async fn download_block_with_retry( ctx: &Context, ) -> Result { let mut errors_count = 0; + let mut backoff: f64 = 1.0; + let mut rng = thread_rng(); + let block = loop { let response = { match download_block(http_client, block_hash, bitcoin_config, ctx).await { Ok(result) => result, Err(_e) => { errors_count += 1; - ctx.try_log(|logger| { - slog::warn!( - logger, - "unable to fetch block #{block_hash}: will retry in a few seconds (attempt #{errors_count}).", - ) - }); - std::thread::sleep(std::time::Duration::from_millis(500)); + backoff = 2.0 * backoff + (backoff * rng.gen_range(0.0..1.0)); + let duration = std::time::Duration::from_millis((backoff * 1_000.0) as u64); + if errors_count > 1 { + ctx.try_log(|logger| { + slog::warn!( + logger, + "unable to fetch block #{block_hash}: will retry in a few seconds (attempt #{errors_count}).", + ) + }); + } + std::thread::sleep(duration); continue; } } From 2380303134e43180ea3263344a407938ed4f5e14 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sat, 22 Jul 2023 00:55:27 -0400 Subject: [PATCH 03/12] feat: introduce new rest method --- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 76 ++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index 33b92c3d5..e6c1ce762 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -135,7 +135,12 @@ pub struct RewardParticipant { pub fn build_http_client() -> HttpClient { HttpClient::builder() - .timeout(Duration::from_secs(20)) + .timeout(Duration::from_secs(15)) + .http1_only() + .no_trust_dns() + .connect_timeout(Duration::from_secs(15)) + .tcp_keepalive(Some(Duration::from_secs(15))) + .no_proxy() .danger_accept_invalid_certs(true) .build() .expect("Unable to build http client") @@ -225,6 +230,46 @@ pub async fn retrieve_block_hash( Ok(block_hash) } + +pub async fn try_fetch_block_bytes_with_retry( + http_client: HttpClient, + block_height: u64, + bitcoin_config: BitcoinConfig, + ctx: Context, +) -> Result, String> { + let block_hash = retrieve_block_hash_with_retry( + &http_client, + &block_height, + &bitcoin_config, + &ctx, + ).await.unwrap(); + + let mut errors_count = 0; + let mut backoff: f64 = 1.0; + + let response = loop { + match fetch_block(&http_client, &block_hash, &bitcoin_config, &ctx).await { + Ok(result) => break result, + Err(_e) => { + errors_count += 1; + backoff = 2.0 * backoff; + let duration = std::time::Duration::from_millis((backoff * 1_000.0) as u64); + if errors_count > 1 { + ctx.try_log(|logger| { + slog::warn!( + logger, + "unable to fetch block #{block_hash}: will retry in a few seconds (attempt #{errors_count}).", + ) + }); + } + std::thread::sleep(duration); + continue; + } + } + }; + Ok(response) +} + pub async fn download_block_with_retry( http_client: &HttpClient, block_hash: &str, @@ -313,6 +358,35 @@ pub fn parse_downloaded_block( Ok(block) } +pub async fn fetch_block( + http_client: &HttpClient, + block_hash: &str, + bitcoin_config: &BitcoinConfig, + _ctx: &Context, +) -> Result, String> { + let block = http_client + .get(format!("{}/rest/block/{}.json", bitcoin_config.rpc_url, block_hash)) + .header("Content-Type", "application/json") + .header("Host", &bitcoin_config.rpc_url[7..]) + .send() + .await + .map_err(|e| format!("unable to send request ({})", e))? + .bytes() + .await + .map_err(|e| format!("unable to get bytes ({})", e))? + .to_vec(); + Ok(block) +} + +pub fn parse_fetched_block( + downloaded_block: Vec, +) -> Result { + let block = serde_json::from_slice::(&downloaded_block[..]) + .map_err(|e: serde_json::Error| format!("unable to parse block ({})", e))?; + Ok(block) +} + + pub async fn download_and_parse_block( http_client: &HttpClient, block_hash: &str, From 5b4994ffb7c71438dff6ce8d407132c973e399ee Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 31 Jul 2023 09:21:00 +0200 Subject: [PATCH 04/12] fix: revisit exponential backoff --- Cargo.lock | 2 +- components/chainhook-sdk/Cargo.toml | 2 +- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 60 ++++++++++++++----- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 823d012f0..68963e070 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,7 +457,7 @@ dependencies = [ [[package]] name = "chainhook-sdk" -version = "0.7.7" +version = "0.7.10" dependencies = [ "base58 0.2.0", "base64", diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index f06ff964b..19e222572 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook-sdk" -version = "0.7.7" +version = "0.7.10" description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin" license = "GPL-3.0" edition = "2021" diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index e6c1ce762..e3ef2735a 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -230,30 +230,24 @@ pub async fn retrieve_block_hash( Ok(block_hash) } - pub async fn try_fetch_block_bytes_with_retry( http_client: HttpClient, block_height: u64, bitcoin_config: BitcoinConfig, ctx: Context, ) -> Result, String> { - let block_hash = retrieve_block_hash_with_retry( - &http_client, - &block_height, - &bitcoin_config, - &ctx, - ).await.unwrap(); + let block_hash = + retrieve_block_hash_with_retry(&http_client, &block_height, &bitcoin_config, &ctx) + .await + .unwrap(); let mut errors_count = 0; - let mut backoff: f64 = 1.0; let response = loop { match fetch_block(&http_client, &block_hash, &bitcoin_config, &ctx).await { Ok(result) => break result, Err(_e) => { errors_count += 1; - backoff = 2.0 * backoff; - let duration = std::time::Duration::from_millis((backoff * 1_000.0) as u64); if errors_count > 1 { ctx.try_log(|logger| { slog::warn!( @@ -262,7 +256,41 @@ pub async fn try_fetch_block_bytes_with_retry( ) }); } - std::thread::sleep(duration); + std::thread::sleep(std::time::Duration::from_millis(1500)); + continue; + } + } + }; + Ok(response) +} + +pub async fn try_download_block_bytes_with_retry( + http_client: HttpClient, + block_height: u64, + bitcoin_config: BitcoinConfig, + ctx: Context, +) -> Result, String> { + let block_hash = + retrieve_block_hash_with_retry(&http_client, &block_height, &bitcoin_config, &ctx) + .await + .unwrap(); + + let mut errors_count = 0; + + let response = loop { + match download_block(&http_client, &block_hash, &bitcoin_config, &ctx).await { + Ok(result) => break result, + Err(_e) => { + errors_count += 1; + if errors_count > 1 { + ctx.try_log(|logger| { + slog::warn!( + logger, + "unable to fetch block #{block_hash}: will retry in a few seconds (attempt #{errors_count}).", + ) + }); + } + std::thread::sleep(std::time::Duration::from_millis(1500)); continue; } } @@ -365,7 +393,10 @@ pub async fn fetch_block( _ctx: &Context, ) -> Result, String> { let block = http_client - .get(format!("{}/rest/block/{}.json", bitcoin_config.rpc_url, block_hash)) + .get(format!( + "{}/rest/block/{}.json", + bitcoin_config.rpc_url, block_hash + )) .header("Content-Type", "application/json") .header("Host", &bitcoin_config.rpc_url[7..]) .send() @@ -378,15 +409,12 @@ pub async fn fetch_block( Ok(block) } -pub fn parse_fetched_block( - downloaded_block: Vec, -) -> Result { +pub fn parse_fetched_block(downloaded_block: Vec) -> Result { let block = serde_json::from_slice::(&downloaded_block[..]) .map_err(|e: serde_json::Error| format!("unable to parse block ({})", e))?; Ok(block) } - pub async fn download_and_parse_block( http_client: &HttpClient, block_hash: &str, From 3dd76e8f34b6b0ac5b5873223c41fa5c5dadf0be Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 31 Jul 2023 10:20:54 +0200 Subject: [PATCH 05/12] fix: coinbase tx fee handling --- components/chainhook-sdk/src/indexer/bitcoin/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index e3ef2735a..88a5ce9d4 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -535,7 +535,7 @@ pub fn standardize_bitcoin_block( stacks_operations, ordinal_operations: vec![], proof: None, - fee: sats_in - sats_out, + fee: sats_in.saturating_sub(sats_out), }, }; transactions.push(tx); From a51bc27c49cc63e5179e457d658a9ec0791c33f6 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 31 Jul 2023 10:22:00 +0200 Subject: [PATCH 06/12] fix: log messages --- components/chainhook-cli/src/storage/mod.rs | 6 ++---- components/chainhook-sdk/src/chainhooks/types.rs | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index 4e053ddd6..2868301b2 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -193,10 +193,8 @@ pub fn get_stacks_block_at_block_height( }) { Ok(Some(entry)) => { return Ok(Some({ - let spec: StacksBlockData = - serde_json::from_slice(&entry[..]).map_err(|e| { - format!("unable to deserialize Stacks chainhook {}", e.to_string()) - })?; + let spec: StacksBlockData = serde_json::from_slice(&entry[..]) + .map_err(|e| format!("unable to deserialize Stacks block {}", e.to_string()))?; spec })) } diff --git a/components/chainhook-sdk/src/chainhooks/types.rs b/components/chainhook-sdk/src/chainhooks/types.rs index 4bc62073a..0a1c179a5 100644 --- a/components/chainhook-sdk/src/chainhooks/types.rs +++ b/components/chainhook-sdk/src/chainhooks/types.rs @@ -204,7 +204,7 @@ impl ChainhookSpecification { pub fn deserialize_specification(spec: &str) -> Result { let spec: ChainhookSpecification = serde_json::from_str(spec) - .map_err(|e| format!("unable to deserialize Stacks chainhook {}", e.to_string()))?; + .map_err(|e| format!("unable to deserialize predicate {}", e.to_string()))?; Ok(spec) } @@ -295,7 +295,7 @@ impl ChainhookFullSpecification { _key: &str, ) -> Result { let spec: ChainhookFullSpecification = serde_json::from_str(spec) - .map_err(|e| format!("unable to deserialize Stacks chainhook {}", e.to_string()))?; + .map_err(|e| format!("unable to deserialize predicate {}", e.to_string()))?; Ok(spec) } } From 3c94ff7ffbdaa7ca3c202a246b79c84db6f02b36 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 2 Aug 2023 16:21:10 +0200 Subject: [PATCH 07/12] feat: additional ordhook adjustments (v0.7.11) --- Cargo.lock | 2 +- components/chainhook-cli/src/service/mod.rs | 1 + components/chainhook-cli/src/storage/mod.rs | 6 +- components/chainhook-sdk/Cargo.toml | 2 +- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 2 - components/chainhook-sdk/src/observer/mod.rs | 58 +++++++++++++++++++ .../chainhook-sdk/src/observer/tests/mod.rs | 4 ++ components/chainhook-types-rs/src/rosetta.rs | 1 + 8 files changed, 70 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68963e070..ddd9969d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,7 +457,7 @@ dependencies = [ [[package]] name = "chainhook-sdk" -version = "0.7.10" +version = "0.7.11" dependencies = [ "base58 0.2.0", "base64", diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index fffbb83db..2016c5179 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -157,6 +157,7 @@ impl Service { observer_command_tx, observer_command_rx, Some(observer_event_tx), + None, self.ctx.clone(), ); diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index 2868301b2..496d701c2 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -193,8 +193,10 @@ pub fn get_stacks_block_at_block_height( }) { Ok(Some(entry)) => { return Ok(Some({ - let spec: StacksBlockData = serde_json::from_slice(&entry[..]) - .map_err(|e| format!("unable to deserialize Stacks block {}", e.to_string()))?; + let spec: StacksBlockData = + serde_json::from_slice(&entry[..]).map_err(|e| { + format!("unable to deserialize Stacks block {}", e.to_string()) + })?; spec })) } diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index 19e222572..9e6ab40bb 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook-sdk" -version = "0.7.10" +version = "0.7.11" description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin" license = "GPL-3.0" edition = "2021" diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index 88a5ce9d4..a6014f4a8 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -440,8 +440,6 @@ pub fn standardize_bitcoin_block( for mut tx in block.tx.into_iter() { let txid = tx.txid.to_string(); - ctx.try_log(|logger| slog::debug!(logger, "Standardizing Bitcoin transaction {txid}")); - let mut stacks_operations = vec![]; if let Some(op) = try_parse_stacks_operation( block_height, diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 750e7cbab..82ced692b 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -397,6 +397,7 @@ pub fn start_event_observer( observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, + block_pre_processor: Option<(Sender, Receiver>)>, ctx: Context, ) -> Result<(), Box> { match config.bitcoin_block_signaling { @@ -413,6 +414,7 @@ pub fn start_event_observer( observer_commands_tx_moved, observer_commands_rx, observer_events_tx, + block_pre_processor, context_cloned, ); let _ = hiro_system_kit::nestable_block_on(future); @@ -429,6 +431,7 @@ pub fn start_event_observer( observer_commands_tx_moved, observer_commands_rx, observer_events_tx, + block_pre_processor, context_cloned, ); let _ = hiro_system_kit::nestable_block_on(future); @@ -455,6 +458,7 @@ pub async fn start_bitcoin_event_observer( observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, + block_pre_processor: Option<(Sender, Receiver>)>, ctx: Context, ) -> Result<(), Box> { let chainhook_store = ChainhookStore::new(); @@ -481,6 +485,7 @@ pub async fn start_bitcoin_event_observer( observer_events_tx, None, observer_metrics_rw_lock.clone(), + block_pre_processor, ctx, ) .await @@ -491,6 +496,7 @@ pub async fn start_stacks_event_observer( observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, + block_pre_processor: Option<(Sender, Receiver>)>, ctx: Context, ) -> Result<(), Box> { let indexer_config = IndexerConfig { @@ -607,6 +613,7 @@ pub async fn start_stacks_event_observer( observer_events_tx, ingestion_shutdown, observer_metrics_rw_lock.clone(), + block_pre_processor, ctx, ) .await @@ -786,6 +793,11 @@ pub fn gather_proofs<'a>( } } +pub enum HandleBlock { + ApplyBlocks(Vec), + UndoBlocks(Vec), +} + pub async fn start_observer_commands_handler( config: EventObserverConfig, mut chainhook_store: ChainhookStore, @@ -793,6 +805,7 @@ pub async fn start_observer_commands_handler( observer_events_tx: Option>, ingestion_shutdown: Option, observer_metrics: Arc>, + block_pre_processor: Option<(Sender, Receiver>)>, ctx: Context, ) -> Result<(), Box> { let mut chainhooks_occurrences_tracker: HashMap = HashMap::new(); @@ -909,6 +922,8 @@ pub async fn start_observer_commands_handler( } } + new_blocks = handle_blocks_pre_processing(&block_pre_processor, new_blocks, true, &ctx); + for header in data.confirmed_headers.iter() { match bitcoin_block_store.remove(&header.block_identifier) { Some(block) => { @@ -969,6 +984,8 @@ pub async fn start_observer_commands_handler( } } + blocks_to_rollback = handle_blocks_pre_processing(&block_pre_processor, blocks_to_rollback, false, &ctx); + for header in data.headers_to_apply.iter() { match bitcoin_block_store.get_mut(&header.block_identifier) { Some(block) => { @@ -986,6 +1003,8 @@ pub async fn start_observer_commands_handler( } } + blocks_to_apply = handle_blocks_pre_processing(&block_pre_processor, blocks_to_apply, true, &ctx); + for header in data.confirmed_headers.iter() { match bitcoin_block_store.remove(&header.block_identifier) { Some(block) => { @@ -1489,5 +1508,44 @@ pub async fn start_observer_commands_handler( } Ok(()) } + +fn handle_blocks_pre_processing( + block_pre_processor: &Option<(Sender, Receiver>)>, + mut blocks: Vec, + apply: bool, + ctx: &Context, +) -> Vec { + if let Some(ref processor) = block_pre_processor { + ctx.try_log(|logger| { + slog::error!( + logger, + "Sending blocks to pre-processor", + ) + }); + let _ = processor + .0 + .send(match apply { + true => HandleBlock::ApplyBlocks(blocks.clone()), + false => HandleBlock::UndoBlocks(blocks.clone()), + }); + ctx.try_log(|logger| { + slog::error!( + logger, + "Waiting for blocks from pre-processor", + ) + }); + if let Ok(updated_blocks) = processor.1.recv() { + blocks = updated_blocks; + } + ctx.try_log(|logger| { + slog::error!( + logger, + "Blocks received from pre-processor", + ) + }); + } + blocks +} + #[cfg(test)] pub mod tests; diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs index 342253008..8d4b3a7f1 100644 --- a/components/chainhook-sdk/src/observer/tests/mod.rs +++ b/components/chainhook-sdk/src/observer/tests/mod.rs @@ -218,6 +218,7 @@ fn test_stacks_chainhook_register_deregister() { Some(observer_events_tx), None, observer_metrics_rw_lock_moved, + None, Context::empty(), )); }); @@ -492,6 +493,7 @@ fn test_stacks_chainhook_auto_deregister() { Some(observer_events_tx), None, observer_metrics_rw_lock_moved, + None, Context::empty(), )); }); @@ -694,6 +696,7 @@ fn test_bitcoin_chainhook_register_deregister() { Some(observer_events_tx), None, observer_metrics_rw_lock_moved, + None, Context::empty(), )); }); @@ -966,6 +969,7 @@ fn test_bitcoin_chainhook_auto_deregister() { Some(observer_events_tx), None, observer_metrics_rw_lock_moved, + None, Context::empty(), )); }); diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index 12fba636a..a90fe92f7 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -324,6 +324,7 @@ pub enum OrdinalInscriptionCurseType { Tag(u8), Batch, P2wsh, + Reinscription, } #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] From 8ed9e8362c6d73bae87714fbb3a590345fe16266 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 2 Aug 2023 16:26:19 +0200 Subject: [PATCH 08/12] feat: additional ordhook adjustments (v0.7.12) --- Cargo.lock | 6 +++--- components/chainhook-sdk/Cargo.toml | 7 +++---- components/chainhook-types-rs/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ddd9969d5..f9f261157 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,13 +457,13 @@ dependencies = [ [[package]] name = "chainhook-sdk" -version = "0.7.11" +version = "0.7.12" dependencies = [ "base58 0.2.0", "base64", "bitcoincore-rpc", "bitcoincore-rpc-json", - "chainhook-types 1.0.8", + "chainhook-types 1.0.9", "clarinet-utils", "crossbeam-channel 0.5.8", "dashmap 5.4.0", @@ -504,7 +504,7 @@ dependencies = [ [[package]] name = "chainhook-types" -version = "1.0.8" +version = "1.0.9" dependencies = [ "hex", "schemars 0.8.12", diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index 9e6ab40bb..25f8b6afd 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook-sdk" -version = "0.7.11" +version = "0.7.12" description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin" license = "GPL-3.0" edition = "2021" @@ -12,14 +12,13 @@ serde = {version = "1", features = ["rc"]} serde_json = { version = "1", features = ["arbitrary_precision"] } serde-hex = "0.1.0" serde_derive = "1" -stacks-rpc-client = "=1.0.7" +stacks-rpc-client = "=1.0.8" clarinet-utils = "1.0.0" hiro-system-kit = "0.1.0" # stacks-rpc-client = { version = "1", path = "../../../clarinet/components/stacks-rpc-client" } # clarinet-utils = { version = "1", path = "../../../clarinet/components/clarinet-utils" } -# clarity-repl = { version = "1", path = "../../../clarinet/components/clarity-repl" } # hiro-system-kit = { version = "0.1.0", path = "../../../clarinet/components/hiro-system-kit" } -chainhook-types = { version = "1.0.8", path = "../chainhook-types-rs" } +chainhook-types = { version = "1.0.9", path = "../chainhook-types-rs" } rocket = { version = "=0.5.0-rc.3", features = ["json"] } bitcoincore-rpc = "0.16.0" bitcoincore-rpc-json = "0.16.0" diff --git a/components/chainhook-types-rs/Cargo.toml b/components/chainhook-types-rs/Cargo.toml index 2826bc59c..409f36a1a 100644 --- a/components/chainhook-types-rs/Cargo.toml +++ b/components/chainhook-types-rs/Cargo.toml @@ -2,7 +2,7 @@ name = "chainhook-types" description = "Bitcoin and Stacks data schemas, based on the Rosetta specification" license = "MIT" -version = "1.0.8" +version = "1.0.9" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 6bcfb9936273727ff462762b9cd25b388ff18f42 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 2 Aug 2023 23:38:50 +0200 Subject: [PATCH 09/12] fix: log level --- components/chainhook-sdk/Cargo.toml | 2 +- components/chainhook-sdk/src/observer/mod.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index 25f8b6afd..954c2e80a 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook-sdk" -version = "0.7.12" +version = "0.8.0" description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin" license = "GPL-3.0" edition = "2021" diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 82ced692b..139a7fa11 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -1517,7 +1517,7 @@ fn handle_blocks_pre_processing( ) -> Vec { if let Some(ref processor) = block_pre_processor { ctx.try_log(|logger| { - slog::error!( + slog::info!( logger, "Sending blocks to pre-processor", ) @@ -1529,7 +1529,7 @@ fn handle_blocks_pre_processing( false => HandleBlock::UndoBlocks(blocks.clone()), }); ctx.try_log(|logger| { - slog::error!( + slog::info!( logger, "Waiting for blocks from pre-processor", ) @@ -1538,7 +1538,7 @@ fn handle_blocks_pre_processing( blocks = updated_blocks; } ctx.try_log(|logger| { - slog::error!( + slog::info!( logger, "Blocks received from pre-processor", ) From 7b302450b9fbe75d80d64a9d84139a24150c5b54 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 3 Aug 2023 08:05:38 +0200 Subject: [PATCH 10/12] chore: cargo fmt --- components/chainhook-sdk/src/observer/mod.rs | 70 +++++++++++--------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 139a7fa11..e40b47c35 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -922,7 +922,12 @@ pub async fn start_observer_commands_handler( } } - new_blocks = handle_blocks_pre_processing(&block_pre_processor, new_blocks, true, &ctx); + new_blocks = handle_blocks_pre_processing( + &block_pre_processor, + new_blocks, + true, + &ctx, + ); for header in data.confirmed_headers.iter() { match bitcoin_block_store.remove(&header.block_identifier) { @@ -984,7 +989,12 @@ pub async fn start_observer_commands_handler( } } - blocks_to_rollback = handle_blocks_pre_processing(&block_pre_processor, blocks_to_rollback, false, &ctx); + blocks_to_rollback = handle_blocks_pre_processing( + &block_pre_processor, + blocks_to_rollback, + false, + &ctx, + ); for header in data.headers_to_apply.iter() { match bitcoin_block_store.get_mut(&header.block_identifier) { @@ -1003,7 +1013,12 @@ pub async fn start_observer_commands_handler( } } - blocks_to_apply = handle_blocks_pre_processing(&block_pre_processor, blocks_to_apply, true, &ctx); + blocks_to_apply = handle_blocks_pre_processing( + &block_pre_processor, + blocks_to_apply, + true, + &ctx, + ); for header in data.confirmed_headers.iter() { match bitcoin_block_store.remove(&header.block_identifier) { @@ -1511,39 +1526,34 @@ pub async fn start_observer_commands_handler( fn handle_blocks_pre_processing( block_pre_processor: &Option<(Sender, Receiver>)>, - mut blocks: Vec, + blocks: Vec, apply: bool, ctx: &Context, ) -> Vec { if let Some(ref processor) = block_pre_processor { - ctx.try_log(|logger| { - slog::info!( - logger, - "Sending blocks to pre-processor", - ) - }); - let _ = processor - .0 - .send(match apply { - true => HandleBlock::ApplyBlocks(blocks.clone()), - false => HandleBlock::UndoBlocks(blocks.clone()), - }); - ctx.try_log(|logger| { - slog::info!( - logger, - "Waiting for blocks from pre-processor", - ) + ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",)); + let _ = processor.0.send(match apply { + true => HandleBlock::ApplyBlocks(blocks.clone()), + false => HandleBlock::UndoBlocks(blocks.clone()), }); - if let Ok(updated_blocks) = processor.1.recv() { - blocks = updated_blocks; + ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",)); + match processor.1.recv() { + Ok(updated_blocks) => { + ctx.try_log(|logger| slog::info!(logger, "Blocks received from pre-processor",)); + return updated_blocks; + } + Err(e) => { + ctx.try_log(|logger| { + slog::error!( + logger, + "Unable to receive block from pre-processor {}", + e.to_string() + ) + }); + return blocks; + } } - ctx.try_log(|logger| { - slog::info!( - logger, - "Blocks received from pre-processor", - ) - }); - } + } blocks } From 515488598dc3f98d12c9853ffb8bd56490c17f60 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 3 Aug 2023 11:49:33 +0200 Subject: [PATCH 11/12] chore: bump version --- Cargo.lock | 12 ++++++------ components/chainhook-cli/Cargo.toml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9f261157..9338a064d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,7 +416,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chainhook" -version = "0.17.0" +version = "0.20.0" dependencies = [ "ansi_term", "atty", @@ -457,7 +457,7 @@ dependencies = [ [[package]] name = "chainhook-sdk" -version = "0.7.12" +version = "0.8.0" dependencies = [ "base58 0.2.0", "base64", @@ -642,9 +642,9 @@ dependencies = [ [[package]] name = "clarity-repl" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2d096ff86def33801c5ad13dc222befa42118326a80cfc1cf34d9ee1abc70b" +checksum = "6fbc2179431d853889075f92b984a56c3c988b944ca6682ff840b2792bb155bf" dependencies = [ "ansi_term", "atty", @@ -3929,9 +3929,9 @@ dependencies = [ [[package]] name = "stacks-rpc-client" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "417254594e89666f68c76267a620d3751ae6604e39c97f3abb57a845390bc1a0" +checksum = "4903f6164688594cb80856abc27d7a76fa7b6ac84c6fc70a45e414d097c191ab" dependencies = [ "clarity-repl", "hmac 0.12.1", diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml index 93f876765..e56ef044f 100644 --- a/components/chainhook-cli/Cargo.toml +++ b/components/chainhook-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook" -version = "0.17.0" +version = "0.20.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -17,7 +17,7 @@ hex = "0.4.3" rand = "0.8.5" # tikv-client = { git = "https://github.com/tikv/client-rust.git", rev = "8f54e6114227718e256027df2577bbacdf425f86" } # raft-proto = { git = "https://github.com/tikv/raft-rs", rev="f73766712a538c2f6eb135b455297ad6c03fc58d", version = "0.7.0"} -chainhook-sdk = { version = "0.7.5", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" } +chainhook-sdk = { version = "0.8.0", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" } clarinet-files = "1.0.1" hiro-system-kit = "0.1.0" # clarinet-files = { path = "../../../clarinet/components/clarinet-files" } @@ -48,7 +48,7 @@ features = ["lz4", "snappy"] [dev-dependencies] criterion = "0.3" redis = "0.21.5" -clarity-repl = "=1.7.0" +clarity-repl = "=1.7.1" clarity-vm = "=2.1.1" hex = "0.4.3" test-case = "3.1.0" From 8ed4b44ffc87be496efbe3de29c0ea15f1be17ec Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 3 Aug 2023 11:49:55 +0200 Subject: [PATCH 12/12] test: introduce vector 52 --- .../src/indexer/stacks/blocks_pool.rs | 10 +++++++++ .../chainhook-sdk/src/indexer/stacks/tests.rs | 5 +++++ .../indexer/tests/helpers/stacks_shapes.rs | 22 +++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs index 1879d6d4a..33c9c91ea 100644 --- a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs @@ -330,6 +330,16 @@ impl StacksBlockPool { microblocks: Vec, ctx: &Context, ) -> Result, String> { + if self.block_store.is_empty() { + ctx.try_log(|logger| { + slog::info!( + logger, + "Ignoring microblock trail, fork tracking will start with the next anchor block" + ) + }); + return Ok(None); + } + ctx.try_log(|logger| { slog::info!(logger, "Start processing {} microblocks", microblocks.len()) }); diff --git a/components/chainhook-sdk/src/indexer/stacks/tests.rs b/components/chainhook-sdk/src/indexer/stacks/tests.rs index 1df4b685a..84665a1be 100644 --- a/components/chainhook-sdk/src/indexer/stacks/tests.rs +++ b/components/chainhook-sdk/src/indexer/stacks/tests.rs @@ -254,3 +254,8 @@ fn test_stacks_vector_050() { fn test_stacks_vector_051() { process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_051()); } + +#[test] +fn test_stacks_vector_052() { + process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_052()); +} diff --git a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs index ca07ccf31..c0f79f9d9 100644 --- a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs +++ b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs @@ -3717,3 +3717,25 @@ pub fn get_vector_051() -> Vec<(BlockEvent, StacksChainEventExpectation)> { ), ] } + +/// Vector 052: Generate the following blocks +/// +/// [a1](1) - [b1](2) - B1(3) +/// +/// +pub fn get_vector_052() -> Vec<(BlockEvent, StacksChainEventExpectation)> { + vec![ + ( + microblocks::a1(stacks_blocks::A1(None), None), + expect_no_chain_update(), + ), + ( + microblocks::b1(stacks_blocks::A1(None), None), + expect_no_chain_update(), + ), + ( + stacks_blocks::B1(None), + expect_chain_updated_with_block(stacks_blocks::B1(None), vec![]), + ), + ] +}