From c0f0c048caf7ff75a9cad61b8acebc10d6fa6a1d Mon Sep 17 00:00:00 2001 From: jossduff Date: Wed, 29 May 2024 10:35:00 -0400 Subject: [PATCH] added tx-status to query selections and schema/format for receipts, inputs, outputs --- hyperfuel-client/Cargo.toml | 8 ++--- hyperfuel-client/src/filter.rs | 45 +++++++++++++++++++++--- hyperfuel-client/src/from_arrow.rs | 16 +++++++++ hyperfuel-client/src/lib.rs | 55 +++++++----------------------- hyperfuel-format/Cargo.toml | 2 +- hyperfuel-format/src/types/mod.rs | 6 ++++ hyperfuel-net-types/Cargo.toml | 4 +-- hyperfuel-net-types/src/lib.rs | 6 ++++ hyperfuel-schema/Cargo.toml | 2 +- hyperfuel-schema/src/lib.rs | 5 ++- 10 files changed, 94 insertions(+), 55 deletions(-) diff --git a/hyperfuel-client/Cargo.toml b/hyperfuel-client/Cargo.toml index 945e3d4..f87cc75 100644 --- a/hyperfuel-client/Cargo.toml +++ b/hyperfuel-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hyperfuel-client" -version = "1.0.0" +version = "2.0.0" edition = "2021" description = "client library for Envio's HyperSync support of the Fuel Network" license = "MIT" @@ -35,9 +35,9 @@ alloy-dyn-abi = "0.5.0" alloy-json-abi = "0.5.0" xxhash-rust = { version = "0.8", features = ["xxh3"] } -hyperfuel-net-types = { package = "hyperfuel-net-types", path = "../hyperfuel-net-types", version = "1.0.0" } -hyperfuel-format = { package = "hyperfuel-format", path = "../hyperfuel-format", version = "1.0.0" } -hyperfuel-schema = { package = "hyperfuel-schema", path = "../hyperfuel-schema", version = "1.0.0" } +hyperfuel-net-types = { package = "hyperfuel-net-types", path = "../hyperfuel-net-types", version = "2.0.0" } +hyperfuel-format = { package = "hyperfuel-format", path = "../hyperfuel-format", version = "2.0.0" } +hyperfuel-schema = { package = "hyperfuel-schema", path = "../hyperfuel-schema", version = "2.0.0" } [dependencies.reqwest] version = "0.11" diff --git a/hyperfuel-client/src/filter.rs b/hyperfuel-client/src/filter.rs index 36ff090..cd7d043 100644 --- a/hyperfuel-client/src/filter.rs +++ b/hyperfuel-client/src/filter.rs @@ -100,6 +100,7 @@ fn receipt_selections_to_filter( let rb = batch.column::("rb").ok(); let rc = batch.column::("rc").ok(); let rd = batch.column::("rd").ok(); + let tx_status = batch.column::("tx_status").ok(); let chunk_len = batch.chunk.len(); let mut filter = unset_bool_array(chunk_len); @@ -117,6 +118,7 @@ fn receipt_selections_to_filter( rb, rc, rd, + tx_status, selection, chunk_len, ); @@ -139,6 +141,7 @@ fn receipt_selection_to_filter( rb: Option<&UInt64Array>, rc: Option<&UInt64Array>, rd: Option<&UInt64Array>, + tx_status: Option<&UInt8Array>, selection: &ReceiptSelection, chunk_len: usize, ) -> BooleanArray { @@ -179,7 +182,8 @@ fn receipt_selection_to_filter( } if !selection.contract_id.is_empty() && contract_id.is_some() { - let set = selection.contract_id.iter().map(|t| t.as_slice()).collect(); + let set: StdHashSet<&[u8], Xxh3Builder> = + selection.contract_id.iter().map(|t| t.as_slice()).collect(); filter = compute::boolean::and(&filter, &in_set_binary(contract_id.unwrap(), &set)); } @@ -203,6 +207,13 @@ fn receipt_selection_to_filter( filter = compute::boolean::and(&filter, &in_set_u64(rd.unwrap(), &set)); } + if !selection.tx_status.is_empty() && tx_status.is_some() { + filter = compute::boolean::and( + &filter, + &in_set_u8(tx_status.unwrap(), &selection.tx_status), + ); + } + filter } @@ -257,13 +268,15 @@ fn input_selections_to_filter( let sender = batch.column::>("sender").ok(); let recipient = batch.column::>("recipient").ok(); let input_type = batch.column::("input_type").ok(); + let tx_status = batch.column::("tx_status").ok(); let chunk_len = batch.chunk.len(); let mut filter = unset_bool_array(chunk_len); for selection in selections.iter() { let selection = input_selection_to_filter( - owner, asset_id, contract, sender, recipient, input_type, selection, chunk_len, + owner, asset_id, contract, sender, recipient, input_type, tx_status, selection, + chunk_len, ); filter = compute::boolean::or(&filter, &selection); } @@ -279,6 +292,7 @@ fn input_selection_to_filter( sender: Option<&BinaryArray>, recipient: Option<&BinaryArray>, input_type: Option<&UInt8Array>, + tx_status: Option<&UInt8Array>, selection: &InputSelection, chunk_len: usize, ) -> BooleanArray { @@ -314,6 +328,13 @@ fn input_selection_to_filter( filter = compute::boolean::and(&filter, &in_set_u8(input_type.unwrap(), &set)); } + if !selection.tx_status.is_empty() && tx_status.is_some() { + filter = compute::boolean::and( + &filter, + &in_set_u8(tx_status.unwrap(), &selection.tx_status), + ); + } + filter } @@ -325,13 +346,21 @@ fn output_selections_to_filter( let asset_id = batch.column::>("asset_id").ok(); let contract = batch.column::>("contract").ok(); let output_type = batch.column::("output_type").ok(); + let tx_status = batch.column::("tx_status").ok(); let chunk_len = batch.chunk.len(); let mut filter = unset_bool_array(chunk_len); for selection in selections.iter() { - let selection = - output_selection_to_filter(to, asset_id, contract, output_type, selection, chunk_len); + let selection = output_selection_to_filter( + to, + asset_id, + contract, + output_type, + tx_status, + selection, + chunk_len, + ); filter = compute::boolean::or(&filter, &selection); } @@ -343,6 +372,7 @@ fn output_selection_to_filter( asset_id: Option<&BinaryArray>, contract: Option<&BinaryArray>, output_type: Option<&UInt8Array>, + tx_status: Option<&UInt8Array>, selection: &OutputSelection, chunk_len: usize, ) -> BooleanArray { @@ -368,5 +398,12 @@ fn output_selection_to_filter( filter = compute::boolean::and(&filter, &in_set_u8(output_type.unwrap(), &set)); } + if !selection.tx_status.is_empty() && tx_status.is_some() { + filter = compute::boolean::and( + &filter, + &in_set_u8(tx_status.unwrap(), &selection.tx_status), + ); + } + filter } diff --git a/hyperfuel-client/src/from_arrow.rs b/hyperfuel-client/src/from_arrow.rs index 059fcb8..eba5eb6 100644 --- a/hyperfuel-client/src/from_arrow.rs +++ b/hyperfuel-client/src/from_arrow.rs @@ -448,6 +448,12 @@ impl FromArrow for Receipt { } } + if let Ok(col) = batch.column::("tx_status") { + for (target, val) in out.iter_mut().zip(col.values_iter()) { + target.tx_status = TransactionStatus::from_u8(*val).unwrap(); + } + } + if let Ok(col) = batch.column::("block_height") { for (target, &val) in out.iter_mut().zip(col.values_iter()) { target.block_height = val.into(); @@ -611,6 +617,11 @@ impl FromArrow for Input { target.tx_id = val.try_into().unwrap(); } } + if let Ok(col) = batch.column::("tx_status") { + for (target, val) in out.iter_mut().zip(col.values_iter()) { + target.tx_status = TransactionStatus::from_u8(*val).unwrap(); + } + } if let Ok(col) = batch.column::>("utxo_id") { for (target, val) in out.iter_mut().zip(col.iter()) { target.utxo_id = val.map(|v| v.try_into().unwrap()); @@ -720,6 +731,11 @@ impl FromArrow for Output { target.tx_id = val.try_into().unwrap(); } } + if let Ok(col) = batch.column::("tx_status") { + for (target, val) in out.iter_mut().zip(col.values_iter()) { + target.tx_status = TransactionStatus::from_u8(*val).unwrap(); + } + } if let Ok(col) = batch.column::>("to") { for (target, val) in out.iter_mut().zip(col.iter()) { target.to = val.map(|v| v.try_into().unwrap()); diff --git a/hyperfuel-client/src/lib.rs b/hyperfuel-client/src/lib.rs index 3ddf02f..f2b674f 100644 --- a/hyperfuel-client/src/lib.rs +++ b/hyperfuel-client/src/lib.rs @@ -1,14 +1,11 @@ -use std::{ - collections::{BTreeSet, HashSet}, - time::Duration, -}; +use std::{collections::BTreeSet, time::Duration}; use anyhow::{anyhow, Context, Result}; use arrow2::{array::Array, chunk::Chunk}; use filter::filter_out_unselected_data; -use from_arrow::{receipts_from_arrow_data, typed_data_from_arrow_data, FromArrow}; -use hyperfuel_format::{Hash, Receipt, Transaction, TransactionStatus}; +use from_arrow::{receipts_from_arrow_data, typed_data_from_arrow_data}; +use hyperfuel_format::{Hash, Receipt}; use hyperfuel_net_types::{ hyperfuel_net_types_capnp, ArchiveHeight, FieldSelection, Query, ReceiptSelection, }; @@ -179,13 +176,10 @@ impl Client { from_block: u64, to_block: Option, ) -> Result { - let mut transaction_field_selection = BTreeSet::new(); - transaction_field_selection.insert("id".to_owned()); - transaction_field_selection.insert("status".to_owned()); - let mut receipt_field_selection = BTreeSet::new(); receipt_field_selection.insert("block_height".to_owned()); receipt_field_selection.insert("tx_id".to_owned()); + receipt_field_selection.insert("tx_status".to_owned()); receipt_field_selection.insert("receipt_index".to_owned()); receipt_field_selection.insert("receipt_type".to_owned()); receipt_field_selection.insert("contract_id".to_owned()); @@ -206,20 +200,13 @@ impl Client { let query = Query { from_block, to_block, - receipts: vec![ - ReceiptSelection { - root_contract_id: emitting_contracts.clone(), - receipt_type: vec![5, 6], - ..Default::default() - }, - ReceiptSelection { - contract_id: emitting_contracts, - receipt_type: vec![5, 6], - ..Default::default() - }, - ], + receipts: vec![ReceiptSelection { + root_contract_id: emitting_contracts.clone(), + receipt_type: vec![5, 6], + tx_status: vec![1], + ..Default::default() + }], field_selection: FieldSelection { - transaction: transaction_field_selection, receipt: receipt_field_selection, ..Default::default() }, @@ -239,32 +226,16 @@ impl Client { sort_receipts(&mut typed_receipts); - let mut failed_txns = HashSet::new(); - for batch in filtered_data.transactions.iter() { - let data = Transaction::from_arrow(batch).context("transaction from arrow")?; - for transaction in data { - if transaction.status == TransactionStatus::Failure { - failed_txns.insert(transaction.id); - } - } - } - - let successful_logs: Vec = typed_receipts + let logs: Vec = typed_receipts .into_iter() - .filter_map(|receipt| { - if !failed_txns.contains(&receipt.tx_id) { - Some(receipt.into()) - } else { - None - } - }) + .map(|receipt| receipt.into()) .collect(); Ok(LogResponse { archive_height: res.archive_height, next_block: res.next_block, total_execution_time: res.total_execution_time, - data: successful_logs, + data: logs, }) } diff --git a/hyperfuel-format/Cargo.toml b/hyperfuel-format/Cargo.toml index 1f60d5e..bc4b8fc 100644 --- a/hyperfuel-format/Cargo.toml +++ b/hyperfuel-format/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hyperfuel-format" -version = "1.0.0" +version = "2.0.0" edition = "2021" description = "fuel network format library" license = "MIT" diff --git a/hyperfuel-format/src/types/mod.rs b/hyperfuel-format/src/types/mod.rs index 5d935d2..ff20f82 100644 --- a/hyperfuel-format/src/types/mod.rs +++ b/hyperfuel-format/src/types/mod.rs @@ -153,6 +153,8 @@ pub struct Receipt { pub root_contract_id: Option, /// transaction that this receipt originated from pub tx_id: Hash, + /// The status type of the transaction this receipt originated from + pub tx_status: TransactionStatus, /// block that the receipt originated in pub block_height: UInt, /// The value of the program counter register $pc, which is the memory address of the current instruction. @@ -217,6 +219,8 @@ pub struct Receipt { pub struct Input { /// transaction that this input originated from pub tx_id: Hash, + /// The status type of the transaction this input originated from + pub tx_status: TransactionStatus, /// block that the input originated in pub block_height: UInt, /// InputCoin, InputContract, or InputMessage @@ -263,6 +267,8 @@ pub struct Input { pub struct Output { /// transaction that this out originated from pub tx_id: Hash, + /// The status type of the transaction this output originated from + pub tx_status: TransactionStatus, /// block that the output originated in pub block_height: UInt, /// CoinOutput, ContractOutput, ChangeOutput, VariableOutput, or ContractCreated diff --git a/hyperfuel-net-types/Cargo.toml b/hyperfuel-net-types/Cargo.toml index 50d0769..addde79 100644 --- a/hyperfuel-net-types/Cargo.toml +++ b/hyperfuel-net-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hyperfuel-net-types" -version = "1.0.0" +version = "2.0.0" edition = "2021" description = "hyperfuel types for transport over network" license = "MIT" @@ -10,7 +10,7 @@ capnp = "0.18" serde = { version = "1", features = ["derive"] } arrayvec = { version = "0.7", features = ["serde"] } -hyperfuel-format = { package = "hyperfuel-format", path = "../hyperfuel-format", version = "1.0.0" } +hyperfuel-format = { package = "hyperfuel-format", path = "../hyperfuel-format", version = "2.0.0" } [build-dependencies] capnpc = "0.18" diff --git a/hyperfuel-net-types/src/lib.rs b/hyperfuel-net-types/src/lib.rs index bd1cbc9..a7e9572 100644 --- a/hyperfuel-net-types/src/lib.rs +++ b/hyperfuel-net-types/src/lib.rs @@ -33,6 +33,8 @@ pub struct ReceiptSelection { pub rc: Vec, #[serde(default)] pub rd: Vec, + #[serde(default)] + pub tx_status: Vec, } #[derive(Default, Serialize, Deserialize, Clone, Debug)] @@ -49,6 +51,8 @@ pub struct InputSelection { pub recipient: Vec, #[serde(default)] pub input_type: Vec, + #[serde(default)] + pub tx_status: Vec, } #[derive(Default, Serialize, Deserialize, Clone, Debug)] @@ -61,6 +65,8 @@ pub struct OutputSelection { pub contract: Vec, #[serde(default)] pub output_type: Vec, + #[serde(default)] + pub tx_status: Vec, } #[derive(Default, Serialize, Deserialize, Clone, Debug)] diff --git a/hyperfuel-schema/Cargo.toml b/hyperfuel-schema/Cargo.toml index fd5fadf..475ba54 100644 --- a/hyperfuel-schema/Cargo.toml +++ b/hyperfuel-schema/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hyperfuel-schema" -version = "1.0.0" +version = "2.0.0" edition = "2021" description = "schema utilities for hyperfuel" license = "MIT" diff --git a/hyperfuel-schema/src/lib.rs b/hyperfuel-schema/src/lib.rs index 6d6bda6..8a42bf9 100644 --- a/hyperfuel-schema/src/lib.rs +++ b/hyperfuel-schema/src/lib.rs @@ -69,7 +69,7 @@ pub fn transaction() -> SchemaRef { // vec Field::new("witnesses", DataType::Binary, true), Field::new("receipts_root", DataType::Binary, true), - Field::new("status", DataType::UInt8, true), + Field::new("status", DataType::UInt8, false), Field::new("time", DataType::Int64, false), Field::new("reason", DataType::Utf8, true), Field::new("script", DataType::Binary, true), @@ -109,6 +109,7 @@ pub fn receipt() -> SchemaRef { Field::new("receipt_index", DataType::UInt64, false), Field::new("root_contract_id", DataType::Binary, true), Field::new("tx_id", DataType::Binary, false), + Field::new("tx_status", DataType::UInt8, false), // new Field::new("block_height", DataType::UInt64, false), Field::new("pc", DataType::UInt64, true), Field::new("is", DataType::UInt64, true), @@ -145,6 +146,7 @@ pub fn input() -> SchemaRef { Schema::from(vec![ // for mapping Field::new("tx_id", DataType::Binary, false), + Field::new("tx_status", DataType::UInt8, false), // new Field::new("block_height", DataType::UInt64, false), Field::new("input_type", DataType::UInt8, false), Field::new("utxo_id", DataType::Binary, true), @@ -172,6 +174,7 @@ pub fn output() -> SchemaRef { Schema::from(vec![ // for mapping Field::new("tx_id", DataType::Binary, false), + Field::new("tx_status", DataType::UInt8, false), // new Field::new("block_height", DataType::UInt64, false), Field::new("output_type", DataType::UInt8, false), Field::new("to", DataType::Binary, true),