update blobs

blocks/evm/Makefile

@@ -11,19 +11,19 @@ build:
 
 .PHONY: gui
 gui:
-	substreams gui . -e eth.substreams.pinax.network:443 map_events -s 20500000 -t 20500001 --network mainnet
+	substreams gui . -e eth.substreams.pinax.network:443 map_events -s 21200000 -t 21200001 --network mainnet
 
 .PHONY: protogen
 protogen:
 	substreams protogen
 
 .PHONY: parquet
 parquet:
-	rm -rf out state.yaml && substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events ./out 20500000:20500001 --encoder parquet --file-block-count 1 --development-mode --parquet-default-column-compression snappy
+	rm -rf out state.yaml && substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events ./out 21200000:21200001 --encoder parquet --file-block-count 1 --development-mode --parquet-default-column-compression snappy
 
 .PHONY: s3
 s3:
-	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events s3://pinax/eth/b7b803a5a2560cdf0fbf5e44d68d666977df3e6b?region=us-east-1 20500000: --encoder parquet --development-mode --parquet-default-column-compression snappy --file-block-count 1
+	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events s3://pinax/eth/d091b44a01fb21c3a882e56a49659cda9382cb48?region=us-east-1 21200000: --encoder parquet --development-mode --parquet-default-column-compression snappy --file-block-count 1
 
 .PHONY: hash
 hash:

blocks/evm/README.md

@@ -36,5 +36,5 @@ Kind: map
 Input: source: sf.substreams.v1.Clock
 Input: source: sf.ethereum.type.v2.Block
 Output Type: proto:evm.Events
-Hash: b7b803a5a2560cdf0fbf5e44d68d666977df3e6b
+Hash: d091b44a01fb21c3a882e56a49659cda9382cb48
 ```

blocks/evm/schema.snowflake.sql

@@ -0,0 +1,94 @@
+CREATE OR REPLACE SECURE VIEW v1_transactions AS
+SELECT
+  -- block --
+  TO_TIMESTAMP(block_time / 1000000000) as block_time,
+  block_number,
+  block_hash,
+  block_date,
+
+  -- block roots --
+  transactions_root,
+  receipts_root,
+
+  -- transaction --
+  index,
+  hash,
+  from,
+  to,
+  nonce,
+  status,
+  status_code,
+  success,
+  ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(gas_price_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as gas_price,
+  gas_price_bytes,
+  gas_limit,
+  ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(value_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as value,
+  value_bytes,
+  data,
+  v,
+  r,
+  s,
+  gas_used,
+  type,
+  type_code,
+  ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(max_fee_per_gas_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as max_fee_per_gas,
+  max_fee_per_gas_bytes,
+  ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(max_priority_fee_per_gas_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as max_priority_fee_per_gas,
+  max_priority_fee_per_gas_bytes,
+  begin_ordinal,
+  end_ordinal,
+
+  -- blobs --
+  ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(blob_gas_price_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as blob_gas_price,
+  blob_gas_price_bytes,
+  blob_gas_used,
+  ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(blob_gas_fee_cap_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as blob_gas_fee_cap,
+  blob_gas_fee_cap_bytes,
+  blob_gas,
+
+  -- transaction receipt --
+  cumulative_gas_used,
+  logs_bloom,
+  state_root
+FROM "v1.0.0-transactions";
+
+CREATE OR REPLACE SECURE VIEW v1_traces AS
+SELECT
+    -- block --
+    TO_TIMESTAMP(block_time / 1000000000) as block_time,
+    block_number,
+    block_hash,
+    block_date,
+
+    -- transaction --
+    tx_hash,
+    tx_index,
+    tx_status,
+    tx_status_code,
+    tx_success,
+    "from",
+    "to",
+
+    -- traces --
+    index,
+    parent_index,
+    depth,
+    caller,
+    call_type,
+    call_type_code,
+    address,
+    ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(value_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as value,
+    value_bytes,
+    gas_limit,
+    gas_consumed,
+    return_data,
+    input,
+    suicide,
+    failure_reason,
+    state_reverted,
+    status_reverted,
+    status_failed,
+    executed_code,
+    begin_ordinal,
+    end_ordinal,
+FROM "v1.0.0-traces";

blocks/evm/src/blocks.rs

@@ -1,7 +1,8 @@
 use common::structs::BlockTimestamp;
 use common::utils::optional_bigint_to_string;
-use common::utils::{bytes_to_hex, optional_bigint_to_u64, optional_u64_to_string};
+use common::utils::{bytes_to_hex, optional_bigint_to_u64};
 
+// use substreams::log;
 use substreams_ethereum::pb::eth::v2::Block;
 
 use crate::pb::pinax::evm::v1::Block as BlockHeader;
@@ -20,10 +21,22 @@ pub fn block_detail_to_string(detail_level: i32) -> String {
 pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
     let header = block.header.as_ref().unwrap();
 
-    let total_transactions = block.transaction_traces.len() as u64;
-    let successful_transactions = block.transaction_traces.iter().filter(|t| t.status == 1).count() as u64;
+    // counters
+    let total_transactions = block.transaction_traces.len() as u32;
+    let successful_transactions = block.transaction_traces.iter().filter(|t| t.status == 1).count() as u32;
     let failed_transactions = total_transactions - successful_transactions;
-    let total_withdrawals = block.balance_changes.iter().filter(|t| t.reason == 16).count() as u64;
+    let total_withdrawals = block.balance_changes.iter().filter(|t| t.reason == 16).count() as u32;
+    let blob_transactions = block.transaction_traces.iter().filter(|t| t.r#type == 3).count() as u32;
+
+    // blob price
+    let blob_gas_price = block.transaction_traces.iter().find_map(|t| t.receipt.as_ref().and_then(|r| r.blob_gas_price.clone()));
+    // let blob_gas_used = block.transaction_traces.iter().filter_map(|t| t.receipt.as_ref()?.blob_gas_used);
+    // let blob_gas_used_sum: u64 = blob_gas_used.sum();
+    // let blob_gas_used: u64 = block.transaction_traces.iter().filter_map(|t| t.receipt.as_ref()?.blob_gas_used).sum();
+    // let blob_gas: u64 = block.transaction_traces.iter().filter_map(|t| t.blob_gas).sum();
+    // let blob_gas: u64 = block.transaction_traces.iter().filter_map(|t| t.blob_gas_fee_cap).sum();
+
+    // log::debug!("blob_gas_used: {}, blob_gas: {}", blob_gas_used, blob_gas);
 
     BlockHeader {
         // clock
@@ -32,16 +45,18 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
         date: timestamp.date.clone(),
         hash: bytes_to_hex(&block.hash),
 
-        // header
-        parent_hash: bytes_to_hex(&header.parent_hash),
-        nonce: header.nonce,
+        // roots
         ommers_hash: bytes_to_hex(&header.uncle_hash),
         logs_bloom: bytes_to_hex(&header.logs_bloom),
         transactions_root: bytes_to_hex(&header.transactions_root),
         state_root: bytes_to_hex(&header.state_root),
         receipts_root: bytes_to_hex(&header.receipt_root),
         withdrawals_root: bytes_to_hex(&header.withdrawals_root),
         parent_beacon_root: bytes_to_hex(&header.parent_beacon_root),
+
+        // header
+        parent_hash: bytes_to_hex(&header.parent_hash),
+        nonce: header.nonce,
         miner: bytes_to_hex(&header.coinbase),
         difficulty: optional_bigint_to_u64(&header.difficulty),
         total_difficulty_bytes: header.total_difficulty.clone().unwrap_or_default().bytes,
@@ -51,16 +66,22 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
         gas_limit: header.gas_limit,
         gas_used: header.gas_used,
         base_fee_per_gas: optional_bigint_to_string(&header.base_fee_per_gas, ""),
-        blob_gas_used: optional_u64_to_string(&header.blob_gas_used, ""),
-        excess_blob_gas: optional_u64_to_string(&header.excess_blob_gas, ""),
+
+        // blobs
+        blob_gas_used: header.blob_gas_used(),
+        excess_blob_gas: header.excess_blob_gas(),
+        blob_gas_price_bytes: blob_gas_price.unwrap_or_default().bytes,
 
         // counters
         size: block.size,
-        total_transactions: block.transaction_traces.len() as u64,
+        total_transactions: block.transaction_traces.len() as u32,
         successful_transactions,
         failed_transactions,
-        total_balance_changes: block.balance_changes.len() as u64,
+        total_balance_changes: block.balance_changes.len() as u32,
         total_withdrawals,
+        blob_transactions,
+
+        // block detail level
         detail_level: block_detail_to_string(block.detail_level),
         detail_level_code: block.detail_level as u32,
     }

blocks/evm/src/pb/pinax.evm.v1.rs

@@ -38,27 +38,28 @@ pub struct Block {
     pub date: ::prost::alloc::string::String,
     #[prost(string, tag="4")]
     pub hash: ::prost::alloc::string::String,
-    /// -- header --
+    /// -- roots --
     #[prost(string, tag="5")]
-    pub parent_hash: ::prost::alloc::string::String,
-    #[prost(uint64, tag="6")]
-    pub nonce: u64,
-    #[prost(string, tag="7")]
     pub ommers_hash: ::prost::alloc::string::String,
-    #[prost(string, tag="8")]
+    #[prost(string, tag="6")]
     pub logs_bloom: ::prost::alloc::string::String,
-    #[prost(string, tag="9")]
+    #[prost(string, tag="7")]
     pub transactions_root: ::prost::alloc::string::String,
-    #[prost(string, tag="10")]
+    #[prost(string, tag="8")]
     pub state_root: ::prost::alloc::string::String,
-    #[prost(string, tag="11")]
+    #[prost(string, tag="9")]
     pub receipts_root: ::prost::alloc::string::String,
     /// EVM Root EIP-4895 (Shangai Fork)
-    #[prost(string, tag="12")]
+    #[prost(string, tag="10")]
     pub withdrawals_root: ::prost::alloc::string::String,
     /// EVM Root EIP-4788 (Dencun Fork)
-    #[prost(string, tag="13")]
+    #[prost(string, tag="11")]
     pub parent_beacon_root: ::prost::alloc::string::String,
+    /// -- header --
+    #[prost(string, tag="12")]
+    pub parent_hash: ::prost::alloc::string::String,
+    #[prost(uint64, tag="13")]
+    pub nonce: u64,
     #[prost(string, tag="14")]
     pub miner: ::prost::alloc::string::String,
     #[prost(uint64, tag="15")]
@@ -79,38 +80,47 @@ pub struct Block {
     /// EIP-1559 (London Fork)
     #[prost(string, tag="22")]
     pub base_fee_per_gas: ::prost::alloc::string::String,
+    /// -- blobs --
+    ///
     /// EIP-4844 (Dencun Fork)
-    #[prost(string, tag="23")]
-    pub blob_gas_used: ::prost::alloc::string::String,
+    #[prost(uint64, tag="23")]
+    pub blob_gas_used: u64,
     /// EIP-4844 (Dencun Fork)
-    #[prost(string, tag="24")]
-    pub excess_blob_gas: ::prost::alloc::string::String,
+    #[prost(uint64, tag="24")]
+    pub excess_blob_gas: u64,
+    /// UInt256
+    #[prost(bytes="vec", tag="25")]
+    pub blob_gas_price_bytes: ::prost::alloc::vec::Vec<u8>,
     /// -- counters --
     ///
     /// block size in bytes
-    #[prost(uint64, tag="25")]
-    pub size: u64,
-    #[prost(uint64, tag="26")]
-    pub total_transactions: u64,
-    #[prost(uint64, tag="27")]
-    pub successful_transactions: u64,
-    #[prost(uint64, tag="28")]
-    pub failed_transactions: u64,
-    #[prost(uint64, tag="29")]
-    pub total_balance_changes: u64,
     #[prost(uint64, tag="30")]
-    pub total_withdrawals: u64,
+    pub size: u64,
+    #[prost(uint32, tag="31")]
+    pub total_transactions: u32,
+    #[prost(uint32, tag="32")]
+    pub successful_transactions: u32,
+    #[prost(uint32, tag="33")]
+    pub failed_transactions: u32,
+    #[prost(uint32, tag="34")]
+    pub total_balance_changes: u32,
+    #[prost(uint32, tag="35")]
+    pub total_withdrawals: u32,
+    /// EIP-4844 (Dencun Fork)
+    #[prost(uint32, tag="36")]
+    pub blob_transactions: u32,
     /// -- detail level --
-    #[prost(string, tag="31")]
+    #[prost(string, tag="37")]
     pub detail_level: ::prost::alloc::string::String,
-    #[prost(uint32, tag="32")]
+    #[prost(uint32, tag="38")]
     pub detail_level_code: u32,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Transaction {
     /// -- block --
-    /// google.protobuf.Timestamp block_time = 1;
+    #[prost(message, optional, tag="1")]
+    pub block_time: ::core::option::Option<::prost_types::Timestamp>,
     #[prost(uint64, tag="2")]
     pub block_number: u64,
     #[prost(string, tag="3")]
@@ -173,18 +183,22 @@ pub struct Transaction {
     pub begin_ordinal: u64,
     #[prost(uint64, tag="28")]
     pub end_ordinal: u64,
-    /// -- transaction receipt --
+    /// -- blobs --
     ///
     /// UInt256
     #[prost(bytes="vec", tag="29")]
     pub blob_gas_price_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(uint64, tag="30")]
     pub blob_gas_used: u64,
-    #[prost(uint64, tag="31")]
+    /// UInt256
+    #[prost(bytes="vec", tag="31")]
+    pub blob_gas_fee_cap: ::prost::alloc::vec::Vec<u8>,
+    /// -- transaction receipt --
+    #[prost(uint64, tag="33")]
     pub cumulative_gas_used: u64,
-    #[prost(string, tag="32")]
+    #[prost(string, tag="34")]
     pub logs_bloom: ::prost::alloc::string::String,
-    #[prost(string, tag="33")]
+    #[prost(string, tag="35")]
     pub state_root: ::prost::alloc::string::String,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]

blocks/evm/src/transactions.rs

@@ -46,14 +46,15 @@ pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec<Tr
             let receipt = transaction.receipt.clone().unwrap();
             Transaction {
                 // block
-                // block_time: Some(timestamp.time),
+                block_time: Some(timestamp.time),
                 block_number: timestamp.number,
                 block_hash: timestamp.hash.clone(),
                 block_date: timestamp.date.clone(),
 
                 // block roots
                 transactions_root: bytes_to_hex(&block_header.transactions_root),
                 receipts_root: bytes_to_hex(&block_header.receipt_root),
+                state_root: bytes_to_hex(&block_header.state_root),
 
                 // transaction
                 index: transaction.index,
@@ -78,11 +79,15 @@ pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec<Tr
                 max_priority_fee_per_gas_bytes: transaction.max_priority_fee_per_gas.clone().unwrap_or_default().bytes,
                 begin_ordinal: transaction.begin_ordinal,
                 end_ordinal: transaction.end_ordinal,
-                blob_gas_price_bytes: receipt.blob_gas_price.clone().unwrap_or_default().bytes,
-                blob_gas_used: receipt.blob_gas_used(),
+
+                // transaction receipt
                 cumulative_gas_used: receipt.cumulative_gas_used,
                 logs_bloom: bytes_to_hex(&receipt.logs_bloom),
-                state_root: bytes_to_hex(&receipt.state_root),
+
+                // blob
+                blob_gas_price_bytes: receipt.blob_gas_price.clone().unwrap_or_default().bytes,
+                blob_gas_used: receipt.blob_gas_used(),
+                blob_gas_fee_cap: transaction.clone().blob_gas_fee_cap.unwrap_or_default().bytes,
             }
         })
         .collect()