add blobs to blocks

blocks/evm/Makefile

@@ -23,7 +23,7 @@ parquet:
 
 .PHONY: s3
 s3:
-	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events s3://pinax/eth/d091b44a01fb21c3a882e56a49659cda9382cb48?region=us-east-1 21200000: --encoder parquet --development-mode --parquet-default-column-compression snappy --file-block-count 1
+	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events s3://pinax/eth/c532b0ea3ccdaffaa71f7ff6b9b58d0ebf931f60?region=us-east-1 21200000: --encoder parquet --development-mode --parquet-default-column-compression snappy --file-block-count 1
 
 .PHONY: hash
 hash:

blocks/evm/README.md

@@ -36,5 +36,5 @@ Kind: map
 Input: source: sf.substreams.v1.Clock
 Input: source: sf.ethereum.type.v2.Block
 Output Type: proto:evm.Events
-Hash: d091b44a01fb21c3a882e56a49659cda9382cb48
+Hash: c532b0ea3ccdaffaa71f7ff6b9b58d0ebf931f60
 ```

blocks/evm/schema.snowflake.sql

@@ -44,7 +44,7 @@ SELECT
   blob_gas_used,
   ZEROIFNULL(TRY_TO_DECIMAL(hex_encode(blob_gas_fee_cap_bytes), 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')) as blob_gas_fee_cap,
   blob_gas_fee_cap_bytes,
-  blob_gas,
+  blob_hashes,
 
   -- transaction receipt --
   cumulative_gas_used,

blocks/evm/src/blocks.rs

@@ -26,17 +26,13 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
     let successful_transactions = block.transaction_traces.iter().filter(|t| t.status == 1).count() as u32;
     let failed_transactions = total_transactions - successful_transactions;
     let total_withdrawals = block.balance_changes.iter().filter(|t| t.reason == 16).count() as u32;
-    let blob_transactions = block.transaction_traces.iter().filter(|t| t.r#type == 3).count() as u32;
 
     // blob price
     let blob_gas_price = block.transaction_traces.iter().find_map(|t| t.receipt.as_ref().and_then(|r| r.blob_gas_price.clone()));
-    // let blob_gas_used = block.transaction_traces.iter().filter_map(|t| t.receipt.as_ref()?.blob_gas_used);
-    // let blob_gas_used_sum: u64 = blob_gas_used.sum();
-    // let blob_gas_used: u64 = block.transaction_traces.iter().filter_map(|t| t.receipt.as_ref()?.blob_gas_used).sum();
-    // let blob_gas: u64 = block.transaction_traces.iter().filter_map(|t| t.blob_gas).sum();
-    // let blob_gas: u64 = block.transaction_traces.iter().filter_map(|t| t.blob_gas_fee_cap).sum();
-
-    // log::debug!("blob_gas_used: {}, blob_gas: {}", blob_gas_used, blob_gas);
+    let blob_hashes: Vec<String> = block.transaction_traces.iter().flat_map(|t| t.blob_hashes.iter().map(|hash| bytes_to_hex(hash))).collect();
+    let total_blobs: u32 = blob_hashes.len() as u32;
+    let blob_transactions = block.transaction_traces.iter().filter(|t| t.r#type == 3).map(|t| bytes_to_hex(&t.hash)).collect::<Vec<String>>();
+    let total_blob_transactions = blob_transactions.len() as u32;
 
     BlockHeader {
         // clock
@@ -71,6 +67,10 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
         blob_gas_used: header.blob_gas_used(),
         excess_blob_gas: header.excess_blob_gas(),
         blob_gas_price_bytes: blob_gas_price.unwrap_or_default().bytes,
+        blob_hashes,
+        blob_transactions,
+        total_blob_transactions,
+        total_blobs,
 
         // counters
         size: block.size,
@@ -79,7 +79,6 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
         failed_transactions,
         total_balance_changes: block.balance_changes.len() as u32,
         total_withdrawals,
-        blob_transactions,
 
         // block detail level
         detail_level: block_detail_to_string(block.detail_level),

blocks/evm/src/pb/pinax.evm.v1.rs

@@ -91,6 +91,18 @@ pub struct Block {
     /// UInt256
     #[prost(bytes="vec", tag="25")]
     pub blob_gas_price_bytes: ::prost::alloc::vec::Vec<u8>,
+    /// EIP-4844 (Dencun Fork)
+    #[prost(string, repeated, tag="26")]
+    pub blob_transactions: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    /// EIP-4844 (Dencun Fork)
+    #[prost(string, repeated, tag="27")]
+    pub blob_hashes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    /// EIP-4844 (Dencun Fork)
+    #[prost(uint32, tag="28")]
+    pub total_blob_transactions: u32,
+    /// EIP-4844 (Dencun Fork),
+    #[prost(uint32, tag="29")]
+    pub total_blobs: u32,
     /// -- counters --
     ///
     /// block size in bytes
@@ -106,13 +118,10 @@ pub struct Block {
     pub total_balance_changes: u32,
     #[prost(uint32, tag="35")]
     pub total_withdrawals: u32,
-    /// EIP-4844 (Dencun Fork)
-    #[prost(uint32, tag="36")]
-    pub blob_transactions: u32,
     /// -- detail level --
-    #[prost(string, tag="37")]
+    #[prost(string, tag="38")]
     pub detail_level: ::prost::alloc::string::String,
-    #[prost(uint32, tag="38")]
+    #[prost(uint32, tag="39")]
     pub detail_level_code: u32,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -193,6 +202,9 @@ pub struct Transaction {
     /// UInt256
     #[prost(bytes="vec", tag="31")]
     pub blob_gas_fee_cap: ::prost::alloc::vec::Vec<u8>,
+    /// EIP-4844 (Dencun Fork)
+    #[prost(string, repeated, tag="32")]
+    pub blob_hashes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
     /// -- transaction receipt --
     #[prost(uint64, tag="33")]
     pub cumulative_gas_used: u64,

blocks/evm/src/transactions.rs

@@ -88,6 +88,7 @@ pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec<Tr
                 blob_gas_price_bytes: receipt.blob_gas_price.clone().unwrap_or_default().bytes,
                 blob_gas_used: receipt.blob_gas_used(),
                 blob_gas_fee_cap: transaction.clone().blob_gas_fee_cap.unwrap_or_default().bytes,
+                blob_hashes: transaction.blob_hashes.iter().map(|hash| bytes_to_hex(hash)).collect(),
             }
         })
         .collect()

proto/evm.proto

@@ -51,6 +51,10 @@ message Block {
   uint64 blob_gas_used = 23; // EIP-4844 (Dencun Fork)
   uint64 excess_blob_gas = 24; // EIP-4844 (Dencun Fork)
   bytes blob_gas_price_bytes = 25; // UInt256
+  repeated string blob_transactions = 26; // EIP-4844 (Dencun Fork)
+  repeated string blob_hashes = 27; // EIP-4844 (Dencun Fork)
+  uint32 total_blob_transactions = 28; // EIP-4844 (Dencun Fork)
+  uint32 total_blobs = 29; // EIP-4844 (Dencun Fork),
 
   // -- counters --
   uint64 size = 30; // block size in bytes
@@ -59,11 +63,10 @@ message Block {
   uint32 failed_transactions = 33;
   uint32 total_balance_changes = 34;
   uint32 total_withdrawals = 35;
-  uint32 blob_transactions = 36; // EIP-4844 (Dencun Fork)
 
   // -- detail level --
-  string detail_level = 37;
-  uint32 detail_level_code = 38;
+  string detail_level = 38;
+  uint32 detail_level_code = 39;
 }
 
 message Transaction {
@@ -105,6 +108,7 @@ message Transaction {
   bytes blob_gas_price_bytes = 29; // UInt256
   uint64 blob_gas_used = 30;
   bytes blob_gas_fee_cap = 31; // UInt256
+  repeated string blob_hashes = 32; // EIP-4844 (Dencun Fork)
 
   // -- transaction receipt --
   uint64 cumulative_gas_used = 33;