Prometheus metrics (#1589)

Cargo.toml

@@ -48,9 +48,6 @@ publish = false
 repository = "https://github.com/chainwayxyz/citrea"
 
 [workspace.dependencies]
-# Dependencies maintained by Sovereign
-jmt = { git = "https://github.com/penumbra-zone/jmt.git", rev = "fd1c8ef" }
-
 # External dependencies
 async-trait = "0.1.71"
 anyhow = { version = "1.0.68", default-features = false }
@@ -62,46 +59,55 @@ bincode = "1.3.3"
 bitcoin = { version = "0.32.2", features = ["serde", "rand"] }
 bitcoincore-rpc = { version = "0.18.0" }
 bcs = "0.1.6"
+bech32 = { version = "0.9.1", default-features = false }
 brotli = "7"
 byteorder = { version = "1.5.0", default-features = false }
 bytes = { version = "1.2.1", default-features = false }
 chrono = { version = "0.4.37", default-features = false }
+clap = { version = "4.4.10", features = ["derive"] }
 crypto-bigint = { version = "0.5.5" }
 digest = { version = "0.10.6", default-features = false, features = ["alloc"] }
+derive_more = { version = "0.99.11", default-features = false }
+ed25519-dalek = { version = "2", default-features = false, features = ["serde", "fast"] }
+futures = "0.3"
+hyper = { version = "1.4.0" }
 itertools = { version = "0.13.0", default-features = false }
+jmt = { git = "https://github.com/penumbra-zone/jmt.git", rev = "fd1c8ef" }
+jsonrpsee = { version = "0.24.2", features = ["jsonrpsee-types"] }
 lru = "0.12.3"
-rs_merkle = "1.4.2"
-futures = "0.3"
-pin-project = { version = "1.1.3" }
 hex = { version = "0.4.3", default-features = false, features = ["alloc", "serde"] }
 lazy_static = { version = "1.5.0" }
 log-panics = { version = "2", features = ["with-backtrace"] }
 once_cell = { version = "1.19.0", default-features = false, features = ["alloc"] }
+metrics = { version = "0.23.0" }
+metrics-derive = { version = "0.1.0" }
+metrics-exporter-prometheus = { version = "0.15.3" }
+metrics-util = { version = "0.17.0" }
+num_cpus = "1.0"
 parking_lot = { version = "0.12.3" }
-prometheus = { version = "0.13.3", default-features = false }
 proptest = { version = "1.3.1", default-features = false, features = ["alloc"] }
+pin-project = { version = "1.1.3" }
 rand = "0.8"
 rayon = "1.8.0"
 rlimit = "0.10.2"
 rustc_version_runtime = { version = "0.3.0", default-features = false }
+rs_merkle = "1.4.2"
 reqwest = { version = "0.12.5", features = ["rustls-tls", "json", "http2"], default-features = false }
 rocksdb = { version = "0.22.0", features = ["lz4"], default-features = false }
 serde = { version = "1.0.192", default-features = false, features = ["alloc", "derive"] }
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 sha2 = { version = "0.10.8", default-features = false }
+schemars = { version = "0.8.16", features = ["derive"] }
+secp256k1 = { version = "0.29.0", default-features = false, features = ["global-context", "recovery"] }
 thiserror = "1.0.50"
 tracing = { version = "0.1.40", default-features = false, features = ["attributes"] }
 tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "fmt"] }
-bech32 = { version = "0.9.1", default-features = false }
-derive_more = { version = "0.99.11", default-features = false }
-clap = { version = "4.4.10", features = ["derive"] }
 toml = "0.8.0"
-jsonrpsee = { version = "0.24.2", features = ["jsonrpsee-types"] }
-schemars = { version = "0.8.16", features = ["derive"] }
 tempfile = "3.8"
 tokio = { version = "1.40", features = ["full"] }
 tokio-util = { version = "0.7.12", features = ["rt"] }
-num_cpus = "1.0"
+tower-http = { version = "0.5.0", features = ["full"] }
+tower = { version = "0.4.13", features = ["full"] }
 
 # Risc0 dependencies
 risc0-binfmt = { version = "1.1.3" }
@@ -113,6 +119,9 @@ risc0-build = { version = "1.1.3" }
 bonsai-sdk = { version = "1.1.3" }
 
 # EVM dependencies
+revm = { version = "12.1", features = ["serde"], default-features = false }
+# forcing cargo for this version or else chooses 3.1.1 and there is some dependency conflicts
+revm-primitives = { version = "8", default-features = false }
 revm-inspectors = { version = "=0.5.5", default-features = false }
 reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "a206eb3690e5a51d3c797fed2a6ed722e36863eb", default-features = false }
 reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "a206eb3690e5a51d3c797fed2a6ed722e36863eb", default-features = false }
@@ -131,9 +140,6 @@ reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "a206eb3690e5a5
 reth-rpc = { git = "https://github.com/paradigmxyz/reth", rev = "a206eb3690e5a51d3c797fed2a6ed722e36863eb", default-features = false }
 reth-stages = { git = "https://github.com/paradigmxyz/reth", rev = "a206eb3690e5a51d3c797fed2a6ed722e36863eb", default-features = false }
 
-revm = { version = "12.1", features = ["serde"], default-features = false }
-# forcing cargo for this version or else chooses 3.1.1 and there is some dependency conflicts
-revm-primitives = { version = "8", default-features = false }
 alloy-trie = { version = "0.3.8", default-features = false }
 alloy-rlp = { version = "0.3.8", default-features = false }
 alloy-primitives = { version = "0.7.7", default-features = false }
@@ -142,13 +148,6 @@ alloy = { version = "0.2.1", default-features = false }
 alloy-eips = { version = "0.2.1", default-features = false }
 alloy-consensus = { version = "0.2.1", default-features = false }
 
-ed25519-dalek = { version = "2", default-features = false, features = ["serde", "fast"] }
-secp256k1 = { version = "0.29.0", default-features = false, features = ["global-context", "recovery"] }
-
-tower-http = { version = "0.5.0", features = ["full"] }
-tower = { version = "0.4.13", features = ["full"] }
-hyper = { version = "1.4.0" }
-
 citrea-e2e = { git = "https://github.com/chainwayxyz/citrea-e2e", rev = "6ba7230" }
 
 [patch.crates-io]

bin/citrea/Cargo.toml

@@ -51,6 +51,9 @@ hex = { workspace = true, optional = true }
 jsonrpsee = { workspace = true, features = ["http-client", "server"] }
 lazy_static = { workspace = true }
 log-panics = { workspace = true }
+metrics = { workspace = true }
+metrics-exporter-prometheus = { workspace = true, default-features = true }
+metrics-util = { workspace = true }
 reth-primitives = { workspace = true }
 reth-rpc-types = { workspace = true }
 reth-transaction-pool = { workspace = true }

bin/citrea/src/main.rs

@@ -1,6 +1,8 @@
 use core::fmt::Debug as DebugTrait;
+use std::net::SocketAddr;
+use std::time::Duration;
 
-use anyhow::Context as _;
+use anyhow::{anyhow, Context as _};
 use bitcoin_da::service::BitcoinServiceConfig;
 use citrea::{
     initialize_logging, BitcoinRollup, CitreaRollupBlueprint, MockDemoRollup, NetworkArg,
@@ -11,11 +13,13 @@ use citrea_common::{
 };
 use citrea_stf::genesis_config::GenesisPaths;
 use clap::Parser;
+use metrics_exporter_prometheus::PrometheusBuilder;
+use metrics_util::MetricKindMask;
 use sov_mock_da::MockDaConfig;
 use sov_modules_api::Spec;
 use sov_modules_rollup_blueprint::{Network, RollupBlueprint};
 use sov_state::storage::NativeStorage;
-use tracing::{error, info, instrument};
+use tracing::{debug, error, info, instrument};
 
 #[cfg(test)]
 mod test_rpc;
@@ -201,6 +205,26 @@ where
         None => FullNodeConfig::from_env()
             .context("Failed to read rollup configuration from the environment")?,
     };
+
+    if rollup_config.telemetry.bind_host.is_some() && rollup_config.telemetry.bind_port.is_some() {
+        let bind_host = rollup_config.telemetry.bind_host.as_ref().unwrap();
+        let bind_port = rollup_config.telemetry.bind_port.as_ref().unwrap();
+        let telemetry_addr: SocketAddr = format!("{}:{}", bind_host, bind_port)
+            .parse()
+            .map_err(|_| anyhow!("Invalid telemetry address"))?;
+
+        debug!("Starting telemetry server on: {}", telemetry_addr);
+
+        let builder = PrometheusBuilder::new().with_http_listener(telemetry_addr);
+        builder
+            .idle_timeout(
+                MetricKindMask::GAUGE | MetricKindMask::HISTOGRAM,
+                Some(Duration::from_secs(30)),
+            )
+            .install()
+            .map_err(|_| anyhow!("failed to install Prometheus recorder"))?;
+    }
+
     let rollup_blueprint = S::new(network);
 
     if let Some(sequencer_config) = sequencer_config {

bin/citrea/tests/test_helpers/mod.rs

@@ -184,6 +184,7 @@ pub fn create_default_rollup_config(
             sender_address: MockAddress::from([0; 32]),
             db_path: da_path.to_path_buf(),
         },
+        telemetry: Default::default(),
     }
 }
 

crates/batch-prover/Cargo.toml

@@ -31,7 +31,10 @@ borsh = { workspace = true }
 futures = { workspace = true }
 hex = { workspace = true }
 jsonrpsee = { workspace = true, features = ["http-client", "server", "client"] }
+metrics = { workspace = true }
+metrics-derive = { workspace = true }
 num_cpus = { workspace = true }
+once_cell = { workspace = true, default-features = true }
 parking_lot = { workspace = true }
 rand = { workspace = true }
 rayon = { workspace = true }

crates/batch-prover/src/da_block_handler.rs

@@ -33,6 +33,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
 use crate::errors::L1ProcessingError;
+use crate::metrics::BATCH_PROVER_METRICS;
 use crate::proving::{data_to_prove, extract_and_store_proof, prove_l1, GroupCommitments};
 
 type CommitmentStateTransitionData<'txs, Witness, Da, Tx> = (
@@ -270,6 +271,8 @@ where
                 );
             }
 
+            BATCH_PROVER_METRICS.current_l1_block.set(l1_height as f64);
+
             self.pending_l1_blocks.pop_front();
         }
         Ok(())

crates/batch-prover/src/lib.rs

@@ -3,6 +3,7 @@ pub mod db_migrations;
 mod errors;
 mod runner;
 pub use runner::*;
+mod metrics;
 mod proving;
 pub mod rpc;
 

crates/batch-prover/src/metrics.rs

@@ -0,0 +1,20 @@
+use metrics::{Gauge, Histogram};
+use metrics_derive::Metrics;
+use once_cell::sync::Lazy;
+
+#[derive(Metrics)]
+#[metrics(scope = "batch_prover")]
+pub struct BatchProverMetrics {
+    #[metric(describe = "The current L1 block number which is used to produce L2 blocks")]
+    pub current_l1_block: Gauge,
+    #[metric(describe = "The current L2 block number")]
+    pub current_l2_block: Gauge,
+    #[metric(describe = "The duration of processing a single soft confirmation")]
+    pub process_soft_confirmation: Histogram,
+}
+
+/// Batch prover metrics
+pub static BATCH_PROVER_METRICS: Lazy<BatchProverMetrics> = Lazy::new(|| {
+    BatchProverMetrics::describe();
+    BatchProverMetrics::default()
+});

crates/batch-prover/src/runner.rs

@@ -2,7 +2,7 @@ use core::panic;
 use std::collections::{HashMap, VecDeque};
 use std::net::SocketAddr;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use anyhow::{anyhow, bail, Context as _};
 use backoff::exponential::ExponentialBackoffBuilder;
@@ -38,6 +38,7 @@ use tokio::time::sleep;
 use tracing::{debug, error, info, instrument};
 
 use crate::da_block_handler::L1BlockHandler;
+use crate::metrics::BATCH_PROVER_METRICS;
 use crate::rpc::{create_rpc_module, RpcContext};
 
 type StfStateRoot<C, Da, RT> = <StfBlueprint<C, Da, RT> as StateTransitionFunction<Da>>::StateRoot;
@@ -395,6 +396,8 @@ where
         l2_height: u64,
         soft_confirmation: &SoftConfirmationResponse,
     ) -> anyhow::Result<()> {
+        let start = Instant::now();
+
         let current_l1_block = get_da_block_at_height(
             &self.da_service,
             soft_confirmation.da_slot_height,
@@ -487,6 +490,13 @@ where
             l2_height, self.state_root
         );
 
+        BATCH_PROVER_METRICS.current_l2_block.set(l2_height as f64);
+        BATCH_PROVER_METRICS.process_soft_confirmation.record(
+            Instant::now()
+                .saturating_duration_since(start)
+                .as_secs_f64(),
+        );
+
         Ok(())
     }
 

crates/bitcoin-da/Cargo.toml

@@ -26,6 +26,7 @@ crypto-bigint = { workspace = true }
 hex = { workspace = true, features = ["serde"] }
 itertools = { workspace = true }
 jsonrpsee = { workspace = true, optional = true }
+metrics = { workspace = true, optional = true }
 pin-project = { workspace = true, optional = true, features = [] }
 rand = { workspace = true }
 reqwest = { workspace = true, optional = true }
@@ -49,6 +50,7 @@ native = [
   "dep:backoff",
   "dep:tokio",
   "dep:tokio-util",
+  "dep:metrics",
   "dep:pin-project",
   "dep:tracing",
   "sov-rollup-interface/native",

crates/bitcoin-da/src/helpers/builders/batch_proof_namespace.rs

@@ -2,6 +2,7 @@ use core::result::Result::Ok;
 use std::fs::File;
 use std::io::{BufWriter, Write};
 use std::path::PathBuf;
+use std::time::Instant;
 
 use bitcoin::blockdata::opcodes::all::{OP_ENDIF, OP_IF};
 use bitcoin::blockdata::opcodes::OP_FALSE;
@@ -13,6 +14,7 @@ use bitcoin::opcodes::all::{OP_CHECKSIGVERIFY, OP_NIP};
 use bitcoin::script::PushBytesBuf;
 use bitcoin::secp256k1::{Secp256k1, SecretKey, XOnlyPublicKey};
 use bitcoin::{Address, Amount, Network, Transaction};
+use metrics::histogram;
 use serde::Serialize;
 use tracing::{instrument, trace, warn};
 
@@ -119,7 +121,7 @@ pub fn create_batchproof_type_0(
         .push_slice(PushBytesBuf::try_from(body).expect("Cannot push sequencer commitment"))
         .push_opcode(OP_ENDIF);
 
-    println!("reveal_script_builder: {:?}", reveal_script_builder);
+    let start = Instant::now();
     // Start loop to find a 'nonce' i.e. random number that makes the reveal tx hash starting with zeros given length
     let mut nonce: i64 = 16; // skip the first digits to avoid OP_PUSHNUM_X
     loop {
@@ -211,6 +213,12 @@ pub fn create_batchproof_type_0(
                     commit_tx_address
                 );
 
+                histogram!("mine_da_transaction").record(
+                    Instant::now()
+                        .saturating_duration_since(start)
+                        .as_secs_f64(),
+                );
+
                 return Ok(BatchProvingTxs {
                     commit: unsigned_commit_tx,
                     reveal: TxWithId {