Merge pull request #1243 from chainwayxyz/nightly

Release 0.5.2

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v0.5.2 (2024-09-30)
+- Added config for disableing prover proving session recovery. ([#1241](https://github.com/chainwayxyz/citrea/pull/1241))
+- Nodes now log each RPC request and response. ([#1236](https://github.com/chainwayxyz/citrea/pull/1236))
+
 ## v0.5.1 (2024-09-26)
 
 - Fix bug where full nodes would query more soft confirmations than intended. ([#1230](https://github.com/chainwayxyz/citrea/pull/1230))

bin/citrea/tests/all_tests.rs

@@ -10,7 +10,6 @@ mod soft_confirmation_rule_enforcer;
 mod test_client;
 mod test_helpers;
 
-const DEFAULT_MIN_SOFT_CONFIRMATIONS_PER_COMMITMENT: u64 = 1000;
-const DEFAULT_DEPOSIT_MEMPOOL_FETCH_LIMIT: usize = 10;
+const TEST_SEND_NO_COMMITMENT_MIN_SOFT_CONFIRMATIONS_PER_COMMITMENT: u64 = 1000;
 const DEFAULT_PROOF_WAIT_DURATION: u64 = 600; // 10 minutes
 const TEST_DATA_GENESIS_PATH: &str = "../../resources/test-data/integration-tests";

bin/citrea/tests/bitcoin_e2e/bitcoin.rs

@@ -1,37 +1,39 @@
 use std::collections::HashSet;
 use std::path::PathBuf;
+use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use anyhow::{bail, Context};
 use async_trait::async_trait;
 use bitcoin::Address;
 use bitcoin_da::service::{get_relevant_blobs_from_txs, FINALITY_DEPTH};
 use bitcoin_da::spec::blob::BlobWithSender;
+use bitcoincore_rpc::json::AddressType::Bech32m;
 use bitcoincore_rpc::{Auth, Client, RpcApi};
 use citrea_primitives::REVEAL_BATCH_PROOF_PREFIX;
+use futures::TryStreamExt;
 use tokio::process::Command;
+use tokio::sync::OnceCell;
 use tokio::time::sleep;
 
 use super::config::BitcoinConfig;
 use super::docker::DockerEnv;
 use super::framework::TestContext;
-use super::node::{LogProvider, Node, SpawnOutput};
+use super::node::{LogProvider, Node, Restart, SpawnOutput};
 use super::Result;
 use crate::bitcoin_e2e::node::NodeKind;
 
 pub struct BitcoinNode {
     spawn_output: SpawnOutput,
     pub config: BitcoinConfig,
     client: Client,
-    gen_addr: Address,
+    gen_addr: OnceCell<Address>,
+    docker_env: Arc<Option<DockerEnv>>,
 }
 
 impl BitcoinNode {
-    pub async fn new(config: &BitcoinConfig, docker: &Option<DockerEnv>) -> Result<Self> {
-        let spawn_output = match docker {
-            Some(docker) => docker.spawn(config.into()).await?,
-            None => Self::spawn(config)?,
-        };
+    pub async fn new(config: &BitcoinConfig, docker: Arc<Option<DockerEnv>>) -> Result<Self> {
+        let spawn_output = Self::spawn(config, &docker).await?;
 
         let rpc_url = format!(
             "http://127.0.0.1:{}/wallet/{}",
@@ -45,28 +47,14 @@ impl BitcoinNode {
         .await
         .context("Failed to create RPC client")?;
 
-        wait_for_rpc_ready(&client, Duration::from_secs(60)).await?;
-        println!("bitcoin RPC is ready");
-
-        client
-            .create_wallet(&NodeKind::Sequencer.to_string(), None, None, None, None)
-            .await?;
-        client
-            .create_wallet(&NodeKind::Prover.to_string(), None, None, None, None)
-            .await?;
-        client
-            .create_wallet(&NodeKind::Bitcoin.to_string(), None, None, None, None)
-            .await?;
+        wait_for_rpc_ready(&client, None).await?;
 
-        let gen_addr = client
-            .get_new_address(None, Some(bitcoincore_rpc::json::AddressType::Bech32m))
-            .await?
-            .assume_checked();
         Ok(Self {
             spawn_output,
             config: config.clone(),
             client,
-            gen_addr,
+            gen_addr: OnceCell::new(),
+            docker_env: docker,
         })
     }
 
@@ -100,7 +88,7 @@ impl BitcoinNode {
         .context("Failed to create RPC client")?;
 
         let gen_addr = client
-            .get_new_address(None, Some(bitcoincore_rpc::json::AddressType::Bech32m))
+            .get_new_address(None, Some(Bech32m))
             .await?
             .assume_checked();
         client.generate_to_address(blocks, &gen_addr).await?;
@@ -120,6 +108,46 @@ impl BitcoinNode {
             REVEAL_BATCH_PROOF_PREFIX,
         ))
     }
+
+    async fn wait_for_shutdown(&self) -> Result<()> {
+        let timeout_duration = Duration::from_secs(30);
+        let start = std::time::Instant::now();
+
+        while start.elapsed() < timeout_duration {
+            if !self.is_process_running().await? {
+                println!("Bitcoin daemon has stopped successfully");
+                return Ok(());
+            }
+            sleep(Duration::from_millis(200)).await;
+        }
+
+        bail!("Timeout waiting for Bitcoin daemon to stop")
+    }
+
+    async fn is_process_running(&self) -> Result<bool> {
+        let data_dir = &self.config.data_dir;
+        let output = Command::new("pgrep")
+            .args(["-f", &format!("bitcoind.*{}", data_dir.display())])
+            .output()
+            .await?;
+
+        Ok(output.status.success())
+    }
+
+    // Infallible, discard already loaded errors
+    async fn load_wallets(&self) {
+        let _ = self.load_wallet(&NodeKind::Bitcoin.to_string()).await;
+        let _ = self.load_wallet(&NodeKind::Sequencer.to_string()).await;
+        let _ = self.load_wallet(&NodeKind::Prover.to_string()).await;
+    }
+
+    // Switch this over to Node signature once we add support for docker to citrea nodes
+    async fn spawn(config: &BitcoinConfig, docker: &Arc<Option<DockerEnv>>) -> Result<SpawnOutput> {
+        match docker.as_ref() {
+            Some(docker) => docker.spawn(config.into()).await,
+            None => <Self as Node>::spawn(config),
+        }
+    }
 }
 
 #[async_trait]
@@ -133,13 +161,24 @@ impl RpcApi for BitcoinNode {
     }
 
     // Override deprecated generate method.
-    // Uses node gen address and forward to `generate_to_address`
+    // Uses or lazy init gen_addr and forward to `generate_to_address`
     async fn generate(
         &self,
         block_num: u64,
         _maxtries: Option<u64>,
     ) -> bitcoincore_rpc::Result<Vec<bitcoin::BlockHash>> {
-        self.generate_to_address(block_num, &self.gen_addr).await
+        let addr = self
+            .gen_addr
+            .get_or_init(|| async {
+                self.client
+                    .get_new_address(None, Some(Bech32m))
+                    .await
+                    .expect("Failed to generate address")
+                    .assume_checked()
+            })
+            .await;
+
+        self.generate_to_address(block_num, addr).await
     }
 }
 
@@ -164,11 +203,15 @@ impl Node for BitcoinNode {
         &mut self.spawn_output
     }
 
-    async fn wait_for_ready(&self, timeout: Duration) -> Result<()> {
+    async fn wait_for_ready(&self, timeout: Option<Duration>) -> Result<()> {
         println!("Waiting for ready");
         let start = Instant::now();
+        let timeout = timeout.unwrap_or(Duration::from_secs(30));
         while start.elapsed() < timeout {
-            if wait_for_rpc_ready(&self.client, timeout).await.is_ok() {
+            if wait_for_rpc_ready(&self.client, Some(timeout))
+                .await
+                .is_ok()
+            {
                 return Ok(());
             }
             tokio::time::sleep(Duration::from_millis(500)).await;
@@ -183,6 +226,49 @@ impl Node for BitcoinNode {
     fn env(&self) -> Vec<(&'static str, &'static str)> {
         self.config.env.clone()
     }
+
+    fn config_mut(&mut self) -> &mut Self::Config {
+        &mut self.config
+    }
+}
+
+impl Restart for BitcoinNode {
+    async fn wait_until_stopped(&mut self) -> Result<()> {
+        self.client.stop().await?;
+        self.stop().await?;
+
+        match &self.spawn_output {
+            SpawnOutput::Child(_) => self.wait_for_shutdown().await,
+            SpawnOutput::Container(output) => {
+                let Some(env) = self.docker_env.as_ref() else {
+                    bail!("Missing docker environment")
+                };
+                env.docker.stop_container(&output.id, None).await?;
+
+                env.docker
+                    .wait_container::<String>(&output.id, None)
+                    .try_collect::<Vec<_>>()
+                    .await?;
+                env.docker.remove_container(&output.id, None).await?;
+                println!("Docker container {} succesfully removed", output.id);
+                Ok(())
+            }
+        }
+    }
+
+    async fn start(&mut self, config: Option<Self::Config>) -> Result<()> {
+        if let Some(config) = config {
+            self.config = config
+        }
+        self.spawn_output = Self::spawn(&self.config, &self.docker_env).await?;
+
+        self.wait_for_ready(None).await?;
+
+        // Reload wallets after restart
+        self.load_wallets().await;
+
+        Ok(())
+    }
 }
 
 impl LogProvider for BitcoinNode {
@@ -201,12 +287,12 @@ pub struct BitcoinNodeCluster {
 
 impl BitcoinNodeCluster {
     pub async fn new(ctx: &TestContext) -> Result<Self> {
-        let n_nodes = ctx.config.test_case.num_nodes;
+        let n_nodes = ctx.config.test_case.n_nodes;
         let mut cluster = Self {
             inner: Vec::with_capacity(n_nodes),
         };
         for config in ctx.config.bitcoin.iter() {
-            let node = BitcoinNode::new(config, &ctx.docker).await?;
+            let node = BitcoinNode::new(config, Arc::clone(&ctx.docker)).await?;
             cluster.inner.push(node)
         }
 
@@ -227,7 +313,6 @@ impl BitcoinNodeCluster {
             let mut heights = HashSet::new();
             for node in &self.inner {
                 let height = node.get_block_count().await?;
-                println!("height : {height}");
                 heights.insert(height);
             }
 
@@ -268,8 +353,9 @@ impl BitcoinNodeCluster {
     }
 }
 
-async fn wait_for_rpc_ready(client: &Client, timeout: Duration) -> Result<()> {
+async fn wait_for_rpc_ready(client: &Client, timeout: Option<Duration>) -> Result<()> {
     let start = Instant::now();
+    let timeout = timeout.unwrap_or(Duration::from_secs(300));
     while start.elapsed() < timeout {
         match client.get_blockchain_info().await {
             Ok(_) => return Ok(()),

bin/citrea/tests/bitcoin_e2e/config/bitcoin.rs

@@ -14,6 +14,7 @@ pub struct BitcoinConfig {
     pub network: Network,
     pub docker_image: Option<String>,
     pub env: Vec<(&'static str, &'static str)>,
+    pub idx: usize,
 }
 
 impl Default for BitcoinConfig {
@@ -30,6 +31,7 @@ impl Default for BitcoinConfig {
             network: Network::Regtest,
             docker_image: Some("bitcoin/bitcoin:latest".to_string()),
             env: Vec::new(),
+            idx: 0,
         }
     }
 }
@@ -44,7 +46,7 @@ impl BitcoinConfig {
             format!("-rpcuser={}", self.rpc_user),
             format!("-rpcpassword={}", self.rpc_password),
             "-server".to_string(),
-            "-daemon".to_string(),
+            "-daemonwait".to_string(),
             "-txindex".to_string(),
             "-addresstype=bech32m".to_string(),
             "-debug=net".to_string(),

bin/citrea/tests/bitcoin_e2e/config/docker.rs

@@ -3,12 +3,19 @@ use std::path::PathBuf;
 use super::{BitcoinConfig, FullSequencerConfig};
 use crate::bitcoin_e2e::utils::get_genesis_path;
 
+#[derive(Debug)]
+pub struct VolumeConfig {
+    pub name: String,
+    pub target: String,
+}
+
+#[derive(Debug)]
 pub struct DockerConfig {
     pub ports: Vec<u16>,
     pub image: String,
     pub cmd: Vec<String>,
-    pub dir: String,
     pub log_path: PathBuf,
+    pub volume: VolumeConfig,
 }
 
 impl From<&BitcoinConfig> for DockerConfig {
@@ -17,22 +24,23 @@ impl From<&BitcoinConfig> for DockerConfig {
 
         // Docker specific args
         args.extend([
-            "-datadir=/bitcoin/data".to_string(),
             "-rpcallowip=0.0.0.0/0".to_string(),
             "-rpcbind=0.0.0.0".to_string(),
-            "-daemon=0".to_string(),
+            "-daemonwait=0".to_string(),
         ]);
 
-        println!("Running bitcoind with args : {args:?}");
         Self {
             ports: vec![v.rpc_port, v.p2p_port],
             image: v
                 .docker_image
                 .clone()
                 .unwrap_or_else(|| "bitcoin/bitcoin:latest".to_string()),
             cmd: args,
-            dir: format!("{}:/bitcoin/data", v.data_dir.display()),
             log_path: v.data_dir.join("regtest").join("debug.log"),
+            volume: VolumeConfig {
+                name: format!("bitcoin-{}", v.idx),
+                target: "/home/bitcoin/.bitcoin".to_string(),
+            },
         }
     }
 }
@@ -59,8 +67,11 @@ impl From<&FullSequencerConfig> for DockerConfig {
                 .clone()
                 .unwrap_or_else(|| "citrea:latest".to_string()), // Default to local image
             cmd: args,
-            dir: format!("{}:/sequencer/data", v.dir.display()),
             log_path: v.dir.join("stdout"),
+            volume: VolumeConfig {
+                name: "sequencer".to_string(),
+                target: "/sequencer/data".to_string(),
+            },
         }
     }
 }