diff --git a/Cargo.toml b/Cargo.toml index 5568d4bca..3406d4f61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ byte-unit = "4.0.12" bytes = "1.0" chrono = { version = "0.4" } clap = { version = "4.1.11", features = ["derive", "env"] } -derive_more = { version="0.99.9" } +derive_more = { version = "0.99.9" } futures = "0.3" lazy_static = "1.4.0" maplit = "1.0.2" @@ -31,16 +31,17 @@ proc-macro2 = { version = ">=1.0.0,<1.0.80", features = [] } quote = "1.0" rand = "0.8" semver = "1.0.14" -serde = { version="1.0.114", features=["derive", "rc"]} +serde = { version = "1.0.114", features = ["derive", "rc"] } serde_json = "1.0.57" syn = "2.0" tempfile = { version = "3.4.0" } +test-harness = "0.3.0" thiserror = "1.0.49" -tokio = { version="1.22", default-features=false, features=["io-util", "macros", "rt", "rt-multi-thread", "sync", "time"] } +tokio = { version = "1.22", default-features = false, features = ["io-util", "macros", "rt", "rt-multi-thread", "sync", "time"] } tracing = { version = "0.1.40" } tracing-appender = "0.2.0" tracing-futures = "0.2.4" -tracing-subscriber = { version = "0.3.3", features=["env-filter"] } +tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } validit = { version = "0.2.2" } [workspace] diff --git a/openraft/src/core/sm/worker.rs b/openraft/src/core/sm/worker.rs index a75470a64..71b452021 100644 --- a/openraft/src/core/sm/worker.rs +++ b/openraft/src/core/sm/worker.rs @@ -1,4 +1,5 @@ use anyerror::AnyError; +use tracing_futures::Instrument; use crate::async_runtime::MpscUnboundedReceiver; use crate::async_runtime::MpscUnboundedSender; @@ -59,6 +60,7 @@ where state_machine: SM, log_reader: LR, resp_tx: MpscUnboundedSenderOf>, + span: tracing::Span, ) -> Handle { let (cmd_tx, cmd_rx) = C::mpsc_unbounded(); @@ -69,13 +71,13 @@ where resp_tx, }; - let join_handle = worker.do_spawn(); + let join_handle = worker.do_spawn(span); Handle { cmd_tx, join_handle } } - fn do_spawn(mut self) -> JoinHandleOf { - C::spawn(async move { + fn do_spawn(mut self, span: tracing::Span) -> JoinHandleOf { + let fu = async move { let res = self.worker_loop().await; if let Err(err) = res { @@ -85,7 +87,8 @@ where command_result: CommandResult { result: Err(err) }, }); } - }) + }; + C::spawn(fu.instrument(span)) } #[tracing::instrument(level = "debug", skip_all)] diff --git a/openraft/src/raft/mod.rs b/openraft/src/raft/mod.rs index e1f29dcab..eeb4f912c 100644 --- a/openraft/src/raft/mod.rs +++ b/openraft/src/raft/mod.rs @@ -271,7 +271,14 @@ where C: RaftTypeConfig let engine = Engine::new(state, eng_config); - let sm_handle = worker::Worker::spawn(state_machine, log_store.get_log_reader().await, tx_notify.clone()); + let sm_span = tracing::span!(parent: &core_span, Level::DEBUG, "sm_worker"); + + let sm_handle = worker::Worker::spawn( + state_machine, + log_store.get_log_reader().await, + tx_notify.clone(), + sm_span, + ); let core: RaftCore = RaftCore { id, diff --git a/scripts/check.kdl b/scripts/check.kdl index e1327a832..ac6245c1d 100755 --- a/scripts/check.kdl +++ b/scripts/check.kdl @@ -21,14 +21,17 @@ layout { pane { command "cargo" args "test" "--lib" + close_on_exit true } pane { command "cargo" args "test" "--test" "*" + close_on_exit true } pane { command "cargo" args "clippy" "--no-deps" "--all-targets" "--" "-D" "warnings" + close_on_exit true } } // status-bar diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 1615a2226..5336db22a 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -30,6 +30,7 @@ lazy_static = { workspace = true } maplit = { workspace = true } pretty_assertions = { workspace = true } rand = { workspace = true } +test-harness = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-appender = { workspace = true } diff --git a/tests/tests/append_entries/t10_conflict_with_empty_entries.rs b/tests/tests/append_entries/t10_conflict_with_empty_entries.rs index 7fce03338..c0ca53561 100644 --- a/tests/tests/append_entries/t10_conflict_with_empty_entries.rs +++ b/tests/tests/append_entries/t10_conflict_with_empty_entries.rs @@ -15,7 +15,7 @@ use openraft::LogId; use openraft::Vote; use openraft_memstore::ClientRequest; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster conflict_with_empty_entries test. @@ -36,7 +36,8 @@ use crate::fixtures::RaftRouter; /// /// - send `append_logs` message with conflicting prev_log_index and empty `entries`. /// - asserts that a response with ConflictOpt set. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn conflict_with_empty_entries() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/append_entries/t10_see_higher_vote.rs b/tests/tests/append_entries/t10_see_higher_vote.rs index 3263f88c1..8f4327589 100644 --- a/tests/tests/append_entries/t10_see_higher_vote.rs +++ b/tests/tests/append_entries/t10_see_higher_vote.rs @@ -15,11 +15,12 @@ use openraft::Vote; use openraft_memstore::ClientRequest; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// A leader reverts to follower if a higher vote is seen when append-entries. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_sees_higher_vote() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/append_entries/t11_append_conflicts.rs b/tests/tests/append_entries/t11_append_conflicts.rs index 106a50316..5436f11a1 100644 --- a/tests/tests/append_entries/t11_append_conflicts.rs +++ b/tests/tests/append_entries/t11_append_conflicts.rs @@ -15,13 +15,14 @@ use openraft::RaftTypeConfig; use openraft::ServerState; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test append-entries response in every case. /// /// - bring up a learner and send to it append_entries request. Check the response in every case. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_conflicts() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/append_entries/t11_append_entries_with_bigger_term.rs b/tests/tests/append_entries/t11_append_entries_with_bigger_term.rs index 7f8f26bcf..12a084fa4 100644 --- a/tests/tests/append_entries/t11_append_entries_with_bigger_term.rs +++ b/tests/tests/append_entries/t11_append_entries_with_bigger_term.rs @@ -13,7 +13,7 @@ use openraft::Config; use openraft::LogId; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// append-entries should update the vote when adding new logs with greater vote. @@ -21,7 +21,8 @@ use crate::fixtures::RaftRouter; /// - Bring up a learner and send to it append_entries request. /// /// Check the vote updated. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_entries_with_bigger_term() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/append_entries/t11_append_inconsistent_log.rs b/tests/tests/append_entries/t11_append_inconsistent_log.rs index 1782c570a..9ff32005c 100644 --- a/tests/tests/append_entries/t11_append_inconsistent_log.rs +++ b/tests/tests/append_entries/t11_append_inconsistent_log.rs @@ -11,7 +11,7 @@ use openraft::RaftLogReader; use openraft::ServerState; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Too many inconsistent log should not block replication. @@ -31,7 +31,8 @@ use crate::fixtures::RaftRouter; /// /// - Start the cluster and node 2 start to replicate logs. /// - test the log should be replicated to node 0. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_inconsistent_log() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/append_entries/t11_append_updates_membership.rs b/tests/tests/append_entries/t11_append_updates_membership.rs index 7fcca4ee0..b8c61bd5c 100644 --- a/tests/tests/append_entries/t11_append_updates_membership.rs +++ b/tests/tests/append_entries/t11_append_updates_membership.rs @@ -14,14 +14,15 @@ use openraft::Membership; use openraft::ServerState; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// append-entries should update membership correctly when adding new logs and deleting /// inconsistent logs. /// /// - bring up a learner and send to it append_entries request. Check the membership updated. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_updates_membership() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/append_entries/t30_replication_1_voter_to_isolated_learner.rs b/tests/tests/append_entries/t30_replication_1_voter_to_isolated_learner.rs index 0af1e2ebc..c45211919 100644 --- a/tests/tests/append_entries/t30_replication_1_voter_to_isolated_learner.rs +++ b/tests/tests/append_entries/t30_replication_1_voter_to_isolated_learner.rs @@ -5,7 +5,7 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test replication to learner that is not in membership should not block. @@ -15,7 +15,8 @@ use crate::fixtures::RaftRouter; /// - bring on a cluster of 1 voter and 1 learner. /// - isolate replication to node 1. /// - client write should not be blocked. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn replication_1_voter_to_isolated_learner() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/append_entries/t60_enable_heartbeat.rs b/tests/tests/append_entries/t60_enable_heartbeat.rs index 60bd4a34f..820e9b463 100644 --- a/tests/tests/append_entries/t60_enable_heartbeat.rs +++ b/tests/tests/append_entries/t60_enable_heartbeat.rs @@ -9,11 +9,12 @@ use openraft::type_config::TypeConfigExt; use openraft::Config; use openraft_memstore::TypeConfig; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Enable heartbeat, heartbeat should be replicated. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn enable_heartbeat() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/append_entries/t61_heartbeat_reject_vote.rs b/tests/tests/append_entries/t61_heartbeat_reject_vote.rs index 5161e8a9b..baf7e5e46 100644 --- a/tests/tests/append_entries/t61_heartbeat_reject_vote.rs +++ b/tests/tests/append_entries/t61_heartbeat_reject_vote.rs @@ -11,11 +11,12 @@ use openraft::TokioInstant; use openraft::Vote; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// If a follower receives heartbeat, it should reject vote request until leader lease expired. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn heartbeat_reject_vote() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/append_entries/t61_large_heartbeat.rs b/tests/tests/append_entries/t61_large_heartbeat.rs index 40c373e28..2d63ffa9a 100644 --- a/tests/tests/append_entries/t61_large_heartbeat.rs +++ b/tests/tests/append_entries/t61_large_heartbeat.rs @@ -5,12 +5,13 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Large heartbeat should not block replication. /// I.e., replication should not be driven by heartbeat. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn large_heartbeat() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/append_entries/t90_issue_216_stale_last_log_id.rs b/tests/tests/append_entries/t90_issue_216_stale_last_log_id.rs index c4d68a4b2..2b947f824 100644 --- a/tests/tests/append_entries/t90_issue_216_stale_last_log_id.rs +++ b/tests/tests/append_entries/t90_issue_216_stale_last_log_id.rs @@ -5,7 +5,7 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Ensures the stale value of ReplicationCore.last_log_id won't affect replication. @@ -15,7 +15,8 @@ use crate::fixtures::RaftRouter; /// TODO(xp): `max_applied_log_to_keep` to be 0 makes it very easy to enter snapshot replication and /// it will keeps replicating every log by snapshot and get timeout. /// Thus it is disabled until we find another way to test it. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] #[ignore] async fn stale_last_log_id() -> Result<()> { // Setup test dependencies. diff --git a/tests/tests/client_api/t10_client_writes.rs b/tests/tests/client_api/t10_client_writes.rs index aa98a20c8..d363fcc9b 100644 --- a/tests/tests/client_api/t10_client_writes.rs +++ b/tests/tests/client_api/t10_client_writes.rs @@ -12,13 +12,14 @@ use openraft_memstore::ClientRequest; use openraft_memstore::IntoMemClientRequest; use openraft_memstore::TypeConfig; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// - create a stable 3-node cluster. /// - write a lot of data to it. /// - assert that the cluster stayed stable and has all of the expected data. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn client_writes() -> Result<()> { let config = Arc::new( Config { @@ -67,7 +68,8 @@ async fn client_writes() -> Result<()> { /// Test Raft::client_write_ff, /// /// Manually receive the client-write response via the returned `Responder::Receiver` -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn client_write_ff() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t11_client_reads.rs b/tests/tests/client_api/t11_client_reads.rs index 7db692e2e..31e49d1c6 100644 --- a/tests/tests/client_api/t11_client_reads.rs +++ b/tests/tests/client_api/t11_client_reads.rs @@ -10,7 +10,7 @@ use openraft::Config; use openraft::LogIdOptionExt; use openraft::RPCTypes; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RPCRequest; use crate::fixtures::RaftRouter; @@ -21,7 +21,8 @@ use crate::fixtures::RaftRouter; /// - create a stable 3-node cluster. /// - call the ensure_linearizable interface on the leader, and assert success. /// - call the ensure_linearizable interface on the followers, and assert failure. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn client_reads() -> Result<()> { let config = Arc::new( Config { @@ -68,7 +69,8 @@ async fn client_reads() -> Result<()> { /// - A leader that has not yet committed any log entries returns leader initialization log id(blank /// log id). /// - Return the last committed log id if the leader has committed any log entries. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn get_read_log_id() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t12_trigger_purge_log.rs b/tests/tests/client_api/t12_trigger_purge_log.rs index 18a1aa5e8..d34448a3d 100644 --- a/tests/tests/client_api/t12_trigger_purge_log.rs +++ b/tests/tests/client_api/t12_trigger_purge_log.rs @@ -6,11 +6,12 @@ use openraft::testing::log_id; use openraft::Config; use openraft::SnapshotPolicy; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Call `Raft::trigger_purged()` to purge logs. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn trigger_purge_log() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t13_begin_receiving_snapshot.rs b/tests/tests/client_api/t13_begin_receiving_snapshot.rs index 87a51b6f8..6ddd2a24a 100644 --- a/tests/tests/client_api/t13_begin_receiving_snapshot.rs +++ b/tests/tests/client_api/t13_begin_receiving_snapshot.rs @@ -4,10 +4,11 @@ use std::time::Duration; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn begin_receiving_snapshot() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t13_get_snapshot.rs b/tests/tests/client_api/t13_get_snapshot.rs index 9f1fd8102..f04c848eb 100644 --- a/tests/tests/client_api/t13_get_snapshot.rs +++ b/tests/tests/client_api/t13_get_snapshot.rs @@ -5,11 +5,12 @@ use maplit::btreeset; use openraft::testing::log_id; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Get snapshot with `Raft::get_snapshot()` -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn get_snapshot() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t13_install_full_snapshot.rs b/tests/tests/client_api/t13_install_full_snapshot.rs index e40098433..e8d891c02 100644 --- a/tests/tests/client_api/t13_install_full_snapshot.rs +++ b/tests/tests/client_api/t13_install_full_snapshot.rs @@ -6,10 +6,11 @@ use openraft::testing::log_id; use openraft::Config; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn install_full_snapshot() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t13_trigger_snapshot.rs b/tests/tests/client_api/t13_trigger_snapshot.rs index d03918799..6a1f2d003 100644 --- a/tests/tests/client_api/t13_trigger_snapshot.rs +++ b/tests/tests/client_api/t13_trigger_snapshot.rs @@ -6,11 +6,12 @@ use openraft::CommittedLeaderId; use openraft::Config; use openraft::LogId; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Manually trigger a snapshot with `Raft::trigger_snapshot()` on Leader and Follower. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn trigger_snapshot() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t16_with_raft_state.rs b/tests/tests/client_api/t16_with_raft_state.rs index a1f900e85..c95621f9e 100644 --- a/tests/tests/client_api/t16_with_raft_state.rs +++ b/tests/tests/client_api/t16_with_raft_state.rs @@ -6,11 +6,12 @@ use openraft::error::Fatal; use openraft::testing::log_id; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Access Raft state via `Raft::with_raft_state()` -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn with_raft_state() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t50_lagging_network_write.rs b/tests/tests/client_api/t50_lagging_network_write.rs index 29214dc0f..f76189057 100644 --- a/tests/tests/client_api/t50_lagging_network_write.rs +++ b/tests/tests/client_api/t50_lagging_network_write.rs @@ -6,7 +6,7 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Lagging network test. @@ -17,7 +17,8 @@ use crate::fixtures::RaftRouter; /// - bring a single-node cluster online. /// - add two Learner and then try to commit one log. /// - change config to a 3 members cluster and commit another log. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn lagging_network_write() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/client_api/t51_write_when_leader_quit.rs b/tests/tests/client_api/t51_write_when_leader_quit.rs index 6466c5f37..49e4c0a76 100644 --- a/tests/tests/client_api/t51_write_when_leader_quit.rs +++ b/tests/tests/client_api/t51_write_when_leader_quit.rs @@ -14,14 +14,15 @@ use openraft_memstore::ClientRequest; use openraft_memstore::IntoMemClientRequest; use tokio::sync::oneshot; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Client write will receive a [`ForwardToLeader`] error because of log reversion, when leader /// quit, even after log is appended. /// /// [`ForwardToLeader`]: openraft::error::ForwardToLeader -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn write_when_leader_quit_and_log_revert() -> Result<()> { let config = Arc::new( Config { @@ -95,7 +96,8 @@ async fn write_when_leader_quit_and_log_revert() -> Result<()> { /// switched. /// /// [`ForwardToLeader`]: openraft::error::ForwardToLeader -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn write_when_leader_switched() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/elect/t10_elect_compare_last_log.rs b/tests/tests/elect/t10_elect_compare_last_log.rs index 01e5786c7..dc25812b3 100644 --- a/tests/tests/elect/t10_elect_compare_last_log.rs +++ b/tests/tests/elect/t10_elect_compare_last_log.rs @@ -11,14 +11,15 @@ use openraft::Config; use openraft::ServerState; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// The last_log in a vote request must be greater or equal than the local one. /// /// - Fake a cluster with two node: with last log {2,1} and {1,1}. /// - Bring up the cluster and only node 0 can become leader. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn elect_compare_last_log() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/elect/t11_elect_seize_leadership.rs b/tests/tests/elect/t11_elect_seize_leadership.rs index 089e6e228..095f10edd 100644 --- a/tests/tests/elect/t11_elect_seize_leadership.rs +++ b/tests/tests/elect/t11_elect_seize_leadership.rs @@ -6,11 +6,12 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// A node with higher term takes leadership from the current leader. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn elect_seize_leadership() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/fixtures/mod.rs b/tests/tests/fixtures/mod.rs index 5bcb97152..99cdaa26b 100644 --- a/tests/tests/fixtures/mod.rs +++ b/tests/tests/fixtures/mod.rs @@ -81,6 +81,33 @@ pub type MemStateMachine = Arc; /// A concrete Raft type used during testing. pub type MemRaft = Raft; +/// Create a harness that sets up tracing and a tokio runtime for testing. +pub fn ut_harness(f: F) -> anyhow::Result<()> +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future> + 'static, +{ + fn func_name() -> &'static str { + let full_name = std::any::type_name::(); + full_name.rsplit("::").find(|name| *name != "{{closure}}").unwrap() + } + + #[allow(clippy::let_unit_value)] + let _g = init_default_ut_tracing(); + + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(8) + .enable_all() + .build() + .expect("Failed building the Runtime"); + + let res = rt.block_on(f()); + if let Err(e) = &res { + tracing::error!("{} error: {:?}", func_name::(), e); + } + res +} + pub fn init_default_ut_tracing() { static START: Once = Once::new(); diff --git a/tests/tests/life_cycle/t10_initialization.rs b/tests/tests/life_cycle/t10_initialization.rs index 2bd11ddb7..e13e36799 100644 --- a/tests/tests/life_cycle/t10_initialization.rs +++ b/tests/tests/life_cycle/t10_initialization.rs @@ -19,7 +19,7 @@ use openraft::ServerState; use openraft::StoredMembership; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster initialization test. @@ -33,7 +33,8 @@ use crate::fixtures::RaftRouter; /// - asserts that the cluster was able to come online, elect a leader and maintain a stable state. /// - asserts that the leader was able to successfully commit its initial payload and that all /// followers have successfully replicated the payload. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn initialization() -> anyhow::Result<()> { let config = Arc::new( Config { @@ -177,7 +178,8 @@ async fn initialization() -> anyhow::Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn initialize_err_target_not_include_target() -> anyhow::Result<()> { // Initialize a node with membership config that does not include the target node that accepts // the `initialize` request. @@ -212,7 +214,8 @@ async fn initialize_err_target_not_include_target() -> anyhow::Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn initialize_err_not_allowed() -> anyhow::Result<()> { // Initialize a node with membership config that does not include the target node that accepts // the `initialize` request. diff --git a/tests/tests/life_cycle/t11_shutdown.rs b/tests/tests/life_cycle/t11_shutdown.rs index 7764cdaf9..e2c52a83a 100644 --- a/tests/tests/life_cycle/t11_shutdown.rs +++ b/tests/tests/life_cycle/t11_shutdown.rs @@ -6,11 +6,12 @@ use openraft::error::Fatal; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Shutdown raft node and check the metrics change. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn shutdown() -> Result<()> { let config = Arc::new( Config { @@ -37,7 +38,8 @@ async fn shutdown() -> Result<()> { } /// A panicked RaftCore should also return a proper error the next time accessing the `Raft`. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn return_error_after_panic() -> Result<()> { let config = Arc::new( Config { @@ -74,7 +76,8 @@ async fn return_error_after_panic() -> Result<()> { } /// After shutdown(), access to Raft should return a Fatal::Stopped error. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn return_error_after_shutdown() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/life_cycle/t50_follower_restart_does_not_interrupt.rs b/tests/tests/life_cycle/t50_follower_restart_does_not_interrupt.rs index cba13338a..bb8ba9b27 100644 --- a/tests/tests/life_cycle/t50_follower_restart_does_not_interrupt.rs +++ b/tests/tests/life_cycle/t50_follower_restart_does_not_interrupt.rs @@ -4,11 +4,12 @@ use std::time::Duration; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// A follower that restarted should not interrupt a stable cluster by a too quick election. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn follower_restart_does_not_interrupt() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/life_cycle/t50_single_follower_restart.rs b/tests/tests/life_cycle/t50_single_follower_restart.rs index 4b36705e5..80c221245 100644 --- a/tests/tests/life_cycle/t50_single_follower_restart.rs +++ b/tests/tests/life_cycle/t50_single_follower_restart.rs @@ -8,14 +8,15 @@ use openraft::RaftLogReader; use openraft::ServerState; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Brings up a cluster of 1 node and restart it, when it is a follower. /// /// The single follower should become leader very quickly. Because it does not need to wait for an /// active leader to replicate to it. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn single_follower_restart() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/life_cycle/t50_single_leader_restart_re_apply_logs.rs b/tests/tests/life_cycle/t50_single_leader_restart_re_apply_logs.rs index fe0c97dea..c5e588a8d 100644 --- a/tests/tests/life_cycle/t50_single_leader_restart_re_apply_logs.rs +++ b/tests/tests/life_cycle/t50_single_leader_restart_re_apply_logs.rs @@ -5,7 +5,7 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::MemLogStore; use crate::fixtures::MemRaft; use crate::fixtures::MemStateMachine; @@ -13,7 +13,8 @@ use crate::fixtures::RaftRouter; /// A single leader should re-apply all logs upon startup, /// because itself is a quorum. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn single_leader_restart_re_apply_logs() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/life_cycle/t90_issue_607_single_restart.rs b/tests/tests/life_cycle/t90_issue_607_single_restart.rs index 1bffa0c9d..9fff7291c 100644 --- a/tests/tests/life_cycle/t90_issue_607_single_restart.rs +++ b/tests/tests/life_cycle/t90_issue_607_single_restart.rs @@ -4,7 +4,7 @@ use std::time::Duration; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Brings up a cluster of 1 node and restart it. @@ -12,7 +12,8 @@ use crate::fixtures::RaftRouter; /// Assert that `RaftCore.engine.state.server_state` and `RaftCore.leader_data` are consistent: /// `server_state == Leader && leader_data.is_some() || server_state != Leader && /// leader_data.is_none()`. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn single_restart() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/life_cycle/t90_issue_920_non_voter_leader_restart.rs b/tests/tests/life_cycle/t90_issue_920_non_voter_leader_restart.rs index a781f50b4..d8cff0cd2 100644 --- a/tests/tests/life_cycle/t90_issue_920_non_voter_leader_restart.rs +++ b/tests/tests/life_cycle/t90_issue_920_non_voter_leader_restart.rs @@ -6,12 +6,13 @@ use openraft::Config; use openraft::ServerState; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Special case: A leader that is not a member(neither a voter or non-voter) should be started too, /// as a learner. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn issue_920_non_member_leader_restart() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/log_store/t10_save_committed.rs b/tests/tests/log_store/t10_save_committed.rs index 899cee80b..8f91e6a90 100644 --- a/tests/tests/log_store/t10_save_committed.rs +++ b/tests/tests/log_store/t10_save_committed.rs @@ -7,11 +7,12 @@ use openraft::storage::RaftLogStorage; use openraft::testing::log_id; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Before applying log, write `committed` log id to log store. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn write_committed_log_id_to_log_store() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/management/t10_raft_config.rs b/tests/tests/management/t10_raft_config.rs index 6f7ecf4cc..3c665c4ad 100644 --- a/tests/tests/management/t10_raft_config.rs +++ b/tests/tests/management/t10_raft_config.rs @@ -4,11 +4,12 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Get config via [`Raft::config`](openraft::Raft::config) -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn raft_config() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/membership/t10_learner_restart.rs b/tests/tests/membership/t10_learner_restart.rs index 200121e2d..92116d03d 100644 --- a/tests/tests/membership/t10_learner_restart.rs +++ b/tests/tests/membership/t10_learner_restart.rs @@ -7,7 +7,7 @@ use openraft::Config; use openraft::Raft; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster learner_restart test. @@ -20,7 +20,8 @@ use crate::fixtures::RaftRouter; /// learner has successfully replicated the payload. /// - shutdown all and restart the learner node. /// - asserts the learner stays in non-voter state. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn learner_restart() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/membership/t10_single_node.rs b/tests/tests/membership/t10_single_node.rs index 3f806c9b3..1ff30c317 100644 --- a/tests/tests/membership/t10_single_node.rs +++ b/tests/tests/membership/t10_single_node.rs @@ -7,7 +7,7 @@ use openraft::CommittedLeaderId; use openraft::Config; use openraft::LogId; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Single-node cluster initialization test. @@ -19,7 +19,8 @@ use crate::fixtures::RaftRouter; /// - initializes the cluster with membership config including just the one node. /// - asserts that the cluster was able to come online, and that the one node became leader. /// - asserts that the leader was able to successfully commit its initial payload. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn single_node() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/membership/t11_add_learner.rs b/tests/tests/membership/t11_add_learner.rs index 4f1db6189..7aa72f48a 100644 --- a/tests/tests/membership/t11_add_learner.rs +++ b/tests/tests/membership/t11_add_learner.rs @@ -16,10 +16,11 @@ use openraft::RaftLogReader; use openraft::StorageHelper; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn add_learner_basic() -> Result<()> { // // - Add leader, expect NoChange @@ -96,7 +97,8 @@ async fn add_learner_basic() -> Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn add_learner_non_blocking() -> Result<()> { // // - Add leader, expect NoChange @@ -158,7 +160,8 @@ async fn add_learner_non_blocking() -> Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn add_learner_with_set_nodes() -> Result<()> { // Add learners and update nodes with ChangeMembers::SetNodes // Node updating is ensured by unit tests of ChangeMembers @@ -197,7 +200,8 @@ async fn add_learner_with_set_nodes() -> Result<()> { /// /// Because adding learner is also a change-membership operation, a new membership config log will /// let raft consider the previous membership config log as committed, which is actually not. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn add_learner_when_previous_membership_not_committed() -> Result<()> { let config = Arc::new( Config { @@ -245,7 +249,8 @@ async fn add_learner_when_previous_membership_not_committed() -> Result<()> { /// add a learner, then shutdown the leader to make leader transferred, /// check after new leader come, the learner can receive new log. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn check_learner_after_leader_transferred() -> Result<()> { // TODO(1): flaky with --features single-term-leader diff --git a/tests/tests/membership/t12_concurrent_write_and_add_learner.rs b/tests/tests/membership/t12_concurrent_write_and_add_learner.rs index 94702af3f..9338e2e7a 100644 --- a/tests/tests/membership/t12_concurrent_write_and_add_learner.rs +++ b/tests/tests/membership/t12_concurrent_write_and_add_learner.rs @@ -8,8 +8,9 @@ use openraft::Config; use openraft::LogIdOptionExt; use openraft::ServerState; use openraft::Vote; +use tracing::Instrument; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster concurrent_write_and_add_learner test. @@ -35,7 +36,8 @@ use crate::fixtures::RaftRouter; /// - brings a 3 candidates cluster online. /// - add another learner and at the same time write a log. /// - asserts that all of the leader, followers and the learner receives all logs. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn concurrent_write_and_add_learner() -> Result<()> { let candidates = btreeset![0, 1, 2]; diff --git a/tests/tests/membership/t20_change_membership.rs b/tests/tests/membership/t20_change_membership.rs index dc5b68133..b9825588a 100644 --- a/tests/tests/membership/t20_change_membership.rs +++ b/tests/tests/membership/t20_change_membership.rs @@ -9,11 +9,12 @@ use openraft::LogIdOptionExt; use openraft::RaftLogReader; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When a change-membership log is committed, the `RaftState.membership_state` should be updated. -#[async_entry::test(worker_threads = 3, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn update_membership_state() -> anyhow::Result<()> { let config = Arc::new( Config { @@ -50,7 +51,8 @@ async fn update_membership_state() -> anyhow::Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn change_with_new_learner_blocking() -> anyhow::Result<()> { // Add a member without adding it as learner, in blocking mode it should finish successfully. @@ -100,7 +102,8 @@ async fn change_with_new_learner_blocking() -> anyhow::Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn change_without_adding_learner() -> anyhow::Result<()> { let config = Arc::new(Config { ..Default::default() }.validate()?); let mut router = RaftRouter::new(config.clone()); @@ -147,7 +150,8 @@ async fn change_without_adding_learner() -> anyhow::Result<()> { Ok(()) } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn change_with_turn_removed_voter_to_learner() -> anyhow::Result<()> { // Add a member without adding it as learner, in blocking mode it should finish successfully. diff --git a/tests/tests/membership/t21_change_membership_cases.rs b/tests/tests/membership/t21_change_membership_cases.rs index eb3599fb2..18d6cefa0 100644 --- a/tests/tests/membership/t21_change_membership_cases.rs +++ b/tests/tests/membership/t21_change_membership_cases.rs @@ -8,91 +8,107 @@ use openraft::Config; use openraft::ServerState; use openraft_memstore::MemNodeId; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; // --- change --- -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m0_change_m12() -> anyhow::Result<()> { change_from_to(btreeset! {0}, btreeset! {1,2}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m0_change_m123() -> anyhow::Result<()> { change_from_to(btreeset! {0}, btreeset! {1,2,3}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m01_change_m12() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1}, btreeset! {1,2}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m01_change_m1() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1}, btreeset! {1}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m01_change_m2() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1}, btreeset! {2}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m01_change_m3() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1}, btreeset! {3}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m012_change_m4() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1, 2}, btreeset! {4}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m012_change_m456() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1, 2}, btreeset! {4,5,6}).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m01234_change_m0123() -> anyhow::Result<()> { change_from_to(btreeset! {0, 1, 2, 3, 4}, btreeset! {0,1,2,3}).await } // --- add --- -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m0_add_m01() -> anyhow::Result<()> { change_by_add(btreeset! {0}, &[0, 1]).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m0_add_m12() -> anyhow::Result<()> { change_by_add(btreeset! {0}, &[1, 2]).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m01_add_m() -> anyhow::Result<()> { change_by_add(btreeset! {0,1}, &[]).await } // --- remove --- -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m012_remove_m01() -> anyhow::Result<()> { change_by_remove(btreeset! {0,1,2}, &[0, 1]).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m012_remove_m3() -> anyhow::Result<()> { change_by_remove(btreeset! {0,1,2}, &[3]).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m012_remove_m() -> anyhow::Result<()> { change_by_remove(btreeset! {0,1,2}, &[]).await } -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn m012_remove_m13() -> anyhow::Result<()> { change_by_remove(btreeset! {0,1,2}, &[1, 3]).await } diff --git a/tests/tests/membership/t30_commit_joint_config.rs b/tests/tests/membership/t30_commit_joint_config.rs index 951b7b9d9..bda0ca736 100644 --- a/tests/tests/membership/t30_commit_joint_config.rs +++ b/tests/tests/membership/t30_commit_joint_config.rs @@ -5,8 +5,9 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; use openraft::LogIdOptionExt; +use tracing::Instrument; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// A leader must wait for learner to commit member-change from [0] to [0,1,2]. @@ -16,7 +17,8 @@ use crate::fixtures::RaftRouter; /// - Init 1 leader and 2 learner. /// - Isolate learner. /// - Asserts that membership change won't success. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn commit_joint_config_during_0_to_012() -> Result<()> { // Setup test dependencies. let config = Arc::new( @@ -90,7 +92,8 @@ async fn commit_joint_config_during_0_to_012() -> Result<()> { /// - bring a cluster of node 0,1,2 online. /// - isolate 3,4; change config to 2,3,4 /// - since new config can not form a quorum, the joint config should not be committed. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn commit_joint_config_during_012_to_234() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/membership/t30_elect_with_new_config.rs b/tests/tests/membership/t30_elect_with_new_config.rs index c4cf42c11..a5e42cf94 100644 --- a/tests/tests/membership/t30_elect_with_new_config.rs +++ b/tests/tests/membership/t30_elect_with_new_config.rs @@ -7,7 +7,7 @@ use openraft::Config; use openraft::LogIdOptionExt; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Dynamic membership test. @@ -20,7 +20,8 @@ use crate::fixtures::RaftRouter; /// down. /// - temporarily isolate the new master, and assert that a new master takes over. /// - restore the isolated node and assert that it becomes a follower. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn leader_election_after_changing_0_to_01234() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/membership/t31_add_remove_follower.rs b/tests/tests/membership/t31_add_remove_follower.rs index a33b7a1f3..eb6e8cbe0 100644 --- a/tests/tests/membership/t31_add_remove_follower.rs +++ b/tests/tests/membership/t31_add_remove_follower.rs @@ -6,7 +6,7 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When a node is removed from cluster, replication to it should be stopped. @@ -16,7 +16,8 @@ use crate::fixtures::RaftRouter; /// successfully replicated the payload. /// - remove one follower: node-4 /// - asserts node-4 becomes learner and the leader stops sending logs to it. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn add_remove_voter() -> Result<()> { let c01234 = btreeset![0, 1, 2, 3, 4]; let c0123 = btreeset![0, 1, 2, 3]; diff --git a/tests/tests/membership/t31_remove_leader.rs b/tests/tests/membership/t31_remove_leader.rs index f2a475a9b..f14a53715 100644 --- a/tests/tests/membership/t31_remove_leader.rs +++ b/tests/tests/membership/t31_remove_leader.rs @@ -11,14 +11,15 @@ use openraft::ServerState; use openraft_memstore::ClientRequest; use openraft_memstore::IntoMemClientRequest; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Change membership from {0,1} to {1,2,3}. /// /// - Then the leader should step down after joint log is committed. /// - Check logs on other node. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn remove_leader() -> Result<()> { // TODO(1): flaky with --features single-term-leader @@ -112,7 +113,8 @@ async fn remove_leader() -> Result<()> { /// Change membership from {0,1} to {1,2,3}, keep node-0 as learner; /// /// - The leader should NOT step down after joint log is committed. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn remove_leader_and_convert_to_learner() -> Result<()> { let config = Arc::new( Config { @@ -175,7 +177,8 @@ async fn remove_leader_and_convert_to_learner() -> Result<()> { /// Change membership from {0,1,2} to {2}. Access {2} at once. /// /// It should not respond a ForwardToLeader error that pointing to the removed leader. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn remove_leader_access_new_cluster() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/membership/t31_removed_follower.rs b/tests/tests/membership/t31_removed_follower.rs index d905b8111..877b5c40d 100644 --- a/tests/tests/membership/t31_removed_follower.rs +++ b/tests/tests/membership/t31_removed_follower.rs @@ -6,11 +6,12 @@ use maplit::btreeset; use openraft::Config; use openraft::LogIdOptionExt; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Replication should stop after a follower is removed from membership. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn stop_replication_to_removed_follower() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/membership/t51_remove_unreachable_follower.rs b/tests/tests/membership/t51_remove_unreachable_follower.rs index 17a3582e3..e51f528c9 100644 --- a/tests/tests/membership/t51_remove_unreachable_follower.rs +++ b/tests/tests/membership/t51_remove_unreachable_follower.rs @@ -6,11 +6,12 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Replication should stop after a **unreachable** follower is removed from membership. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn stop_replication_to_removed_unreachable_follower_network_failure() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/membership/t99_issue_471_adding_learner_uses_uninit_leader_id.rs b/tests/tests/membership/t99_issue_471_adding_learner_uses_uninit_leader_id.rs index 34e056afc..da2750136 100644 --- a/tests/tests/membership/t99_issue_471_adding_learner_uses_uninit_leader_id.rs +++ b/tests/tests/membership/t99_issue_471_adding_learner_uses_uninit_leader_id.rs @@ -4,7 +4,7 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When adding learner and waiting for the learner to become up to date, @@ -14,7 +14,8 @@ use crate::fixtures::RaftRouter; /// - Brings up 1 leader. /// - Add learner at once. /// - It should not panic. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn adding_learner_do_not_use_matched_leader_id() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/membership/t99_issue_584_replication_state_reverted.rs b/tests/tests/membership/t99_issue_584_replication_state_reverted.rs index 4bb89bea2..e7b93c2a4 100644 --- a/tests/tests/membership/t99_issue_584_replication_state_reverted.rs +++ b/tests/tests/membership/t99_issue_584_replication_state_reverted.rs @@ -5,10 +5,11 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn t99_issue_584_replication_state_reverted() -> Result<()> { // - Add a learner and replicate all logs to it. // - Add the learner as a voter. When membership changes, openraft internally restarts all diff --git a/tests/tests/membership/t99_new_leader_auto_commit_uniform_config.rs b/tests/tests/membership/t99_new_leader_auto_commit_uniform_config.rs index 5e3733bbd..78a19df30 100644 --- a/tests/tests/membership/t99_new_leader_auto_commit_uniform_config.rs +++ b/tests/tests/membership/t99_new_leader_auto_commit_uniform_config.rs @@ -10,7 +10,7 @@ use openraft::EntryPayload; use openraft::Membership; use openraft::Raft; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster members_leader_fix_partial test. @@ -21,7 +21,8 @@ use crate::fixtures::RaftRouter; /// - manually append a joint config log. /// - shutdown and restart, it should NOT add another final config log to complete the partial /// membership changing -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn new_leader_auto_commit_uniform_config() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/metrics/t10_current_leader.rs b/tests/tests/metrics/t10_current_leader.rs index f67014134..d40bb622d 100644 --- a/tests/tests/metrics/t10_current_leader.rs +++ b/tests/tests/metrics/t10_current_leader.rs @@ -4,7 +4,7 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Current leader tests. @@ -13,7 +13,8 @@ use crate::fixtures::RaftRouter; /// /// - create a stable 3-node cluster. /// - call the current_leader interface on the all nodes, and assert success. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn current_leader() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/metrics/t10_leader_last_ack.rs b/tests/tests/metrics/t10_leader_last_ack.rs index e2c55bc09..33c0b0eb5 100644 --- a/tests/tests/metrics/t10_leader_last_ack.rs +++ b/tests/tests/metrics/t10_leader_last_ack.rs @@ -7,12 +7,13 @@ use openraft::type_config::TypeConfigExt; use openraft::Config; use openraft_memstore::TypeConfig; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Get the last timestamp when a leader is acknowledged by a quorum, /// from RaftMetrics and RaftServerMetrics. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn leader_last_ack_3_nodes() -> Result<()> { let heartbeat_interval = 50; // ms let config = Arc::new( @@ -111,7 +112,8 @@ async fn leader_last_ack_3_nodes() -> Result<()> { /// Get the last timestamp when a leader is acknowledged by a quorum, /// from RaftMetrics and RaftServerMetrics. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn leader_last_ack_1_node() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/metrics/t10_purged.rs b/tests/tests/metrics/t10_purged.rs index defa83340..b4632c572 100644 --- a/tests/tests/metrics/t10_purged.rs +++ b/tests/tests/metrics/t10_purged.rs @@ -7,11 +7,12 @@ use openraft::storage::RaftLogStorage; use openraft::testing::log_id; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Metric `purged` should be the last purged log id. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn metrics_purged() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs index f4d13f08c..3e391aaaa 100644 --- a/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs +++ b/tests/tests/metrics/t10_server_metrics_and_data_metrics.rs @@ -11,11 +11,12 @@ use pretty_assertions::assert_eq; #[allow(unused_imports)] use pretty_assertions::assert_ne; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Server metrics and data metrics method should work. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn server_metrics_and_data_metrics() -> Result<()> { // Setup test dependencies. let config = Arc::new( @@ -67,7 +68,8 @@ async fn server_metrics_and_data_metrics() -> Result<()> { } /// Test if heartbeat metrics work -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn heartbeat_metrics() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/metrics/t20_metrics_state_machine_consistency.rs b/tests/tests/metrics/t20_metrics_state_machine_consistency.rs index 094d2755b..ee74c6794 100644 --- a/tests/tests/metrics/t20_metrics_state_machine_consistency.rs +++ b/tests/tests/metrics/t20_metrics_state_machine_consistency.rs @@ -6,7 +6,7 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster metrics_state_machine_consistency test. @@ -17,7 +17,8 @@ use crate::fixtures::RaftRouter; /// - write one log to the leader. /// - asserts that when metrics.last_applied is upto date, the state machine should be upto date /// too. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn metrics_state_machine_consistency() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/metrics/t30_leader_metrics.rs b/tests/tests/metrics/t30_leader_metrics.rs index 9023f8026..151499298 100644 --- a/tests/tests/metrics/t30_leader_metrics.rs +++ b/tests/tests/metrics/t30_leader_metrics.rs @@ -15,7 +15,7 @@ use pretty_assertions::assert_eq; use pretty_assertions::assert_ne; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Cluster leader_metrics test. @@ -26,7 +26,8 @@ use crate::fixtures::RaftRouter; /// successfully replicated the payload. /// - remove one follower: node-4 /// - asserts node-4 becomes learner and the leader stops sending logs to it. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn leader_metrics() -> Result<()> { let c01234 = btreeset![0, 1, 2, 3, 4]; let c0123 = btreeset![0, 1, 2, 3]; diff --git a/tests/tests/metrics/t40_metrics_wait.rs b/tests/tests/metrics/t40_metrics_wait.rs index cc05548d1..17eccdaae 100644 --- a/tests/tests/metrics/t40_metrics_wait.rs +++ b/tests/tests/metrics/t40_metrics_wait.rs @@ -7,7 +7,7 @@ use openraft::metrics::WaitError; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test wait() utils @@ -17,7 +17,8 @@ use crate::fixtures::RaftRouter; /// - brings 1 nodes online: /// - wait for expected state. /// - wait for invalid state and expect a timeout error. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn metrics_wait() -> Result<()> { // Setup test dependencies. let config = Arc::new( diff --git a/tests/tests/replication/t10_append_entries_partial_success.rs b/tests/tests/replication/t10_append_entries_partial_success.rs index 5f316c094..66c4de33c 100644 --- a/tests/tests/replication/t10_append_entries_partial_success.rs +++ b/tests/tests/replication/t10_append_entries_partial_success.rs @@ -5,13 +5,14 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// RaftNetwork::append_entries can return a partial success. /// For example, it tries to send log entries `[1-2..2-10]`, the application is allowed to send just /// `[1-2..1-3]` and return `PartialSuccess(1-3)`. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_entries_partial_success() -> Result<()> { let config = Arc::new(Config { ..Default::default() }.validate()?); diff --git a/tests/tests/replication/t50_append_entries_backoff.rs b/tests/tests/replication/t50_append_entries_backoff.rs index 5f6a4b503..b7a72da93 100644 --- a/tests/tests/replication/t50_append_entries_backoff.rs +++ b/tests/tests/replication/t50_append_entries_backoff.rs @@ -9,11 +9,12 @@ use openraft::error::Unreachable; use openraft::Config; use openraft::RPCTypes; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Append-entries should backoff when a `Unreachable` error is found. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_entries_backoff() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/replication/t50_append_entries_backoff_rejoin.rs b/tests/tests/replication/t50_append_entries_backoff_rejoin.rs index e5ee4f6b0..796dc3f54 100644 --- a/tests/tests/replication/t50_append_entries_backoff_rejoin.rs +++ b/tests/tests/replication/t50_append_entries_backoff_rejoin.rs @@ -6,14 +6,15 @@ use maplit::btreeset; use openraft::Config; use openraft::ServerState; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// A restarted unreachable node should recover correctly, and catch up with the leader: /// - Setup cluster {0,1,2}, Shutdown leader node-0; /// - Elect node-1, write some logs; /// - Restart node-0, it should receive append-entries from node-1 and catch up. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_entries_backoff_rejoin() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/replication/t51_append_entries_too_large.rs b/tests/tests/replication/t51_append_entries_too_large.rs index 8f4120dbb..4ad21a97d 100644 --- a/tests/tests/replication/t51_append_entries_too_large.rs +++ b/tests/tests/replication/t51_append_entries_too_large.rs @@ -10,14 +10,15 @@ use openraft::raft::AppendEntriesRequest; use openraft::Config; use openraft::RPCTypes; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// If append-entries returns PayloadTooLarge, Openraft should split the request into smaller /// chunks. /// In this test, RaftNetwork::append_entries() returns PayloadTooLarge if the number of entries is /// greater than 1. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn append_entries_too_large() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/replication/t60_feature_loosen_follower_log_revert.rs b/tests/tests/replication/t60_feature_loosen_follower_log_revert.rs index c0f9e51c5..02d9c6853 100644 --- a/tests/tests/replication/t60_feature_loosen_follower_log_revert.rs +++ b/tests/tests/replication/t60_feature_loosen_follower_log_revert.rs @@ -5,12 +5,13 @@ use anyhow::Result; use maplit::btreeset; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// With "--features loosen-follower-log-revert", the leader allows follower to revert its log to an /// earlier state. -#[async_entry::test(worker_threads = 4, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn feature_loosen_follower_log_revert() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/snapshot_building/t10_build_snapshot.rs b/tests/tests/snapshot_building/t10_build_snapshot.rs index ca5a9b7d2..4b69a315f 100644 --- a/tests/tests/snapshot_building/t10_build_snapshot.rs +++ b/tests/tests/snapshot_building/t10_build_snapshot.rs @@ -19,7 +19,7 @@ use openraft::RaftLogReader; use openraft::SnapshotPolicy; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Compaction test. @@ -29,7 +29,8 @@ use crate::fixtures::RaftRouter; /// - build a stable single node cluster. /// - send enough requests to the node that log compaction will be triggered. /// - add new nodes and assert that they receive the snapshot. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn build_snapshot() -> Result<()> { let snapshot_threshold: u64 = 50; diff --git a/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_append.rs b/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_append.rs index 8d145a174..c6301c21e 100644 --- a/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_append.rs +++ b/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_append.rs @@ -13,11 +13,12 @@ use openraft::Config; use openraft::Vote; use openraft_memstore::BlockOperation; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When building a snapshot, append-entries request should not be blocked. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn building_snapshot_does_not_block_append() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_apply.rs b/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_apply.rs index 1ddb5fbe9..2acc1fac6 100644 --- a/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_apply.rs +++ b/tests/tests/snapshot_building/t35_building_snapshot_does_not_block_apply.rs @@ -13,13 +13,14 @@ use openraft::Config; use openraft::Vote; use openraft_memstore::BlockOperation; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When building a snapshot, applying-entries request should not be blocked. /// /// Issue: https://github.com/datafuselabs/openraft/issues/596 -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn building_snapshot_does_not_block_apply() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/snapshot_building/t60_snapshot_policy_never.rs b/tests/tests/snapshot_building/t60_snapshot_policy_never.rs index 34a51b195..4e4aa065e 100644 --- a/tests/tests/snapshot_building/t60_snapshot_policy_never.rs +++ b/tests/tests/snapshot_building/t60_snapshot_policy_never.rs @@ -7,7 +7,7 @@ use maplit::btreeset; use openraft::Config; use openraft::SnapshotPolicy; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Compaction test. @@ -17,7 +17,8 @@ use crate::fixtures::RaftRouter; /// - build a stable single node cluster. /// - send enough requests to the node that log compaction will be triggered. /// - add new nodes and assert that they receive the snapshot. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_policy_never() -> Result<()> { let n_logs: u64 = 6000; let default_config = Config::default().snapshot_policy; diff --git a/tests/tests/snapshot_streaming/t10_api_install_snapshot.rs b/tests/tests/snapshot_streaming/t10_api_install_snapshot.rs index 0d1c956e4..9a53378f1 100644 --- a/tests/tests/snapshot_streaming/t10_api_install_snapshot.rs +++ b/tests/tests/snapshot_streaming/t10_api_install_snapshot.rs @@ -8,7 +8,7 @@ use openraft::Config; use openraft::SnapshotMeta; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// API test: install_snapshot with various arguments. @@ -17,7 +17,8 @@ use crate::fixtures::RaftRouter; /// /// - build a stable single node cluster. /// - send install_snapshot request with matched/mismatched id and offset -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_arguments() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/snapshot_streaming/t10_api_install_snapshot_with_lower_vote.rs b/tests/tests/snapshot_streaming/t10_api_install_snapshot_with_lower_vote.rs index f3158cd9a..6de507563 100644 --- a/tests/tests/snapshot_streaming/t10_api_install_snapshot_with_lower_vote.rs +++ b/tests/tests/snapshot_streaming/t10_api_install_snapshot_with_lower_vote.rs @@ -11,11 +11,12 @@ use openraft::Snapshot; use openraft::SnapshotMeta; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// API test: install_snapshot with vote lower than the target node. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn install_snapshot_lower_vote() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/snapshot_streaming/t20_startup_snapshot.rs b/tests/tests/snapshot_streaming/t20_startup_snapshot.rs index 9c406f606..c18c6a45b 100644 --- a/tests/tests/snapshot_streaming/t20_startup_snapshot.rs +++ b/tests/tests/snapshot_streaming/t20_startup_snapshot.rs @@ -7,12 +7,13 @@ use openraft::storage::RaftStateMachine; use openraft::testing::log_id; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When startup, if there is no snapshot and there are logs purged, it should build a snapshot at /// once. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn startup_build_snapshot() -> anyhow::Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/snapshot_streaming/t30_purge_in_snapshot_logs.rs b/tests/tests/snapshot_streaming/t30_purge_in_snapshot_logs.rs index 35560b72f..104161b03 100644 --- a/tests/tests/snapshot_streaming/t30_purge_in_snapshot_logs.rs +++ b/tests/tests/snapshot_streaming/t30_purge_in_snapshot_logs.rs @@ -9,13 +9,14 @@ use openraft::LogId; use openraft::RaftLogReader; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Leader logs should be deleted upto snapshot.last_log_id-max_in_snapshot_log_to_keep after /// building snapshot; Follower/learner should delete upto snapshot.last_log_id after installing /// snapshot. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn purge_in_snapshot_logs() -> Result<()> { let max_keep = 2; diff --git a/tests/tests/snapshot_streaming/t31_snapshot_overrides_membership.rs b/tests/tests/snapshot_streaming/t31_snapshot_overrides_membership.rs index 64aef9abc..bbe820f8d 100644 --- a/tests/tests/snapshot_streaming/t31_snapshot_overrides_membership.rs +++ b/tests/tests/snapshot_streaming/t31_snapshot_overrides_membership.rs @@ -19,7 +19,7 @@ use openraft::Membership; use openraft::SnapshotPolicy; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test membership info is sync correctly along with snapshot. @@ -29,7 +29,8 @@ use crate::fixtures::RaftRouter; /// - build a stable single node cluster. /// - send enough requests to the node that log compaction will be triggered. /// - ensure that snapshot overrides the existent membership on the learner. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_overrides_membership() -> Result<()> { let snapshot_threshold: u64 = 10; diff --git a/tests/tests/snapshot_streaming/t32_snapshot_uses_prev_snap_membership.rs b/tests/tests/snapshot_streaming/t32_snapshot_uses_prev_snap_membership.rs index cb17346a8..9fc9afc2e 100644 --- a/tests/tests/snapshot_streaming/t32_snapshot_uses_prev_snap_membership.rs +++ b/tests/tests/snapshot_streaming/t32_snapshot_uses_prev_snap_membership.rs @@ -11,7 +11,7 @@ use openraft::RaftLogReader; use openraft::SnapshotPolicy; use openraft::StorageHelper; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test a second compaction should not lose membership. @@ -27,7 +27,8 @@ use crate::fixtures::RaftRouter; /// - send just enough request to trigger another snapshot. /// - ensure membership is still valid. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_uses_prev_snap_membership() -> Result<()> { let snapshot_threshold: u64 = 10; diff --git a/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs b/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs index e910fe01d..f2d0b1c2b 100644 --- a/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs +++ b/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs @@ -27,7 +27,7 @@ use openraft::SnapshotPolicy; use openraft::StorageHelper; use openraft::Vote; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Installing snapshot on a node that has logs conflict with snapshot.meta.last_log_id will delete @@ -37,7 +37,8 @@ use crate::fixtures::RaftRouter; /// - Init node-1 with conflicting log. /// - Send snapshot to node-1 to override its conflicting logs. /// - ensure that snapshot overrides the existent membership and conflicting logs are deleted. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_delete_conflicting_logs() -> Result<()> { let snapshot_threshold: u64 = 10; diff --git a/tests/tests/snapshot_streaming/t34_replication_does_not_block_purge.rs b/tests/tests/snapshot_streaming/t34_replication_does_not_block_purge.rs index 483d03f7f..df2b1b157 100644 --- a/tests/tests/snapshot_streaming/t34_replication_does_not_block_purge.rs +++ b/tests/tests/snapshot_streaming/t34_replication_does_not_block_purge.rs @@ -9,7 +9,7 @@ use openraft::LogId; use openraft::RaftLogReader; use tokio::time::sleep; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Replication blocks purge, but it should not purge for ever. @@ -20,7 +20,8 @@ use crate::fixtures::RaftRouter; /// - Trigger snapshot on the leader, logs should be able to be purged. Because replication should /// avoid using a log id `i` that is `RaftState.last_purged_log_id() <= i <= /// RaftState.purge_upto()`. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn replication_does_not_block_purge() -> Result<()> { let max_keep = 2; diff --git a/tests/tests/snapshot_streaming/t50_snapshot_line_rate_to_snapshot.rs b/tests/tests/snapshot_streaming/t50_snapshot_line_rate_to_snapshot.rs index d520aa07b..8c5fd351b 100644 --- a/tests/tests/snapshot_streaming/t50_snapshot_line_rate_to_snapshot.rs +++ b/tests/tests/snapshot_streaming/t50_snapshot_line_rate_to_snapshot.rs @@ -8,7 +8,7 @@ use openraft::Config; use openraft::LogId; use openraft::SnapshotPolicy; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test replication when switching line rate to snapshotting. @@ -22,7 +22,8 @@ use crate::fixtures::RaftRouter; /// - restore replication. /// - ensure that replication is switched from line-rate mode to snapshotting mode, on absence of /// logs. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_line_rate_to_snapshot() -> Result<()> { let snapshot_threshold: u64 = 10; diff --git a/tests/tests/snapshot_streaming/t50_snapshot_when_lacking_log.rs b/tests/tests/snapshot_streaming/t50_snapshot_when_lacking_log.rs index 03e1c1257..d0bd293c2 100644 --- a/tests/tests/snapshot_streaming/t50_snapshot_when_lacking_log.rs +++ b/tests/tests/snapshot_streaming/t50_snapshot_when_lacking_log.rs @@ -7,7 +7,7 @@ use openraft::Config; use openraft::LogId; use openraft::SnapshotPolicy; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// A leader switch to snapshot replication if a log a follower/learner needs but is already purged. @@ -15,7 +15,8 @@ use crate::fixtures::RaftRouter; /// - build a stable single node cluster. /// - send enough requests to the node that log compaction will be triggered. /// - add learner and assert that they receive the snapshot and logs. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn switch_to_snapshot_replication_when_lacking_log() -> Result<()> { let snapshot_threshold: u64 = 20; let log_cnt = snapshot_threshold + 11; diff --git a/tests/tests/snapshot_streaming/t51_after_snapshot_add_learner_and_request_a_log.rs b/tests/tests/snapshot_streaming/t51_after_snapshot_add_learner_and_request_a_log.rs index 329cbb4b8..9a7c02164 100644 --- a/tests/tests/snapshot_streaming/t51_after_snapshot_add_learner_and_request_a_log.rs +++ b/tests/tests/snapshot_streaming/t51_after_snapshot_add_learner_and_request_a_log.rs @@ -9,11 +9,12 @@ use openraft::Config; use openraft::LogId; use openraft::SnapshotPolicy; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test log replication after snapshot replication. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn after_snapshot_add_learner_and_request_a_log() -> Result<()> { let snapshot_threshold: u64 = 10; diff --git a/tests/tests/snapshot_streaming/t60_snapshot_chunk_size.rs b/tests/tests/snapshot_streaming/t60_snapshot_chunk_size.rs index aa86bdbd7..069cb0ddb 100644 --- a/tests/tests/snapshot_streaming/t60_snapshot_chunk_size.rs +++ b/tests/tests/snapshot_streaming/t60_snapshot_chunk_size.rs @@ -9,7 +9,7 @@ use openraft::LogId; use openraft::ServerState; use openraft::SnapshotPolicy; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// Test transfer snapshot in small chnuks @@ -19,7 +19,8 @@ use crate::fixtures::RaftRouter; /// - build a stable single node cluster. /// - send enough requests to the node that log compaction will be triggered. /// - add learner and assert that they receive the snapshot and logs. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_chunk_size() -> Result<()> { let snapshot_threshold: u64 = 10; diff --git a/tests/tests/snapshot_streaming/t90_issue_808_snapshot_to_unreachable_node_should_not_block.rs b/tests/tests/snapshot_streaming/t90_issue_808_snapshot_to_unreachable_node_should_not_block.rs index 83a81e7be..a9d4294f2 100644 --- a/tests/tests/snapshot_streaming/t90_issue_808_snapshot_to_unreachable_node_should_not_block.rs +++ b/tests/tests/snapshot_streaming/t90_issue_808_snapshot_to_unreachable_node_should_not_block.rs @@ -6,11 +6,12 @@ use maplit::btreeset; use openraft::testing::log_id; use openraft::Config; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// When transferring snapshot to unreachable node, it should not block for ever. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn snapshot_to_unreachable_node_should_not_block() -> Result<()> { let config = Arc::new( Config { diff --git a/tests/tests/state_machine/t10_total_order_apply.rs b/tests/tests/state_machine/t10_total_order_apply.rs index de6e40149..97a784700 100644 --- a/tests/tests/state_machine/t10_total_order_apply.rs +++ b/tests/tests/state_machine/t10_total_order_apply.rs @@ -9,11 +9,12 @@ use openraft::LogIdOptionExt; use openraft::ServerState; use tokio::sync::watch; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// The logs have to be applied in log index order. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] #[ignore] async fn total_order_apply() -> Result<()> { let config = Arc::new( diff --git a/tests/tests/state_machine/t20_state_machine_apply_membership.rs b/tests/tests/state_machine/t20_state_machine_apply_membership.rs index dc129ba26..ee336f79a 100644 --- a/tests/tests/state_machine/t20_state_machine_apply_membership.rs +++ b/tests/tests/state_machine/t20_state_machine_apply_membership.rs @@ -10,7 +10,7 @@ use openraft::LogIdOptionExt; use openraft::Membership; use openraft::StoredMembership; -use crate::fixtures::init_default_ut_tracing; +use crate::fixtures::ut_harness; use crate::fixtures::RaftRouter; /// All log should be applied to state machine. @@ -20,7 +20,8 @@ use crate::fixtures::RaftRouter; /// - bring a cluster with 3 voter and 2 learner. /// - check last_membership in state machine. -#[async_entry::test(worker_threads = 8, init = "init_default_ut_tracing()", tracing_span = "debug")] +#[tracing::instrument] +#[test_harness::test(harness = ut_harness)] async fn state_machine_apply_membership() -> Result<()> { let config = Arc::new( Config {