From 1cf32f7c08faf05250d90876fca5326f5e86d465 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Wed, 15 Nov 2023 10:26:33 +0100 Subject: [PATCH 01/12] Add util for starting randomized network test --- Cargo.lock | 4 +- crates/core/Cargo.toml | 2 +- crates/core/src/client_events.rs | 341 +++++++++++++----- crates/core/src/config.rs | 26 +- crates/core/src/contract.rs | 2 - crates/core/src/contract/executor.rs | 2 - crates/core/src/contract/in_memory.rs | 6 +- crates/core/src/lib.rs | 2 +- crates/core/src/message.rs | 15 +- crates/core/src/node.rs | 13 +- crates/core/src/node/in_memory_impl.rs | 57 ++- crates/core/src/node/network_bridge.rs | 1 - .../src/node/network_bridge/p2p_protoc.rs | 15 + crates/core/src/node/network_event_log.rs | 10 +- crates/core/src/node/p2p_impl.rs | 14 +- crates/core/src/node/tests.rs | 86 +++-- crates/core/src/operations/get.rs | 6 - crates/core/src/operations/put.rs | 3 - crates/core/src/operations/subscribe.rs | 2 - crates/core/src/ring.rs | 2 +- crates/fdev/Cargo.toml | 5 +- crates/fdev/src/build.rs | 47 +-- crates/fdev/src/config.rs | 21 +- crates/fdev/src/inspect.rs | 6 +- crates/fdev/src/main.rs | 8 +- crates/fdev/src/new_package.rs | 8 +- crates/fdev/src/testing.rs | 149 ++++++++ crates/fdev/src/util.rs | 2 +- .../src/{local_node.rs => wasm_runtime.rs} | 6 +- .../{local_node => wasm_runtime}/commands.rs | 4 +- .../src/{local_node => wasm_runtime}/state.rs | 10 +- .../user_events.rs | 8 +- modules/antiflood-tokens/Cargo.lock | 223 ++++++------ 33 files changed, 765 insertions(+), 341 deletions(-) create mode 100644 crates/fdev/src/testing.rs rename crates/fdev/src/{local_node.rs => wasm_runtime.rs} (92%) rename crates/fdev/src/{local_node => wasm_runtime}/commands.rs (97%) rename crates/fdev/src/{local_node => wasm_runtime}/state.rs (88%) rename crates/fdev/src/{local_node => wasm_runtime}/user_events.rs (98%) diff --git a/Cargo.lock b/Cargo.lock index c41ac3fbc..2d0f58e57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1450,7 +1450,7 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fdev" -version = "0.0.6" +version = "0.0.7" dependencies = [ "anyhow", "bincode", @@ -1539,7 +1539,7 @@ dependencies = [ [[package]] name = "freenet" -version = "0.0.6" +version = "0.0.7" dependencies = [ "anyhow", "arbitrary", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 2ff32ec94..b3dd06990 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "freenet" -version = "0.0.6" +version = "0.0.7" edition = "2021" rust-version = "1.71.1" publish = true diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index 3e2a61170..f62fbc113 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -150,59 +150,85 @@ pub trait ClientEventsProxy { ) -> BoxFuture>; } -#[cfg(test)] pub(crate) mod test { - // FIXME: remove unused - #![allow(unused)] - - use std::sync::Arc; - use std::{collections::HashMap, time::Duration}; - - use freenet_stdlib::client_api::ContractRequest; - use freenet_stdlib::prelude::*; + use std::{ + collections::{HashMap, HashSet}, + time::Duration, + }; + + use freenet_stdlib::{ + client_api::{ContractRequest, ErrorKind}, + prelude::*, + }; use futures::FutureExt; - use rand::{prelude::Rng, thread_rng}; + use rand::{seq::SliceRandom, SeedableRng}; use tokio::sync::watch::Receiver; use crate::node::{tests::EventId, PeerKey}; use super::*; - pub(crate) struct MemoryEventsGen { + pub(crate) struct MemoryEventsGen { id: PeerKey, signal: Receiver<(EventId, PeerKey)>, - non_owned_contracts: Vec, - owned_contracts: Vec<(ContractContainer, WrappedState)>, events_to_gen: HashMap>, - random: bool, + rng: Option, + internal_state: Option, } - impl MemoryEventsGen { - pub fn new(signal: Receiver<(EventId, PeerKey)>, id: PeerKey) -> Self { + impl MemoryEventsGen + where + R: RandomEventGenerator, + { + pub fn new_with_seed(signal: Receiver<(EventId, PeerKey)>, id: PeerKey, seed: u64) -> Self { Self { signal, id, - non_owned_contracts: Vec::new(), - owned_contracts: Vec::new(), events_to_gen: HashMap::new(), - random: false, + rng: Some(R::seed_from_u64(seed)), + internal_state: None, } } - /// Contracts that are available in the network to be requested. - pub fn request_contracts(&mut self, contracts: impl IntoIterator) { - self.non_owned_contracts.extend(contracts) + pub fn rng_params(&mut self, num_peers: usize, max_contract_num: usize, iterations: usize) { + // let hasher = &mut rand::thread_rng(); + // let hash = self.id.hash(hasher); + // let this_peer = hasher.finish(); + let internal_state = InternalGeneratorState { + this_peer: 0, + num_peers, + max_contract_num, + max_iterations: iterations, + ..Default::default() + }; + self.internal_state = Some(internal_state); } - /// Contracts that the user updates. - pub fn has_contract( - &mut self, - contracts: impl IntoIterator, - ) { - self.owned_contracts.extend(contracts); + fn generate_rand_event(&mut self) -> Option> { + let (rng, state) = self + .rng + .as_mut() + .zip(self.internal_state.as_mut()) + .expect("rng should be set"); + rng.gen_event(state) + } + } + + impl MemoryEventsGen { + #[cfg(test)] + pub fn new(signal: Receiver<(EventId, PeerKey)>, id: PeerKey) -> Self { + Self { + signal, + id, + events_to_gen: HashMap::new(), + rng: None, + internal_state: None, + } } + } - /// Events that the user generate. + impl MemoryEventsGen { + #[cfg(test)] pub fn generate_events( &mut self, events: impl IntoIterator)>, @@ -213,77 +239,23 @@ pub(crate) mod test { fn generate_deterministic_event(&mut self, id: &EventId) -> Option { self.events_to_gen.remove(id) } - - fn generate_rand_event(&mut self) -> ClientRequest<'static> { - let mut rng = thread_rng(); - loop { - match rng.gen_range(0u8..3) { - 0 if !self.owned_contracts.is_empty() => { - let contract_no = rng.gen_range(0..self.owned_contracts.len()); - let (contract, state) = self.owned_contracts[contract_no].clone(); - break ContractRequest::Put { - contract, - state, - related_contracts: Default::default(), - } - .into(); - } - 1 if !self.non_owned_contracts.is_empty() => { - let contract_no = rng.gen_range(0..self.non_owned_contracts.len()); - let key = self.non_owned_contracts[contract_no].clone(); - break ContractRequest::Get { - key, - fetch_contract: rng.gen_bool(0.5), - } - .into(); - } - 2 if !self.non_owned_contracts.is_empty() - || !self.owned_contracts.is_empty() => - { - let get_owned = match ( - self.non_owned_contracts.is_empty(), - self.owned_contracts.is_empty(), - ) { - (false, false) => rng.gen_bool(0.5), - (false, true) => false, - (true, false) => true, - _ => unreachable!(), - }; - let key = if get_owned { - let contract_no = rng.gen_range(0..self.owned_contracts.len()); - self.owned_contracts[contract_no].0.clone().key() - } else { - // let contract_no = rng.gen_range(0..self.non_owned_contracts.len()); - // self.non_owned_contracts[contract_no] - todo!() - }; - break ContractRequest::Subscribe { key, summary: None }.into(); - } - 0 => {} - 1 => {} - 2 => { - let msg = "the joint set of owned and non-owned contracts is empty!"; - tracing::error!("{}", msg); - panic!("{}", msg) - } - _ => unreachable!(), - } - } - } } - impl ClientEventsProxy for MemoryEventsGen { + impl ClientEventsProxy for MemoryEventsGen + where + R: RandomEventGenerator + Send, + { fn recv(&mut self) -> BoxFuture<'_, Result, ClientError>> { async { loop { if self.signal.changed().await.is_ok() { let (ev_id, pk) = *self.signal.borrow(); - if pk == self.id && !self.random { + if self.rng.is_some() && pk == self.id { let res = OpenRequest { client_id: ClientId::FIRST, request: self - .generate_deterministic_event(&ev_id) - .expect("event not found") + .generate_rand_event() + .ok_or_else(|| ClientError::from(ErrorKind::Disconnect))? .into(), notification_channel: None, token: None, @@ -292,7 +264,10 @@ pub(crate) mod test { } else if pk == self.id { let res = OpenRequest { client_id: ClientId::FIRST, - request: self.generate_rand_event().into(), + request: self + .generate_deterministic_event(&ev_id) + .expect("event not found") + .into(), notification_channel: None, token: None, }; @@ -316,4 +291,188 @@ pub(crate) mod test { async { Ok(()) }.boxed() } } + + #[derive(Default)] + pub struct InternalGeneratorState { + this_peer: usize, + num_peers: usize, + current_iteration: usize, + max_iterations: usize, + max_contract_num: usize, + owns_contracts: HashSet, + subcribed_contract: HashSet, + existing_contracts: Vec, + } + + pub trait RandomEventGenerator { + fn seed_from_u64(seed: u64) -> Self; + + fn gen_u8(&mut self) -> u8; + + fn gen_range(&mut self, range: std::ops::Range) -> usize; + + fn choose<'a, T>(&mut self, vec: &'a [T]) -> Option<&'a T>; + + /// The goal of this function is to generate a random event that is valid for the current + /// global state of the network. + /// + /// In order to do this all peers must replicate the same exact events so they are aware + /// of the current global state (basically which contracts have been created so far). + /// + /// To guarantee this make sure that calls to this rng are always executed in the same order + /// at all peers. + fn gen_event( + &mut self, + state: &mut InternalGeneratorState, + ) -> Option> { + while state.current_iteration < state.max_iterations { + state.current_iteration += 1; + let for_this_peer = self.gen_range(0..state.num_peers) == state.this_peer; + match self.gen_range(0..4) { + 0 => { + if state.max_contract_num <= state.existing_contracts.len() { + continue; + } + let contract = self.gen_contract_container(); + let request = ContractRequest::Put { + contract: contract.clone(), + state: WrappedState::new(self.random_byte_vec()), + related_contracts: RelatedContracts::new(), + }; + let key = contract.key(); + state.existing_contracts.push(contract); + if for_this_peer { + state.owns_contracts.insert(key); + return Some(request.into()); + } + } + 1 => { + if let Some(contract) = self.choose(&state.existing_contracts) { + let delta = UpdateData::Delta(StateDelta::from(self.random_byte_vec())); + if !for_this_peer { + continue; + } + let request = ContractRequest::Update { + key: contract.key().clone(), + data: delta, + }; + if state.owns_contracts.contains(&contract.key()) { + return Some(request.into()); + } + } + } + 2 => { + if let Some(contract) = self.choose(&state.existing_contracts) { + if !for_this_peer { + continue; + } + let key = contract.key(); + let fetch_contract = state.owns_contracts.contains(&key); + let request = ContractRequest::Get { + key, + fetch_contract, + }; + return Some(request.into()); + } + } + 3 => { + if let Some(contract) = self.choose(&state.existing_contracts) { + let key = contract.key(); + let summary = StateSummary::from(self.random_byte_vec()); + if !for_this_peer || state.subcribed_contract.contains(&key) { + continue; + } + let request = ContractRequest::Subscribe { + key, + summary: Some(summary), + }; + return Some(request.into()); + } + } + _ => unreachable!(), + } + } + None + } + + fn gen_contract_container(&mut self) -> ContractContainer { + let code = ContractCode::from(self.random_byte_vec()); + let params = Parameters::from(self.random_byte_vec()); + ContractWasmAPIVersion::V1(WrappedContract::new(code.into(), params)).into() + } + + fn random_byte_vec(&mut self) -> Vec { + (0..self.gen_u8()) + .map(|_| self.gen_u8()) + .collect::>() + } + } + + impl RandomEventGenerator for rand::rngs::SmallRng { + fn gen_u8(&mut self) -> u8 { + ::gen(self) + } + + fn gen_range(&mut self, range: std::ops::Range) -> usize { + ::gen_range(self, range) + } + + fn choose<'a, T>(&mut self, vec: &'a [T]) -> Option<&'a T> { + vec.choose(self) + } + + fn seed_from_u64(seed: u64) -> Self { + ::seed_from_u64(seed) + } + } + + impl RandomEventGenerator for fastrand::Rng { + fn gen_u8(&mut self) -> u8 { + self.u8(..u8::MAX) + } + + fn gen_range(&mut self, range: std::ops::Range) -> usize { + self.choice(range).expect("non empty") + } + + fn choose<'a, T>(&mut self, vec: &'a [T]) -> Option<&'a T> { + self.choice(0..vec.len()).and_then(|choice| vec.get(choice)) + } + + fn seed_from_u64(seed: u64) -> Self { + Self::with_seed(seed) + } + } + + #[test] + fn test_gen_event() { + const NUM_PEERS: usize = 20; + const ITERATIONS: usize = 10_000; + let mut threads = vec![]; + for this_peer in 0..NUM_PEERS { + let thread = std::thread::spawn(move || { + let mut rng = ::seed_from_u64(15_978); + let mut state = InternalGeneratorState { + this_peer, + num_peers: NUM_PEERS, + max_contract_num: 10, + ..Default::default() + }; + for _ in 0..ITERATIONS { + rng.gen_event(&mut state); + } + state + }); + threads.push(thread); + } + let states = threads + .into_iter() + .map(|t| t.join().unwrap()) + .collect::>(); + + let first_state = &states[0]; + for state in &states[1..] { + assert_eq!(first_state.existing_contracts, state.existing_contracts); + } + } } diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 1b691c013..74478077e 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -19,10 +19,25 @@ use tokio::runtime::Runtime; use crate::local_node::OperationMode; +/// Default maximum number of connections for the peer. +pub const DEFAULT_MAX_CONNECTIONS: usize = 20; +/// Default minimum number of connections for the peer. +pub const DEFAULT_MIN_CONNECTIONS: usize = 10; +/// Default threshold for randomizing potential peers for new connections. +/// +/// If the hops left for the operation is above or equal to this threshold +/// (of the total DEFAULT_MAX_HOPS_TO_LIVE), then the next potential peer +/// will be selected randomly. Otherwise the optimal peer will be selected +/// by Freenet custom algorithms. +pub const DEFAULT_RANDOM_PEER_CONN_THRESHOLD: usize = 7; +/// Default maximum number of hops to live for any operation +/// (if it applies, e.g. connect requests). +pub const DEFAULT_MAX_HOPS_TO_LIVE: usize = 10; const DEFAULT_BOOTSTRAP_PORT: u16 = 7800; const DEFAULT_WEBSOCKET_API_PORT: u16 = 55008; static CONFIG: std::sync::OnceLock = std::sync::OnceLock::new(); + pub(crate) const PEER_TIMEOUT: Duration = Duration::from_secs(60); pub(crate) const OPERATION_TTL: Duration = Duration::from_secs(60); @@ -376,8 +391,8 @@ pub(super) mod tracer { let filter_layer = tracing_subscriber::EnvFilter::builder() .with_default_directive(filter.into()) .from_env_lossy() - .add_directive("stretto=off".parse().unwrap()) - .add_directive("sqlx=error".parse().unwrap()); + .add_directive("stretto=off".parse().expect("infallible")) + .add_directive("sqlx=error".parse().expect("infallible")); // use opentelemetry_sdk::propagation::TraceContextPropagator; use tracing_subscriber::layer::SubscriberExt; @@ -393,6 +408,7 @@ pub(super) mod tracer { }; #[cfg(feature = "trace-ot")] { + let disabled_ot_traces = std::env::var("FREENET_DISABLE_TRACES").is_ok(); let identifier = if matches!(Config::node_mode(), OperationMode::Local) { "freenet-core".to_string() } else { @@ -410,10 +426,12 @@ pub(super) mod tracer { // Get a tracer which will route OT spans to a Jaeger agent tracing_opentelemetry::layer().with_tracer(ot_jaeger_tracer) }; - if !disabled_logs { + if !disabled_logs && !disabled_ot_traces { fmt_layer.and_then(tracing_ot_layer).boxed() - } else { + } else if !disabled_ot_traces { tracing_ot_layer.boxed() + } else { + return Ok(()); } } #[cfg(not(feature = "trace-ot"))] diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index 1dc3a69d5..b0a606ba0 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -4,7 +4,6 @@ use freenet_stdlib::prelude::*; mod executor; mod handler; -#[cfg(test)] mod in_memory; pub mod storages; @@ -16,7 +15,6 @@ pub(crate) use handler::{ ContractHandlerChannel, ContractHandlerEvent, EventId, NetworkContractHandler, SenderHalve, StoreResponse, }; -#[cfg(test)] pub(crate) use in_memory::{MemoryContractHandler, MockRuntime}; pub use executor::{Executor, ExecutorError, OperationMode}; diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index ac807827c..f04cd0bc8 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -1196,7 +1196,6 @@ impl Executor { } } -#[cfg(test)] impl Executor { pub async fn new_mock(data_dir: &str) -> Result { let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{data_dir}")); @@ -1267,7 +1266,6 @@ impl ContractExecutor for Executor { } } -#[cfg(test)] #[async_trait::async_trait] impl ContractExecutor for Executor { async fn fetch_contract( diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs index 6ce8d0a3e..83b080186 100644 --- a/crates/core/src/contract/in_memory.rs +++ b/crates/core/src/contract/in_memory.rs @@ -1,8 +1,5 @@ use crate::{client_events::ClientId, runtime::ContractStore}; -use freenet_stdlib::{ - client_api::{ClientError, ClientRequest, HostResponse}, - prelude::WrappedContract, -}; +use freenet_stdlib::client_api::{ClientError, ClientRequest, HostResponse}; use futures::{future::BoxFuture, FutureExt}; use tokio::sync::mpsc::UnboundedSender; @@ -69,6 +66,7 @@ impl ContractHandler for MemoryContractHandler { #[test] fn serialization() -> Result<(), anyhow::Error> { + use freenet_stdlib::prelude::WrappedContract; let bytes = crate::util::test::random_bytes_1kb(); let mut gen = arbitrary::Unstructured::new(&bytes); let contract: WrappedContract = gen.arbitrary()?; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 665f70808..214de36bc 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -27,7 +27,7 @@ pub mod local_node { pub mod network_sim { use super::*; pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; - pub use node::{InitPeerNode, NodeBuilder, NodeConfig}; + pub use node::{tests::SimNetwork, InitPeerNode, NodeBuilder, NodeConfig}; pub use ring::Location; } diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 1c7cb3630..4085170be 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -256,6 +256,9 @@ pub(crate) enum NodeEvent { /// Error while sending a message by the connection bridge from within the ops. #[serde(skip)] Error(ConnectionError), + Disconnect { + cause: Option, + }, } impl Display for NodeEvent { @@ -264,12 +267,18 @@ impl Display for NodeEvent { NodeEvent::ShutdownNode => f.write_str("ShutdownNode"), NodeEvent::ConfirmedInbound => f.write_str("ConfirmedInbound"), NodeEvent::DropConnection(peer) => { - f.write_str(&format!("DropConnection (from {peer})")) + write!(f, "DropConnection (from {peer})") } NodeEvent::AcceptConnection(peer) => { - f.write_str(&format!("AcceptConnection (from {peer})")) + write!(f, "AcceptConnection (from {peer})") + } + NodeEvent::Error(err) => write!(f, "{err}"), + NodeEvent::Disconnect { cause: Some(cause) } => { + write!(f, "Disconnect node, reason: {cause}") + } + NodeEvent::Disconnect { cause: None } => { + write!(f, "Disconnect node, reason: unknown") } - NodeEvent::Error(err) => f.write_str(&format!("{err}")), } } } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 0484d1b41..5797f4372 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -20,7 +20,6 @@ use freenet_stdlib::client_api::{ClientRequest, ContractRequest}; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; use tracing::Instrument; -#[cfg(test)] use self::in_memory_impl::NodeInMemory; use self::{network_event_log::NetEventLog, p2p_impl::NodeP2P}; use crate::{ @@ -31,7 +30,7 @@ use crate::{ ClientResponses, ClientResponsesSender, ContractError, ExecutorToEventLoopChannel, NetworkContractHandler, NetworkEventListenerHalve, OperationMode, }, - message::{Message, Transaction, TransactionType}, + message::{Message, NodeEvent, Transaction, TransactionType}, operations::{ connect::{self, ConnectMsg, ConnectOp}, get, put, subscribe, OpEnum, OpError, OpOutcome, @@ -47,13 +46,11 @@ pub(crate) use network_bridge::{ConnectionError, EventLoopNotificationsSender, N pub(crate) use network_event_log::{EventRegister, NetEventRegister}; pub(crate) use op_state_manager::{OpManager, OpNotAvailable}; -#[cfg(test)] mod in_memory_impl; mod network_bridge; mod network_event_log; mod op_state_manager; mod p2p_impl; -#[cfg(test)] pub(crate) mod tests; #[derive(clap::Parser, Clone, Debug)] @@ -318,6 +315,7 @@ async fn client_event_handling( op_storage: Arc, mut client_events: ClientEv, mut client_responses: ClientResponses, + node_controller: tokio::sync::mpsc::Sender, ) where ClientEv: ClientEventsProxy + Send + 'static, { @@ -334,8 +332,8 @@ async fn client_event_handling( continue; } }; - if let ClientRequest::Disconnect { .. } = &*req.request { - // todo: notify executor of disconnect + if let ClientRequest::Disconnect { cause } = &*req.request { + node_controller.send(NodeEvent::Disconnect { cause: cause.clone() }).await.ok(); continue; } process_open_request(req, op_storage.clone()).await; @@ -728,8 +726,7 @@ impl<'a> arbitrary::Arbitrary<'a> for PeerKey { } impl PeerKey { - #[cfg(test)] - pub fn random() -> Self { + pub(crate) fn random() -> Self { use libp2p::identity::Keypair; PeerKey::from(Keypair::generate_ed25519().public()) } diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index 607f6e3c1..6941fab0a 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; use either::Either; use freenet_stdlib::prelude::*; @@ -15,9 +15,8 @@ use crate::{ client_events::ClientEventsProxy, config::GlobalExecutor, contract::{ - self, executor_channel, ClientResponsesSender, ContractError, ContractHandler, - ContractHandlerEvent, ExecutorToEventLoopChannel, MemoryContractHandler, - NetworkEventListenerHalve, + self, executor_channel, ClientResponsesSender, ContractHandler, ExecutorToEventLoopChannel, + MemoryContractHandler, NetworkEventListenerHalve, }, message::{Message, NodeEvent, TransactionType}, node::NodeBuilder, @@ -42,7 +41,7 @@ pub(super) struct NodeInMemory { impl NodeInMemory { /// Buils an in-memory node. Does nothing upon construction, pub async fn build( - builder: NodeBuilder<1>, + builder: NodeBuilder<0>, event_register: ER, ch_builder: String, add_noise: bool, @@ -114,21 +113,29 @@ impl NodeInMemory { } let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); let parent_span = self.parent_span.clone(); + let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); GlobalExecutor::spawn( - client_event_handling(self.op_storage.clone(), user_events, client_responses) - .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), + client_event_handling( + self.op_storage.clone(), + user_events, + client_responses, + node_controller_tx, + ) + .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), ); - let parent_span = self.parent_span.clone(); - self.run_event_listener(cli_response_sender) + let parent_span: tracing::Span = self.parent_span.clone(); + self.run_event_listener(cli_response_sender, node_controller_rx) .instrument(parent_span) .await } + #[cfg(test)] pub async fn append_contracts<'a>( &self, contracts: Vec<(ContractContainer, WrappedState)>, - contract_subscribers: HashMap>, - ) -> Result<(), ContractError> { + contract_subscribers: std::collections::HashMap>, + ) -> Result<(), crate::contract::ContractError> { + use crate::contract::ContractHandlerEvent; for (contract, state) in contracts { let key: ContractKey = contract.key(); let parameters = contract.params(); @@ -173,15 +180,25 @@ impl NodeInMemory { /// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. async fn run_event_listener( &mut self, - _client_responses: ClientResponsesSender, // fixme: use this + _client_responses: ClientResponsesSender, + mut node_controller_rx: tokio::sync::mpsc::Receiver, ) -> Result<(), anyhow::Error> { loop { let msg = tokio::select! { msg = self.conn_manager.recv() => { msg.map(Either::Left) } - msg = self.notification_channel.recv() => if let Some(msg) = msg { - Ok(msg.map_left(|(msg, _cli_id)| msg)) - } else { - anyhow::bail!("notification channel shutdown, fatal error"); + msg = self.notification_channel.recv() => { + if let Some(msg) = msg { + Ok(msg.map_left(|(msg, _cli_id)| msg)) + } else { + anyhow::bail!("notification channel shutdown, fatal error"); + } + } + msg = node_controller_rx.recv() => { + if let Some(msg) = msg { + Ok(Either::Right(msg)) + } else { + anyhow::bail!("node controller channel shutdown, fatal error"); + } } }; @@ -229,6 +246,14 @@ impl NodeInMemory { self.op_storage.ring.prune_connection(peer); continue; } + NodeEvent::Disconnect { cause: Some(cause) } => { + tracing::info!(peer = %self.peer_key, "Shutting down node, reason: {cause}"); + return Ok(()); + } + NodeEvent::Disconnect { cause: None } => { + tracing::info!(peer = %self.peer_key, "Shutting down node"); + return Ok(()); + } other => { unreachable!("event {other:?}, shouldn't happen in the in-memory impl") } diff --git a/crates/core/src/node/network_bridge.rs b/crates/core/src/node/network_bridge.rs index 8138ecfaa..5b479d4ea 100644 --- a/crates/core/src/node/network_bridge.rs +++ b/crates/core/src/node/network_bridge.rs @@ -13,7 +13,6 @@ use crate::{ message::{Message, NodeEvent}, }; -#[cfg(test)] pub(crate) mod in_memory; pub(crate) mod p2p_protoc; diff --git a/crates/core/src/node/network_bridge/p2p_protoc.rs b/crates/core/src/node/network_bridge/p2p_protoc.rs index d7ccfcd79..977692d6a 100644 --- a/crates/core/src/node/network_bridge/p2p_protoc.rs +++ b/crates/core/src/node/network_bridge/p2p_protoc.rs @@ -317,6 +317,7 @@ impl P2pConnManager { mut notification_channel: EventLoopNotifications, mut executor_channel: ExecutorToEventLoopChannel, cli_response_sender: ClientResponsesSender, + mut node_controller: Receiver, ) -> Result<(), anyhow::Error> { use ConnMngrActions::*; @@ -422,6 +423,13 @@ impl P2pConnManager { msg = network_msg => { msg } msg = notification_msg => { msg } msg = bridge_msg => { msg } + msg = node_controller.recv() => { + if let Some(msg) = msg { + Ok(Right(NodeAction(msg))) + } else { + Ok(Right(ClosedChannel)) + } + } event_id = op_manager.recv_from_handler() => { if let Some((client_id, transaction)) = event_id.client_id().zip(event_id.transaction()) { tx_to_client.insert(transaction, client_id); @@ -524,6 +532,13 @@ impl P2pConnManager { // todo: if we prefilter connections, should only accept ones informed this way // (except 'join ring' requests) } + Ok(Right(NodeAction(NodeEvent::Disconnect { cause }))) => { + match cause { + Some(cause) => tracing::warn!("Shutting down node: {cause}"), + None => tracing::warn!("Shutting down node"), + } + return Ok(()); + } Ok(Right(ConnectionEstablished { address: addr, peer, diff --git a/crates/core/src/node/network_event_log.rs b/crates/core/src/node/network_event_log.rs index f3ae94664..56dd37502 100644 --- a/crates/core/src/node/network_event_log.rs +++ b/crates/core/src/node/network_event_log.rs @@ -27,7 +27,6 @@ use crate::{ #[cfg(feature = "trace-ot")] pub(super) use opentelemetry_tracer::OTEventRegister; -#[cfg(test)] pub(super) use test::TestEventListener; use super::OpManager; @@ -674,6 +673,7 @@ mod opentelemetry_tracer { use opentelemetry::{ global, trace::{self, Span}, + KeyValue, }; use super::*; @@ -705,6 +705,10 @@ mod opentelemetry_tracer { start_time: Some(start_time), span_id: Some(trace::SpanId::from_bytes(span_id)), trace_id: Some(trace::TraceId::from_bytes(tx_bytes)), + attributes: Some(vec![ + KeyValue::new("transaction", transaction.to_string()), + KeyValue::new("tx_type", transaction.transaction_type().description()), + ]), ..Default::default() }); OTSpan { @@ -986,7 +990,6 @@ enum PutEvent { }, } -#[cfg(test)] pub(super) mod test { use std::{ collections::HashMap, @@ -994,7 +997,6 @@ pub(super) mod test { atomic::{AtomicUsize, Ordering::SeqCst}, Arc, }, - time::Duration, }; use dashmap::DashMap; @@ -1007,6 +1009,7 @@ pub(super) mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn event_register_read_write() -> Result<(), DynError> { + use std::time::Duration; let event_log_path = crate::config::Config::conf().event_log(); // truncate the log if it exists std::fs::File::create(event_log_path).unwrap(); @@ -1114,6 +1117,7 @@ pub(super) mod test { } /// The contract was broadcasted from one peer to an other successfully. + #[cfg(test)] pub fn contract_broadcasted(&self, for_key: &ContractKey) -> bool { let logs = self.logs.lock(); let put_broadcast_ops = logs.iter().filter_map(|l| match &l.kind { diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index 8f5d0fa06..4d071e374 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -23,6 +23,7 @@ use crate::{ self, ClientResponsesSender, ContractHandler, ExecutorToEventLoopChannel, NetworkEventListenerHalve, }, + message::NodeEvent, node::NodeBuilder, util::IterExt, }; @@ -37,6 +38,7 @@ pub(super) struct NodeP2P { is_gateway: bool, executor_listener: ExecutorToEventLoopChannel, cli_response_sender: ClientResponsesSender, + node_controller: tokio::sync::mpsc::Receiver, } impl NodeP2P { @@ -68,6 +70,7 @@ impl NodeP2P { self.notification_channel, self.executor_listener, self.cli_response_sender, + self.node_controller, ) .await } @@ -111,9 +114,15 @@ impl NodeP2P { .instrument(tracing::info_span!(parent: parent_span.clone(), "contract_handling")), ); let clients = ClientEventsCombinator::new(builder.clients); + let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); GlobalExecutor::spawn( - client_event_handling(op_storage.clone(), clients, client_responses) - .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), + client_event_handling( + op_storage.clone(), + clients, + client_responses, + node_controller_tx, + ) + .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), ); Ok(NodeP2P { @@ -124,6 +133,7 @@ impl NodeP2P { is_gateway: builder.location.is_some(), executor_listener, cli_response_sender, + node_controller: node_controller_rx, }) } diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index e6c60a5db..20c960171 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -6,18 +6,20 @@ use std::{ time::{Duration, Instant}, }; -use freenet_stdlib::client_api::ClientRequest; use freenet_stdlib::prelude::*; use itertools::Itertools; use libp2p::{identity, PeerId}; use rand::Rng; -use tokio::sync::watch::{channel, Receiver, Sender}; +use tokio::sync::{ + watch::{channel, Receiver, Sender}, + Barrier, +}; use tracing::{info, Instrument}; #[cfg(feature = "trace-ot")] use crate::node::network_event_log::{CombinedRegister, NetEventRegister}; use crate::{ - client_events::test::MemoryEventsGen, + client_events::test::{MemoryEventsGen, RandomEventGenerator}, config::GlobalExecutor, node::{network_event_log::TestEventListener, InitPeerNode, NodeBuilder, NodeInMemory}, ring::{Distance, Location, PeerKeyLocation}, @@ -47,7 +49,7 @@ pub fn get_dynamic_port() -> u16 { pub(crate) type EventId = usize; #[derive(PartialEq, Eq, Hash, Clone)] -pub(crate) struct NodeLabel(Arc); +pub struct NodeLabel(Arc); impl NodeLabel { fn gateway(id: usize) -> Self { @@ -97,12 +99,11 @@ impl<'a> From<&'a str> for NodeLabel { } } +#[cfg(test)] #[derive(Clone)] pub(crate) struct NodeSpecification { - /// Pair of contract and the initial value pub owned_contracts: Vec<(ContractContainer, WrappedState)>, - pub non_owned_contracts: Vec, - pub events_to_generate: HashMap>, + pub events_to_generate: HashMap>, pub contract_subscribers: HashMap>, } @@ -115,11 +116,11 @@ struct GatewayConfig { } /// A simulated in-memory network topology. -pub(crate) struct SimNetwork { +pub struct SimNetwork { name: String, debug: bool, - pub labels: HashMap, - pub event_listener: TestEventListener, + pub(crate) labels: HashMap, + pub(crate) event_listener: TestEventListener, user_ev_controller: Sender<(EventId, PeerKey)>, receiver_ch: Receiver<(EventId, PeerKey)>, gateways: Vec<(NodeInMemory, GatewayConfig)>, @@ -190,10 +191,7 @@ impl SimNetwork { let port = get_free_port().unwrap(); let location = Location::random(); - let mut config = NodeBuilder::new([Box::new(MemoryEventsGen::new( - self.receiver_ch.clone(), - PeerKey::from(id), - ))]); + let mut config = NodeBuilder::new([]); config .with_ip(Ipv6Addr::LOCALHOST) .with_port(port) @@ -260,10 +258,7 @@ impl SimNetwork { let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); - let mut config = NodeBuilder::new([Box::new(MemoryEventsGen::new( - self.receiver_ch.clone(), - PeerKey::from(id), - ))]); + let mut config = NodeBuilder::new([]); for GatewayConfig { port, id, location, .. } in &gateways @@ -311,21 +306,25 @@ impl SimNetwork { } } - pub async fn start(&mut self) { + #[cfg(test)] + pub(crate) async fn start(&mut self) { self.start_with_spec(HashMap::new()).await } - pub async fn start_with_spec(&mut self, mut specs: HashMap) { - let mut gw_not_init = self.gateways.len(); + #[cfg(test)] + pub(crate) async fn start_with_spec( + &mut self, + mut specs: HashMap, + ) { let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { let node_spec = specs.remove(&label); self.initialize_peer(node, label, node_spec); - gw_not_init = gw_not_init.saturating_sub(1); tokio::time::sleep(self.init_backoff).await; } } + #[cfg(test)] fn initialize_peer( &mut self, mut peer: NodeInMemory, @@ -334,8 +333,6 @@ impl SimNetwork { ) { let mut user_events = MemoryEventsGen::new(self.receiver_ch.clone(), peer.peer_key); if let Some(specs) = node_specs.clone() { - user_events.has_contract(specs.owned_contracts); - user_events.request_contracts(specs.non_owned_contracts); user_events.generate_events(specs.events_to_generate); } tracing::debug!(peer = %label, "initializing"); @@ -351,6 +348,47 @@ impl SimNetwork { GlobalExecutor::spawn(node_task); } + pub async fn start_with_rand_gen( + &mut self, + seed: u64, + max_contract_num: usize, + iterations: usize, + ) -> Vec>> + where + R: RandomEventGenerator + Send + 'static, + { + let total_peer_num = self.gateways.len() + self.nodes.len(); + let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); + let mut peers = vec![]; + for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { + let handle = self.initialize_peer_with_rnd_gen::( + node, + label, + (total_peer_num, max_contract_num, seed, iterations), + ); + peers.push(handle); + tokio::time::sleep(self.init_backoff).await; + } + peers + } + + fn initialize_peer_with_rnd_gen( + &mut self, + mut peer: NodeInMemory, + label: NodeLabel, + (total_peer_num, max_contract_num, seed, iterations): (usize, usize, u64, usize), + ) -> tokio::task::JoinHandle> + where + R: RandomEventGenerator + Send + 'static, + { + tracing::debug!(peer = %label, "initializing"); + let mut user_events = + MemoryEventsGen::::new_with_seed(self.receiver_ch.clone(), peer.peer_key, seed); + user_events.rng_params(total_peer_num, max_contract_num, iterations); + let node_task = async move { peer.run_node(user_events).await }; + GlobalExecutor::spawn(node_task) + } + pub fn get_locations_by_node(&self) -> HashMap { let mut locations_by_node: HashMap = HashMap::new(); diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index b5dd36978..f2f3b1d98 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -873,7 +873,6 @@ mod test { .into(); let node_0 = NodeSpecification { owned_contracts: vec![], - non_owned_contracts: vec![key.clone()], events_to_generate: HashMap::from_iter([(1, get_event)]), contract_subscribers: HashMap::new(), }; @@ -883,7 +882,6 @@ mod test { ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, )], - non_owned_contracts: vec![], events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), }; @@ -929,7 +927,6 @@ mod test { .into(); let node_1 = NodeSpecification { owned_contracts: vec![], - non_owned_contracts: vec![key.clone()], events_to_generate: HashMap::from_iter([(1, get_event)]), contract_subscribers: HashMap::new(), }; @@ -970,7 +967,6 @@ mod test { let node_0 = NodeSpecification { owned_contracts: vec![], - non_owned_contracts: vec![key.clone()], events_to_generate: HashMap::from_iter([(1, get_event)]), contract_subscribers: HashMap::new(), }; @@ -980,14 +976,12 @@ mod test { ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, )], - non_owned_contracts: vec![key.clone()], events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), }; let gw_0 = NodeSpecification { owned_contracts: vec![], - non_owned_contracts: vec![], events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), }; diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 43d339834..52d8cc0da 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -974,7 +974,6 @@ mod test { ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val.clone(), )], - non_owned_contracts: vec![], events_to_generate: HashMap::new(), contract_subscribers: HashMap::from_iter([(key.clone(), vec![node1_loc])]), }; @@ -984,7 +983,6 @@ mod test { ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val.clone(), )], - non_owned_contracts: vec![], events_to_generate: HashMap::new(), contract_subscribers: HashMap::from_iter([(key.clone(), vec![node0_loc])]), }; @@ -1001,7 +999,6 @@ mod test { ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val, )], - non_owned_contracts: vec![], events_to_generate: HashMap::from_iter([(1, put_event)]), contract_subscribers: HashMap::new(), }; diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 90c1c8fc0..f7822f7ff 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -499,13 +499,11 @@ mod test { ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, )], - non_owned_contracts: Vec::new(), events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), }; let second_node = NodeSpecification { owned_contracts: Vec::new(), - non_owned_contracts: vec![contract_key.clone()], events_to_generate: HashMap::from_iter([(1, event)]), contract_subscribers: HashMap::new(), }; diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index f91fff0e8..e5e7c9b56 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -54,7 +54,7 @@ use crate::{ #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] #[cfg_attr(test, derive(arbitrary::Arbitrary))] /// The location of a peer in the ring. This location allows routing towards the peer. -pub(crate) struct PeerKeyLocation { +pub struct PeerKeyLocation { pub peer: PeerKey, /// An unspecified location means that the peer hasn't been asigned a location, yet. pub location: Option, diff --git a/crates/fdev/Cargo.toml b/crates/fdev/Cargo.toml index c6a1c21a8..315824af9 100644 --- a/crates/fdev/Cargo.toml +++ b/crates/fdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fdev" -version = "0.0.6" +version = "0.0.7" edition = "2021" rust-version = "1.71.1" publish = true @@ -34,3 +34,6 @@ xz2 = "0.1" # internal freenet = { path = "../core" } freenet-stdlib = { workspace = true } + +[features] +trace-ot = ["freenet/trace-ot"] diff --git a/crates/fdev/src/build.rs b/crates/fdev/src/build.rs index 5202a0f71..9e265811a 100644 --- a/crates/fdev/src/build.rs +++ b/crates/fdev/src/build.rs @@ -12,7 +12,7 @@ use std::{ use tar::Builder; use crate::{ - config::{BuildToolCliConfig, PackageType}, + config::{BuildToolConfig, PackageType}, util::pipe_std_streams, Error, }; @@ -21,14 +21,14 @@ pub(crate) use contract::*; const DEFAULT_OUTPUT_NAME: &str = "contract-state"; const WASM_TARGET: &str = "wasm32-unknown-unknown"; -pub fn build_package(cli_config: BuildToolCliConfig, cwd: &Path) -> Result<(), anyhow::Error> { +pub fn build_package(cli_config: BuildToolConfig, cwd: &Path) -> Result<(), anyhow::Error> { match cli_config.package_type { PackageType::Contract => contract::package_contract(cli_config, cwd), PackageType::Delegate => delegate::package_delegate(cli_config, cwd), } } -fn compile_options(cli_config: &BuildToolCliConfig) -> impl Iterator { +fn compile_options(cli_config: &BuildToolConfig) -> impl Iterator { let release: &[&str] = if cli_config.debug { &[] } else { @@ -54,7 +54,7 @@ fn compile_options(cli_config: &BuildToolCliConfig) -> impl Iterator Result<(), anyhow::Error> { const RUST_TARGET_ARGS: &[&str] = &["build", "--lib", "--target"]; @@ -108,7 +108,7 @@ fn compile_rust_wasm_lib( fn get_out_lib( work_dir: &Path, - cli_config: &BuildToolCliConfig, + cli_config: &BuildToolConfig, ) -> Result<(String, PathBuf), anyhow::Error> { const ERR: &str = "Cargo.toml definition incorrect"; @@ -159,7 +159,7 @@ mod contract { use super::*; pub(super) fn package_contract( - cli_config: BuildToolCliConfig, + cli_config: BuildToolConfig, cwd: &Path, ) -> Result<(), anyhow::Error> { let mut config = get_config(cwd)?; @@ -185,7 +185,7 @@ mod contract { } #[derive(Serialize, Deserialize)] - pub(crate) struct BuildToolConfig { + pub(crate) struct ContractBuildConfig { pub contract: Contract, pub state: Option, pub webapp: Option, @@ -241,7 +241,7 @@ mod contract { } fn build_web_state( - config: &BuildToolConfig, + config: &ContractBuildConfig, embedded_deps: EmbeddedDeps, cwd: &Path, ) -> Result<(), anyhow::Error> { @@ -398,7 +398,10 @@ mod contract { build_state(sources) } - fn build_generic_state(config: &mut BuildToolConfig, cwd: &Path) -> Result<(), anyhow::Error> { + fn build_generic_state( + config: &mut ContractBuildConfig, + cwd: &Path, + ) -> Result<(), anyhow::Error> { const REQ_ONE_FILE_ERR: &str = "Requires exactly one source file specified for the state."; let sources = config.state.as_mut().and_then(|s| s.files.as_mut()); @@ -441,7 +444,7 @@ mod contract { Ok(()) } - fn get_config(cwd: &Path) -> Result { + fn get_config(cwd: &Path) -> Result { let config_file = cwd.join("freenet.toml"); if config_file.exists() { let mut f_content = vec![]; @@ -453,8 +456,8 @@ mod contract { } fn compile_contract( - config: &BuildToolConfig, - cli_config: &BuildToolCliConfig, + config: &ContractBuildConfig, + cli_config: &BuildToolConfig, cwd: &Path, ) -> Result<(), anyhow::Error> { let work_dir = match config.contract.c_type.unwrap_or(ContractType::Standard) { @@ -488,7 +491,7 @@ mod contract { fn get_versioned_contract( contract_code_path: &Path, - cli_config: &BuildToolCliConfig, + cli_config: &BuildToolConfig, ) -> Result, anyhow::Error> { let code: ContractCode = ContractCode::load_raw(contract_code_path)?; tracing::info!("compiled contract code hash: {}", code.hash_str()); @@ -555,7 +558,7 @@ mod contract { fn embed_deps( cwd: &Path, deps: HashMap, DependencyDefinition>, - cli_config: &BuildToolCliConfig, + cli_config: &BuildToolConfig, ) -> Result { let cwd = fs::canonicalize(cwd)?; let mut deps_json = HashMap::new(); @@ -589,11 +592,11 @@ mod contract { mod test { use super::*; - fn setup_webapp_contract() -> Result<(BuildToolConfig, PathBuf), anyhow::Error> { + fn setup_webapp_contract() -> Result<(ContractBuildConfig, PathBuf), anyhow::Error> { const CRATE_DIR: &str = env!("CARGO_MANIFEST_DIR"); let cwd = PathBuf::from(CRATE_DIR).join("../../apps/freenet-microblogging/web"); Ok(( - BuildToolConfig { + ContractBuildConfig { contract: Contract { c_type: Some(ContractType::WebApp), lang: Some(SupportedContractLangs::Rust), @@ -647,7 +650,7 @@ mod contract { fn compile_webapp_contract() -> Result<(), anyhow::Error> { // let (config, cwd) = setup_webapp_contract()?; - compile_contract(&config, &BuildToolCliConfig::default(), &cwd)?; + compile_contract(&config, &BuildToolConfig::default(), &cwd)?; Ok(()) } @@ -656,7 +659,7 @@ mod contract { const CRATE_DIR: &str = env!("CARGO_MANIFEST_DIR"); let cwd = PathBuf::from(CRATE_DIR).join("../../apps/freenet-microblogging/contracts/posts"); - let mut config = BuildToolConfig { + let mut config = ContractBuildConfig { contract: Contract { c_type: Some(ContractType::Standard), lang: Some(SupportedContractLangs::Rust), @@ -698,7 +701,7 @@ mod contract { posts = { path = "../contracts/posts" } }; let defs = include_deps(&deps).unwrap(); - embed_deps(&cwd, defs, &BuildToolCliConfig::default()).unwrap(); + embed_deps(&cwd, defs, &BuildToolConfig::default()).unwrap(); Ok(()) } } @@ -710,7 +713,7 @@ mod delegate { use super::*; pub(super) fn package_delegate( - cli_config: BuildToolCliConfig, + cli_config: BuildToolConfig, cwd: &Path, ) -> Result<(), anyhow::Error> { compile_rust_wasm_lib(&cli_config, cwd)?; @@ -730,7 +733,7 @@ mod delegate { fn get_versioned_contract( contract_code_path: &Path, - cli_config: &BuildToolCliConfig, + cli_config: &BuildToolConfig, ) -> Result, anyhow::Error> { let code: DelegateCode = DelegateCode::load_raw(contract_code_path)?; tracing::info!("compiled contract code hash: {}", code.hash_str()); diff --git a/crates/fdev/src/config.rs b/crates/fdev/src/config.rs index ae51df979..dcbaea38d 100644 --- a/crates/fdev/src/config.rs +++ b/crates/fdev/src/config.rs @@ -1,6 +1,6 @@ use std::{fmt::Display, path::PathBuf}; -use crate::{commands::PutType, local_node::LocalNodeCliConfig}; +use crate::{commands::PutType, wasm_runtime::ExecutorConfig}; use clap::ValueEnum; use freenet::dev_tool::OperationMode; use semver::Version; @@ -34,15 +34,16 @@ pub struct BaseConfig { #[derive(clap::Subcommand, Clone)] pub enum SubCommand { - RunLocal(LocalNodeCliConfig), - Build(BuildToolCliConfig), - New(NewPackageCliConfig), + New(NewPackageConfig), + Build(BuildToolConfig), + Inspect(crate::inspect::InspectConfig), Publish(PutConfig), + WasmRuntime(ExecutorConfig), Execute(RunCliConfig), - Inspect(crate::inspect::InspectCliConfig), + Test(crate::testing::TestConfig), } -/// Node CLI +/// Core CLI tool for interacting with the Freenet local node. /// /// This tool allows the execution of commands against the local node /// and is intended to be used for development and automated workflows. @@ -92,11 +93,11 @@ pub struct PutConfig { pub(crate) package_type: PutType, } -/// Builds and packages a contract. +/// Builds and packages a contract or delegate. /// /// This tool will build the WASM contract or delegate and publish it to the network. #[derive(clap::Parser, Clone, Debug)] -pub struct BuildToolCliConfig { +pub struct BuildToolConfig { /// Compile the contract or delegate with specific features. #[arg(long)] pub(crate) features: Option, @@ -139,7 +140,7 @@ impl Display for PackageType { } } -impl Default for BuildToolCliConfig { +impl Default for BuildToolConfig { fn default() -> Self { Self { features: None, @@ -156,7 +157,7 @@ fn parse_version(src: &str) -> Result { /// Create a new Freenet contract and/or app. #[derive(clap::Parser, Clone)] -pub struct NewPackageCliConfig { +pub struct NewPackageConfig { #[arg(id = "type", value_enum)] pub(crate) kind: ContractKind, } diff --git a/crates/fdev/src/inspect.rs b/crates/fdev/src/inspect.rs index d5285e8ba..625800186 100644 --- a/crates/fdev/src/inspect.rs +++ b/crates/fdev/src/inspect.rs @@ -4,9 +4,9 @@ use freenet_stdlib::prelude::{ContractCode, DelegateCode}; use crate::Error; -/// Inspect a contract, delegate or freenet kernel compatible executable code properties. +/// Inspect a contract, delegate or freenet core compatible executable code properties. #[derive(clap::Parser, Clone)] -pub struct InspectCliConfig { +pub struct InspectConfig { #[clap(subcommand)] r#type: FileType, file: PathBuf, @@ -22,7 +22,7 @@ enum FileType { #[derive(clap::Parser, Clone)] struct CodeInspection {} -pub fn inspect(config: InspectCliConfig) -> Result<(), anyhow::Error> { +pub fn inspect(config: InspectConfig) -> Result<(), anyhow::Error> { if !config.file.exists() { return Err(Error::CommandFailed("couldn't find file").into()); } diff --git a/crates/fdev/src/main.rs b/crates/fdev/src/main.rs index 7909b71d1..114f4456e 100644 --- a/crates/fdev/src/main.rs +++ b/crates/fdev/src/main.rs @@ -7,17 +7,18 @@ mod build; mod commands; mod config; mod inspect; -mod local_node; mod new_package; +mod testing; mod util; +mod wasm_runtime; use crate::{ build::build_package, commands::{put, update}, config::{Config, SubCommand}, inspect::inspect, - local_node::run_local_node_client, new_package::create_new_package, + wasm_runtime::run_local_executor, }; type CommandReceiver = tokio::sync::mpsc::Receiver>; @@ -38,7 +39,7 @@ async fn main() -> Result<(), anyhow::Error> { let config = Config::parse(); freenet::config::Config::set_op_mode(config.additional.mode); let r = match config.sub_command { - SubCommand::RunLocal(local_node_config) => run_local_node_client(local_node_config).await, + SubCommand::WasmRuntime(local_node_config) => run_local_executor(local_node_config).await, SubCommand::Build(build_tool_config) => build_package(build_tool_config, &cwd), SubCommand::Inspect(inspect_config) => inspect(inspect_config), SubCommand::New(new_pckg_config) => create_new_package(new_pckg_config), @@ -49,6 +50,7 @@ async fn main() -> Result<(), anyhow::Error> { update(update_config, config.additional).await } }, + SubCommand::Test(test_config) => testing::test_framework(test_config).await, }; // todo: make all commands return concrete `thiserror` compatible errors so we can use anyhow r.map_err(|e| anyhow::format_err!(e)) diff --git a/crates/fdev/src/new_package.rs b/crates/fdev/src/new_package.rs index e58116d22..1cb01b6a5 100644 --- a/crates/fdev/src/new_package.rs +++ b/crates/fdev/src/new_package.rs @@ -8,12 +8,12 @@ use std::{ use crate::{ build::*, - config::{ContractKind, NewPackageCliConfig}, + config::{ContractKind, NewPackageConfig}, util::pipe_std_streams, Error, }; -pub fn create_new_package(config: NewPackageCliConfig) -> Result<(), anyhow::Error> { +pub fn create_new_package(config: NewPackageConfig) -> Result<(), anyhow::Error> { let cwd = env::current_dir()?; match config.kind { ContractKind::WebApp => create_view_package(&cwd)?, @@ -25,7 +25,7 @@ pub fn create_new_package(config: NewPackageCliConfig) -> Result<(), anyhow::Err fn create_view_package(cwd: &Path) -> Result<(), anyhow::Error> { create_rust_crate(cwd, ContractKind::WebApp)?; create_web_init_files(cwd)?; - let freenet_file_config = BuildToolConfig { + let freenet_file_config = ContractBuildConfig { contract: Contract { c_type: Some(ContractType::WebApp), lang: Some(SupportedContractLangs::Rust), @@ -52,7 +52,7 @@ fn create_view_package(cwd: &Path) -> Result<(), anyhow::Error> { fn create_regular_contract(cwd: &Path) -> Result<(), anyhow::Error> { create_rust_crate(cwd, ContractKind::Contract)?; - let freenet_file_config = BuildToolConfig { + let freenet_file_config = ContractBuildConfig { contract: Contract { c_type: Some(ContractType::Standard), lang: Some(SupportedContractLangs::Rust), diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs new file mode 100644 index 000000000..9a069bc76 --- /dev/null +++ b/crates/fdev/src/testing.rs @@ -0,0 +1,149 @@ +use anyhow::Error; + +/// Testing framework for running Freenet network simulations. +#[derive(clap::Parser, Clone)] +pub struct TestConfig { + /// Test name. If not provided, a random name will be generated. + #[arg(long)] + name: Option, + /// Seed to use when generating random data. If not provided, a random seed will be used. + #[arg(long)] + seed: Option, + /// Number of total gateways for this test. + #[arg(long, default_value_t = 2)] + gateways: usize, + /// Number of total regular peer nodes for this test. + #[arg(long, default_value_t = 10)] + nodes: usize, + /// Max hops to live for operations (if it applies). + #[arg(long, default_value_t = freenet::config::DEFAULT_MAX_HOPS_TO_LIVE)] + ring_max_htl: usize, + /// Default threshold for randomizing potential peers for new connections. + #[arg(long, default_value_t = freenet::config::DEFAULT_RANDOM_PEER_CONN_THRESHOLD)] + rnd_if_htl_above: usize, + /// Maximum number of connections per peer. + #[arg(long, default_value_t = freenet::config::DEFAULT_MAX_CONNECTIONS)] + max_connections: usize, + /// Minimum number of connections per peer. + #[arg(long, default_value_t = freenet::config::DEFAULT_MIN_CONNECTIONS)] + min_connections: usize, + /// Maximum number of contracts live in the network. + #[arg(long)] + max_contract_number: Option, + /// Number of events that will be executed in this simulation. + /// Events are simulated get, puts and other operations. + #[arg(long, default_value_t = usize::MAX)] + events: usize, + #[clap(subcommand)] + /// Execution mode for the test. + pub command: TestMode, +} + +fn randomize_test_name() -> String { + const ALPHABET: &str = "abcdefghijklmnopqrstuvwxyz"; + use rand::seq::IteratorRandom; + let mut rng = rand::thread_rng(); + let mut name = String::with_capacity(16); + for _ in 0..16 { + name.push(ALPHABET.chars().choose(&mut rng).expect("non empty")); + } + name +} + +/// Under which mode will the test run execute. +#[derive(clap::Parser, Clone)] +pub enum TestMode { + /// Runs multiple simulated nodes in a single process. + SingleProcess, + /// Runs multiple simulated nodes in multiple processes. + MultiProcess, + /// Runs multiple simulated nodes in multiple processes and multiple machines. + Network, +} + +pub(crate) async fn test_framework(base_config: TestConfig) -> Result<(), Error> { + match &base_config.command { + TestMode::SingleProcess => { + single_process::run(&base_config).await?; + } + TestMode::MultiProcess => { + todo!() + } + TestMode::Network => { + todo!() + } + } + Ok(()) +} + +mod single_process { + use freenet::network_sim::SimNetwork; + use futures::StreamExt; + use rand::RngCore; + use tokio::signal; + + pub(super) async fn run(base_config: &super::TestConfig) -> Result<(), super::Error> { + let name = &base_config + .name + .as_ref() + .cloned() + .unwrap_or_else(super::randomize_test_name); + let mut simulated_network = SimNetwork::new( + name, + base_config.gateways, + base_config.nodes, + base_config.ring_max_htl, + base_config.rnd_if_htl_above, + base_config.max_connections, + base_config.min_connections, + ) + .await; + + let join_handles = simulated_network + .start_with_rand_gen::( + base_config + .seed + .unwrap_or_else(|| rand::rngs::OsRng.next_u64()), + base_config + .max_contract_number + .unwrap_or(base_config.nodes * 10), + base_config.events, + ) + .await; + + let join_tasks = async move { + let mut futs = futures::stream::FuturesUnordered::from_iter(join_handles); + while let Some(join_handle) = futs.next().await { + join_handle??; + } + Ok::<_, super::Error>(()) + }; + + // Wait for a signal to gracefully shut down the program + #[cfg(unix)] + { + let mut sigterm = signal::unix::signal(signal::unix::SignalKind::terminate()).unwrap(); + tokio::select! { + _ = signal::ctrl_c() /* SIGINT handling */ => {} + _ = sigterm.recv() => {} + finalized = join_tasks => { + match finalized { + Ok(_) => println!("Test finalized successfully"), + Err(e) => { + println!("Test finalized with error: {}", e); + return Err(e); + } + } + } + } + } + #[cfg(not(unix))] + { + unimplemented!("Windows support is not implemented"); + } + + println!("Shutting down test..."); + + Ok(()) + } +} diff --git a/crates/fdev/src/util.rs b/crates/fdev/src/util.rs index aff69fdc5..576b0ce5d 100644 --- a/crates/fdev/src/util.rs +++ b/crates/fdev/src/util.rs @@ -6,7 +6,7 @@ use std::{ use serde::de::DeserializeOwned; -use crate::local_node::DeserializationFmt; +use crate::wasm_runtime::DeserializationFmt; pub fn deserialize( deser_format: Option, diff --git a/crates/fdev/src/local_node.rs b/crates/fdev/src/wasm_runtime.rs similarity index 92% rename from crates/fdev/src/local_node.rs rename to crates/fdev/src/wasm_runtime.rs index c364e8ff5..ab35f1006 100644 --- a/crates/fdev/src/local_node.rs +++ b/crates/fdev/src/wasm_runtime.rs @@ -8,7 +8,7 @@ mod user_events; const DEFAULT_MAX_CONTRACT_SIZE: i64 = 50 * 1024 * 1024; -pub async fn run_local_node_client(config: LocalNodeCliConfig) -> Result<(), anyhow::Error> { +pub async fn run_local_executor(config: ExecutorConfig) -> Result<(), anyhow::Error> { if config.disable_tui_mode { anyhow::bail!("TUI mode not yet implemented"); } @@ -38,7 +38,7 @@ pub enum DeserializationFmt { Json, } -/// A CLI utility for testing out contracts against a Freenet local node. +/// An interactive runtime for WASM for testing contracts and delegates development. #[derive(clap::Parser, Clone)] #[clap(name = "Freenet Local Development Node Environment")] #[clap(author = "The Freenet Project Inc.")] @@ -47,7 +47,7 @@ pub enum DeserializationFmt { .required(true) .args(&["output-file", "terminal-output"]) ))] -pub struct LocalNodeCliConfig { +pub struct ExecutorConfig { /// Cleanups all state which was created locally during execution #[clap(long, requires = "fmt")] pub(crate) clean_exit: bool, diff --git a/crates/fdev/src/local_node/commands.rs b/crates/fdev/src/wasm_runtime/commands.rs similarity index 97% rename from crates/fdev/src/local_node/commands.rs rename to crates/fdev/src/wasm_runtime/commands.rs index 1d85730a0..c99d5c63b 100644 --- a/crates/fdev/src/local_node/commands.rs +++ b/crates/fdev/src/wasm_runtime/commands.rs @@ -3,10 +3,10 @@ use freenet_stdlib::client_api::{ClientRequest, ContractRequest, ContractRespons use crate::CommandReceiver; -use super::{state::AppState, LocalNodeCliConfig}; +use super::{state::AppState, ExecutorConfig}; pub(super) async fn wasm_runtime( - _config: LocalNodeCliConfig, + _config: ExecutorConfig, mut command_receiver: CommandReceiver, mut app: AppState, ) -> Result<(), anyhow::Error> { diff --git a/crates/fdev/src/local_node/state.rs b/crates/fdev/src/wasm_runtime/state.rs similarity index 88% rename from crates/fdev/src/local_node/state.rs rename to crates/fdev/src/wasm_runtime/state.rs index 638ab6b73..d1391949f 100644 --- a/crates/fdev/src/local_node/state.rs +++ b/crates/fdev/src/wasm_runtime/state.rs @@ -8,21 +8,21 @@ use freenet_stdlib::prelude::*; use futures::TryFutureExt; use tokio::sync::RwLock; -use crate::local_node::DeserializationFmt; +use crate::wasm_runtime::DeserializationFmt; -use super::LocalNodeCliConfig; +use super::ExecutorConfig; #[derive(Clone)] pub(super) struct AppState { pub(crate) local_node: Arc>, - config: LocalNodeCliConfig, + config: ExecutorConfig, } impl AppState { const MAX_MEM_CACHE: u32 = 10_000_000; const DEFAULT_MAX_DELEGATE_SIZE: i64 = 10 * 1024 * 1024; - pub async fn new(config: &LocalNodeCliConfig) -> Result { + pub async fn new(config: &ExecutorConfig) -> Result { let contract_store = ContractStore::new(Config::conf().contracts_dir(), config.max_contract_size)?; let delegate_store = DelegateStore::new( @@ -56,7 +56,7 @@ impl AppState { } pub fn printout_deser + ?Sized>(&self, data: &R) -> Result<(), std::io::Error> { - fn write_res(config: &LocalNodeCliConfig, pprinted: &str) -> Result<(), std::io::Error> { + fn write_res(config: &ExecutorConfig, pprinted: &str) -> Result<(), std::io::Error> { if let Some(p) = &config.output_file { let mut f = File::create(p)?; f.write_all(pprinted.as_bytes())?; diff --git a/crates/fdev/src/local_node/user_events.rs b/crates/fdev/src/wasm_runtime/user_events.rs similarity index 98% rename from crates/fdev/src/local_node/user_events.rs rename to crates/fdev/src/wasm_runtime/user_events.rs index 62cae4d00..267c5a741 100644 --- a/crates/fdev/src/local_node/user_events.rs +++ b/crates/fdev/src/wasm_runtime/user_events.rs @@ -17,7 +17,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::{util, CommandSender}; -use super::{state::AppState, DeserializationFmt, LocalNodeCliConfig}; +use super::{state::AppState, DeserializationFmt, ExecutorConfig}; const HELP: &str = "Freenet Contract Development Environment @@ -31,7 +31,7 @@ SUBCOMMANDS: type HostIncomingMsg = Result, ClientError>; pub(super) async fn user_fn_handler( - config: LocalNodeCliConfig, + config: ExecutorConfig, command_sender: CommandSender, app_state: AppState, ) -> Result<(), anyhow::Error> { @@ -53,7 +53,7 @@ pub(super) async fn user_fn_handler( } struct StdInput { - config: LocalNodeCliConfig, + config: ExecutorConfig, contract: ContractContainer, input: File, buf: Vec, @@ -61,7 +61,7 @@ struct StdInput { } impl StdInput { - fn new(config: LocalNodeCliConfig, app_state: AppState) -> Result { + fn new(config: ExecutorConfig, app_state: AppState) -> Result { let params = config .params .as_ref() diff --git a/modules/antiflood-tokens/Cargo.lock b/modules/antiflood-tokens/Cargo.lock index 64bdac1ef..6f37746b4 100644 --- a/modules/antiflood-tokens/Cargo.lock +++ b/modules/antiflood-tokens/Cargo.lock @@ -21,6 +21,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -57,6 +63,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -84,27 +99,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.65.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.37", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -161,19 +155,9 @@ version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ - "jobserver", "libc", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -301,6 +285,17 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "delegate" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee5df75c70b95bd3aacc8e2fd098797692fb1d54121019c4de481e42f04c8a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "der" version = "0.7.8" @@ -416,6 +411,35 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "freenet-stdlib" +version = "0.0.8" +dependencies = [ + "bincode", + "blake3", + "bs58", + "byteorder", + "chrono", + "flatbuffers", + "freenet-macros 0.0.5", + "futures", + "js-sys", + "once_cell", + "semver", + "serde", + "serde-wasm-bindgen 0.6.0", + "serde_bytes", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "freenet-token-allocation-record" version = "0.1.0" @@ -441,8 +465,8 @@ dependencies = [ "freenet-aft-interface", "freenet-stdlib", "once_cell", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "rsa", "serde", "serde_json", @@ -491,6 +515,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.29" @@ -576,6 +611,9 @@ name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hex" @@ -618,7 +656,7 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "autocfg", + "autocfg 1.1.0", "hashbrown 0.12.3", "serde", ] @@ -649,15 +687,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.65" @@ -673,7 +702,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin", + "spin 0.5.2", ] [[package]] @@ -682,16 +711,6 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libm" version = "0.2.8" @@ -741,7 +760,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "serde", "smallvec", "zeroize", @@ -753,7 +772,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-traits", ] @@ -763,7 +782,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-integer", "num-traits", ] @@ -774,7 +793,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ - "autocfg", + "autocfg 1.1.0", "libm", ] @@ -879,6 +898,25 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -890,6 +928,16 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -1110,7 +1158,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -1125,6 +1173,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.7.2" @@ -1221,28 +1278,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "log", - "ordered-float 2.10.0", - "threadpool", -] - [[package]] name = "time" version = "0.3.30" @@ -1319,16 +1354,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "tracing-log" version = "0.2.0" @@ -1340,22 +1365,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" -dependencies = [ - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", -] - [[package]] name = "tracing-subscriber" version = "0.3.18" From a85ae9e6bfb747adc6f7e76d27c8d990cf65214d Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Wed, 15 Nov 2023 11:31:59 +0100 Subject: [PATCH 02/12] Fix mail app building --- modules/antiflood-tokens/Cargo.lock | 42 ++++++++--------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/modules/antiflood-tokens/Cargo.lock b/modules/antiflood-tokens/Cargo.lock index 6f37746b4..8e828ff1d 100644 --- a/modules/antiflood-tokens/Cargo.lock +++ b/modules/antiflood-tokens/Cargo.lock @@ -21,12 +21,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -415,29 +409,24 @@ dependencies = [ name = "freenet-stdlib" version = "0.0.8" dependencies = [ + "arbitrary", "bincode", "blake3", "bs58", "byteorder", "chrono", "flatbuffers", - "freenet-macros 0.0.5", + "freenet-macros", "futures", - "js-sys", "once_cell", "semver", "serde", - "serde-wasm-bindgen 0.6.0", "serde_bytes", "serde_json", "serde_with", "thiserror", - "tokio", - "tokio-tungstenite", "tracing", "tracing-subscriber", - "wasm-bindgen", - "web-sys", ] [[package]] @@ -465,8 +454,8 @@ dependencies = [ "freenet-aft-interface", "freenet-stdlib", "once_cell", - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand", + "rand_chacha", "rsa", "serde", "serde_json", @@ -656,7 +645,7 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "autocfg 1.1.0", + "autocfg", "hashbrown 0.12.3", "serde", ] @@ -702,7 +691,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin 0.5.2", + "spin", ] [[package]] @@ -760,7 +749,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand", "serde", "smallvec", "zeroize", @@ -772,7 +761,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -782,7 +771,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", ] @@ -793,7 +782,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ - "autocfg 1.1.0", + "autocfg", "libm", ] @@ -1158,7 +1147,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -1173,15 +1162,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - [[package]] name = "spki" version = "0.7.2" From b7ab487b157948c15d72f05db6fc96aa527166c9 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Wed, 15 Nov 2023 18:21:44 +0100 Subject: [PATCH 03/12] Actually generate events in the test --- crates/core/src/client_events.rs | 15 +- crates/core/src/node.rs | 8 +- crates/core/src/node/tests.rs | 191 ++++++++++++++++++------ crates/core/src/operations/connect.rs | 4 +- crates/core/src/operations/get.rs | 6 +- crates/core/src/operations/put.rs | 2 +- crates/core/src/operations/subscribe.rs | 2 +- crates/core/src/server/path_handlers.rs | 4 +- crates/fdev/src/testing.rs | 67 +++++++-- stdlib | 2 +- 10 files changed, 225 insertions(+), 76 deletions(-) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index f62fbc113..339e3c17a 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -190,12 +190,15 @@ pub(crate) mod test { } } - pub fn rng_params(&mut self, num_peers: usize, max_contract_num: usize, iterations: usize) { - // let hasher = &mut rand::thread_rng(); - // let hash = self.id.hash(hasher); - // let this_peer = hasher.finish(); + pub fn rng_params( + &mut self, + id: usize, + num_peers: usize, + max_contract_num: usize, + iterations: usize, + ) { let internal_state = InternalGeneratorState { - this_peer: 0, + this_peer: id, num_peers, max_contract_num, max_iterations: iterations, @@ -276,7 +279,7 @@ pub(crate) mod test { } else { // probably the process finished, wait for a bit and then kill the thread tokio::time::sleep(Duration::from_secs(1)).await; - panic!("finished orphan background thread"); + break Err(ErrorKind::Shutdown.into()); } } } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 5797f4372..f47a8f475 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -16,7 +16,7 @@ use std::{ }; use either::Either; -use freenet_stdlib::client_api::{ClientRequest, ContractRequest}; +use freenet_stdlib::client_api::{ClientRequest, ContractRequest, ErrorKind}; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; use tracing::Instrument; @@ -327,6 +327,10 @@ async fn client_event_handling( tracing::debug!(%req, "got client request event"); req } + Err(err) if matches!(err.kind(), ErrorKind::Shutdown) => { + node_controller.send(NodeEvent::Disconnect { cause: None }).await.ok(); + break; + } Err(err) => { tracing::debug!(error = %err, "client error"); continue; @@ -334,7 +338,7 @@ async fn client_event_handling( }; if let ClientRequest::Disconnect { cause } = &*req.request { node_controller.send(NodeEvent::Disconnect { cause: cause.clone() }).await.ok(); - continue; + break; } process_open_request(req, op_storage.clone()).await; } diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index 20c960171..d896910d3 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -9,11 +9,8 @@ use std::{ use freenet_stdlib::prelude::*; use itertools::Itertools; use libp2p::{identity, PeerId}; -use rand::Rng; -use tokio::sync::{ - watch::{channel, Receiver, Sender}, - Barrier, -}; +use rand::{seq::SliceRandom, Rng}; +use tokio::sync::watch::{channel, Receiver, Sender}; use tracing::{info, Instrument}; #[cfg(feature = "trace-ot")] @@ -48,7 +45,7 @@ pub fn get_dynamic_port() -> u16 { pub(crate) type EventId = usize; -#[derive(PartialEq, Eq, Hash, Clone)] +#[derive(PartialEq, Eq, Hash, Clone, PartialOrd, Ord, Debug)] pub struct NodeLabel(Arc); impl NodeLabel { @@ -67,6 +64,17 @@ impl NodeLabel { pub fn is_node(&self) -> bool { self.0.starts_with("node") } + + pub fn number(&self) -> usize { + let mut parts = self.0.split('-'); + assert!(parts.next().is_some()); + parts + .next() + .map(|s| s.parse::()) + .transpose() + .expect("should be an usize") + .expect("should have an other part") + } } impl std::fmt::Display for NodeLabel { @@ -115,15 +123,44 @@ struct GatewayConfig { location: Location, } +pub struct EventChain<'a> { + network: &'a SimNetwork, + total_events: usize, + count: usize, + rng: rand::rngs::SmallRng, +} + +impl Iterator for EventChain<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + (self.count < self.total_events).then(|| { + let (_, id) = self + .network + .labels + .choose(&mut self.rng) + .expect("not empty"); + self.network + .user_ev_controller + .send((self.count, *id)) + .expect("peer controller should be alive"); + self.count += 1; + self.count + }) + } +} + /// A simulated in-memory network topology. pub struct SimNetwork { name: String, debug: bool, - pub(crate) labels: HashMap, + labels: Vec<(NodeLabel, PeerKey)>, pub(crate) event_listener: TestEventListener, user_ev_controller: Sender<(EventId, PeerKey)>, receiver_ch: Receiver<(EventId, PeerKey)>, + number_of_gateways: usize, gateways: Vec<(NodeInMemory, GatewayConfig)>, + number_of_nodes: usize, nodes: Vec<(NodeInMemory, NodeLabel)>, ring_max_htl: usize, rnd_if_htl_above: usize, @@ -133,6 +170,24 @@ pub struct SimNetwork { add_noise: bool, } +#[cfg(any(debug_assertions, test))] +impl std::fmt::Debug for SimNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SimNetwork") + .field("name", &self.name) + .field("labels", &self.labels) + .field("number_of_gateways", &self.number_of_gateways) + .field("number_of_nodes", &self.number_of_nodes) + .field("ring_max_htl", &self.ring_max_htl) + .field("rnd_if_htl_above", &self.rnd_if_htl_above) + .field("max_connections", &self.max_connections) + .field("min_connections", &self.min_connections) + .field("init_backoff", &self.init_backoff) + .field("add_noise", &self.add_noise) + .finish() + } +} + impl SimNetwork { pub async fn new( name: &str, @@ -149,10 +204,12 @@ impl SimNetwork { name: name.into(), debug: false, event_listener: TestEventListener::new(), - labels: HashMap::new(), + labels: Vec::with_capacity(nodes + gateways), user_ev_controller, receiver_ch, + number_of_gateways: gateways, gateways: Vec::with_capacity(gateways), + number_of_nodes: nodes, nodes: Vec::with_capacity(nodes), ring_max_htl, rnd_if_htl_above, @@ -253,7 +310,7 @@ impl SimNetwork { .cloned() .collect(); - for node_no in 0..num { + for node_no in self.number_of_gateways..num + self.number_of_gateways { let label = NodeLabel::node(node_no); let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); @@ -322,6 +379,7 @@ impl SimNetwork { self.initialize_peer(node, label, node_spec); tokio::time::sleep(self.init_backoff).await; } + self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); } #[cfg(test)] @@ -336,7 +394,7 @@ impl SimNetwork { user_events.generate_events(specs.events_to_generate); } tracing::debug!(peer = %label, "initializing"); - self.labels.insert(label, peer.peer_key); + self.labels.push((label, peer.peer_key)); let node_task = async move { if let Some(specs) = node_specs { peer.append_contracts(specs.owned_contracts, specs.contract_subscribers) @@ -369,6 +427,7 @@ impl SimNetwork { peers.push(handle); tokio::time::sleep(self.init_backoff).await; } + self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); peers } @@ -384,7 +443,8 @@ impl SimNetwork { tracing::debug!(peer = %label, "initializing"); let mut user_events = MemoryEventsGen::::new_with_seed(self.receiver_ch.clone(), peer.peer_key, seed); - user_events.rng_params(total_peer_num, max_contract_num, iterations); + user_events.rng_params(label.number(), total_peer_num, max_contract_num, iterations); + self.labels.push((label, peer.peer_key)); let node_task = async move { peer.run_node(user_events).await }; GlobalExecutor::spawn(node_task) } @@ -403,11 +463,11 @@ impl SimNetwork { } pub fn connected(&self, peer: &NodeLabel) -> bool { - if let Some(key) = self.labels.get(peer) { - self.event_listener.is_connected(key) - } else { - panic!("peer not found"); - } + let pos = self + .labels + .binary_search_by(|(label, _)| label.cmp(peer)) + .expect("peer not found"); + self.event_listener.is_connected(&self.labels[pos].1) } pub fn has_put_contract( @@ -416,27 +476,33 @@ impl SimNetwork { key: &ContractKey, value: &WrappedState, ) -> bool { - if let Some(pk) = self.labels.get(&peer.into()) { - self.event_listener.has_put_contract(pk, key, value) - } else { - panic!("peer not found"); - } + let peer = peer.into(); + let pos = self + .labels + .binary_search_by(|(label, _)| label.cmp(&peer)) + .expect("peer not found"); + self.event_listener + .has_put_contract(&self.labels[pos].1, key, value) } pub fn has_got_contract(&self, peer: impl Into, key: &ContractKey) -> bool { - if let Some(pk) = self.labels.get(&peer.into()) { - self.event_listener.has_got_contract(pk, key) - } else { - panic!("peer not found"); - } + let peer = peer.into(); + let pos = self + .labels + .binary_search_by(|(label, _)| label.cmp(&peer)) + .expect("peer not found"); + self.event_listener + .has_got_contract(&self.labels[pos].1, key) } pub fn is_subscribed_to_contract(&self, peer: impl Into, key: &ContractKey) -> bool { - if let Some(pk) = self.labels.get(&peer.into()) { - self.event_listener.is_subscribed_to_contract(pk, key) - } else { - panic!("peer not found"); - } + let peer = peer.into(); + let pos = self + .labels + .binary_search_by(|(label, _)| label.cmp(&peer)) + .expect("peer not found"); + self.event_listener + .is_subscribed_to_contract(&self.labels[pos].1, key) } /// Builds an histogram of the distribution in the ring of each node relative to each other. @@ -476,16 +542,19 @@ impl SimNetwork { /// - label: node for which to trigger the /// - event_id: which event to trigger /// - await_for: if set, wait for the duration before returning - pub async fn trigger_event( + #[cfg(test)] + pub(crate) async fn trigger_event( &self, label: impl Into, event_id: EventId, await_for: Option, ) -> Result<(), anyhow::Error> { - let peer = self + let label = label.into(); + let pos = self .labels - .get(&label.into()) - .ok_or_else(|| anyhow::anyhow!("node not found"))?; + .binary_search_by(|(other, _)| other.cmp(&label)) + .map_err(|_| anyhow::anyhow!("peer not found"))?; + let (_, peer) = &self.labels[pos]; self.user_ev_controller .send((event_id, *peer)) .expect("node listeners disconnected"); @@ -495,21 +564,48 @@ impl SimNetwork { Ok(()) } + /// Start an event chain for this simulation. + pub fn event_chain(&self, total_events: usize) -> EventChain<'_> { + const SEED: u64 = 0xdeadbeef; + EventChain { + network: self, + total_events, + count: 0, + rng: rand::rngs::SmallRng::seed_from_u64(SEED), + } + } + /// Checks that all peers in the network have acquired at least one connection to any /// other peers. - pub async fn check_connectivity(&self, time_out: Duration) -> Result<(), anyhow::Error> { - let num_nodes = self.nodes.capacity(); + pub fn check_connectivity(&self, time_out: Duration) -> Result<(), anyhow::Error> { + self.connectivity(time_out, 1.0) + } + + /// Checks that a percentage (given as a float between 0 and 1) of the nodes has at least + /// one connection to any other peers. + pub fn check_partial_connectivity( + &self, + time_out: Duration, + percent: f64, + ) -> Result<(), anyhow::Error> { + self.connectivity(time_out, percent) + } + + fn connectivity(&self, time_out: Duration, percent: f64) -> Result<(), anyhow::Error> { + let num_nodes = self.number_of_nodes; let mut connected = HashSet::new(); let elapsed = Instant::now(); - while elapsed.elapsed() < time_out && connected.len() < num_nodes { - for node in 0..num_nodes { + while elapsed.elapsed() < time_out && (connected.len() as f64 / num_nodes as f64) < percent + { + for node in self.number_of_gateways..num_nodes + self.number_of_gateways { if !connected.contains(&node) && self.connected(&NodeLabel::node(node)) { connected.insert(node); } } } - let expected = HashSet::from_iter(0..num_nodes); + let expected = + HashSet::from_iter(self.number_of_gateways..num_nodes + self.number_of_gateways); let mut missing: Vec<_> = expected .difference(&connected) .map(|n| format!("node-{}", n)) @@ -517,9 +613,16 @@ impl SimNetwork { tracing::info!("Number of simulated nodes: {num_nodes}"); - if !missing.is_empty() { + let missing_percent = 1.0 - (missing.len() as f64 / num_nodes as f64); + if missing_percent > percent { missing.sort(); - tracing::error!("Nodes without connection: {:?}", missing); + let show_max = missing.len().min(100); + tracing::error!( + "Nodes without connection: {:?}(..) ({}% > {}%)", + &missing[..show_max], + missing_percent * 100.0, + percent * 100.0 + ); tracing::error!("Total nodes without connection: {:?}", missing.len()); anyhow::bail!("found disconnected nodes"); } @@ -553,7 +656,7 @@ impl SimNetwork { /// - pub fn network_connectivity_quality(&self) -> Result<(), anyhow::Error> { const HIGHER_THAN_MIN_THRESHOLD: f64 = 0.5; - let num_nodes = self.nodes.capacity(); + let num_nodes = self.number_of_nodes; let min_connections_threshold = (num_nodes as f64 * HIGHER_THAN_MIN_THRESHOLD) as usize; let node_connectivity = self.node_connectivity(); @@ -596,7 +699,7 @@ impl SimNetwork { impl Drop for SimNetwork { fn drop(&mut self) { if !self.debug { - for label in self.labels.keys() { + for (label, _) in &self.labels { let p = std::env::temp_dir() .join(format!("freenet-executor-{sim}-{label}", sim = self.name)); let _ = std::fs::remove_dir_all(p); diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index 0ca620e6e..a3f42de2b 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -1170,7 +1170,7 @@ mod test { .await; // sim_nw.with_start_backoff(Duration::from_millis(100)); sim_nw.start().await; - sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3))?; let some_forwarded = sim_nw .node_connectivity() .into_iter() @@ -1201,7 +1201,7 @@ mod test { .await; sim_nw.with_start_backoff(Duration::from_millis(100)); sim_nw.start().await; - sim_nw.check_connectivity(Duration::from_secs(10)).await?; + sim_nw.check_connectivity(Duration::from_secs(10))?; // wait for a bit so peers can acquire more connections tokio::time::sleep(Duration::from_secs(3)).await; sim_nw.network_connectivity_quality()?; diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index f2f3b1d98..5c677db3e 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -900,7 +900,7 @@ mod test { ) .await; sim_nw.start_with_spec(get_specs).await; - sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3))?; // trigger get @ node-0, which does not own the contract sim_nw @@ -937,7 +937,7 @@ mod test { let mut sim_nw = SimNetwork::new("get_contract_not_found", NUM_GW, NUM_NODES, 3, 2, 4, 2).await; sim_nw.start_with_spec(get_specs).await; - sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3))?; // trigger get @ node-1, which does not own the contract sim_nw @@ -1004,7 +1004,7 @@ mod test { ) .await; sim_nw.start_with_spec(get_specs).await; - sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3))?; sim_nw .trigger_event("node-0", 1, Some(Duration::from_secs(1))) .await?; diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 52d8cc0da..2ca1eaa0f 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -1011,7 +1011,7 @@ mod test { ]); sim_nw.start_with_spec(put_specs).await; - sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3))?; // trigger the put op @ gw-0 sim_nw diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index f7822f7ff..cce1bd6b6 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -523,7 +523,7 @@ mod test { ) .await; sim_nw.start_with_spec(subscribe_specs).await; - sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3))?; sim_nw .trigger_event("node-1", 1, Some(Duration::from_secs(1))) .await?; diff --git a/crates/core/src/server/path_handlers.rs b/crates/core/src/server/path_handlers.rs index a7184b392..905378d13 100644 --- a/crates/core/src/server/path_handlers.rs +++ b/crates/core/src/server/path_handlers.rs @@ -129,7 +129,9 @@ pub(super) async fn contract_home( result: Err(err), .. }) => { tracing::error!("error getting contract `{key}`: {err}"); - return Err(WebSocketApiError::AxumError { error: err.kind() }); + return Err(WebSocketApiError::AxumError { + error: err.kind().clone(), + }); } None => { return Err(WebSocketApiError::NodeError { diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index 9a069bc76..6033bf3b8 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -34,6 +34,13 @@ pub struct TestConfig { /// Events are simulated get, puts and other operations. #[arg(long, default_value_t = usize::MAX)] events: usize, + /// Time in milliseconds to wait for the network to be sufficiently connected + /// to start requesting events. (20% of the network) + #[arg(long, default_value_t = 15_000)] + wait_duration: u64, + /// Time in milliseconds to wait for the next event to be executed. + #[arg(long)] + event_wait_time: Option, #[clap(subcommand)] /// Execution mode for the test. pub command: TestMode, @@ -77,6 +84,8 @@ pub(crate) async fn test_framework(base_config: TestConfig) -> Result<(), Error> } mod single_process { + use std::time::Duration; + use freenet::network_sim::SimNetwork; use futures::StreamExt; use rand::RngCore; @@ -111,7 +120,22 @@ mod single_process { ) .await; - let join_tasks = async move { + let events = base_config.events; + let connectivity_timeout = Duration::from_millis(base_config.wait_duration); + let next_event_wait_time = base_config.event_wait_time.map(Duration::from_millis); + let events_generated = tokio::task::spawn_blocking(move || { + // todo: come with a good time_out wait value experimentally, + // on avg how long should it take for the 20% of the network to be connected? + simulated_network.check_partial_connectivity(connectivity_timeout, 0.20)?; + for _ in simulated_network.event_chain(events) { + if let Some(t) = next_event_wait_time { + std::thread::sleep(t); + } + } + Ok::<_, super::Error>(()) + }); + + let join_peer_tasks = async move { let mut futs = futures::stream::FuturesUnordered::from_iter(join_handles); while let Some(join_handle) = futs.next().await { join_handle??; @@ -119,16 +143,35 @@ mod single_process { Ok::<_, super::Error>(()) }; - // Wait for a signal to gracefully shut down the program - #[cfg(unix)] - { - let mut sigterm = signal::unix::signal(signal::unix::SignalKind::terminate()).unwrap(); + let ctrl_c = signal::ctrl_c(); + + tokio::pin!(events_generated); + tokio::pin!(join_peer_tasks); + tokio::pin!(ctrl_c); + + loop { tokio::select! { - _ = signal::ctrl_c() /* SIGINT handling */ => {} - _ = sigterm.recv() => {} - finalized = join_tasks => { + _ = &mut ctrl_c /* SIGINT handling */ => { + break; + } + res = &mut events_generated => { + match res? { + Ok(_) => { + println!("Test events generated successfully"); + *events_generated = tokio::task::spawn(futures::future::pending::>()); + continue; + } + Err(err) => { + return Err(err); + } + } + } + finalized = &mut join_peer_tasks => { match finalized { - Ok(_) => println!("Test finalized successfully"), + Ok(_) => { + println!("Test finalized successfully"); + break; + } Err(e) => { println!("Test finalized with error: {}", e); return Err(e); @@ -137,12 +180,6 @@ mod single_process { } } } - #[cfg(not(unix))] - { - unimplemented!("Windows support is not implemented"); - } - - println!("Shutting down test..."); Ok(()) } diff --git a/stdlib b/stdlib index 08d0017fc..398295388 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit 08d0017fc84cf36eb9016ea6467cfb51c005328b +Subproject commit 3982953881f11092e61b000431fa13b3235f61d4 From 770677d18fc84594d3f9103576ef5a6fb73e9b81 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 16 Nov 2023 12:57:34 +0100 Subject: [PATCH 04/12] Fix error with overwritting store paths for different peers --- crates/core/src/runtime/contract_store.rs | 25 ++++++++--------------- crates/core/src/runtime/delegate_store.rs | 25 ++++++++--------------- crates/core/src/runtime/secrets_store.rs | 23 +++++++-------------- 3 files changed, 23 insertions(+), 50 deletions(-) diff --git a/crates/core/src/runtime/contract_store.rs b/crates/core/src/runtime/contract_store.rs index 34cdf1809..4f2367fcf 100644 --- a/crates/core/src/runtime/contract_store.rs +++ b/crates/core/src/runtime/contract_store.rs @@ -14,6 +14,7 @@ use super::{error::RuntimeInnerError, store::StoreFsManagement, RuntimeResult}; /// Handle contract blob storage on the file system. pub struct ContractStore { contracts_dir: PathBuf, + key_file: PathBuf, contract_cache: Cache>>, key_to_code_part: Arc>, index_file: BufWriter, @@ -21,8 +22,6 @@ pub struct ContractStore { // TODO: add functionality to delete old contracts which have not been used for a while // to keep the total space used under a configured threshold -static KEY_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); - impl StoreFsManagement for ContractStore { type MemContainer = Arc>; type Key = ContractInstanceId; @@ -43,13 +42,7 @@ impl ContractStore { pub fn new(contracts_dir: PathBuf, max_size: i64) -> RuntimeResult { const ERR: &str = "failed to build mem cache"; let mut key_to_code_part = Arc::new(DashMap::new()); - let key_file = match KEY_FILE_PATH - .try_insert(contracts_dir.join("KEY_DATA")) - .map_err(|(e, _)| e) - { - Ok(f) => f, - Err(f) => f, - }; + let key_file = contracts_dir.join("KEY_DATA"); if !key_file.exists() { std::fs::create_dir_all(&contracts_dir).map_err(|err| { tracing::error!("error creating contract dir: {err}"); @@ -57,15 +50,16 @@ impl ContractStore { })?; File::create(contracts_dir.join("KEY_DATA"))?; } else { - Self::load_from_file(key_file, &mut key_to_code_part)?; + Self::load_from_file(&key_file, &mut key_to_code_part)?; } - Self::watch_changes(key_to_code_part.clone(), key_file)?; + Self::watch_changes(key_to_code_part.clone(), &key_file)?; let index_file = - std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(key_file)?); + std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(&key_file)?); Ok(Self { contract_cache: Cache::new(100, max_size).expect(ERR), contracts_dir, + key_file, key_to_code_part, index_file, }) @@ -160,10 +154,7 @@ impl ContractStore { let current_version_offset = v.get().0; let prev_val = &mut v.get_mut().1; // first mark the old entry (if it exists) as removed - Self::remove( - KEY_FILE_PATH.get().expect("should be set"), - current_version_offset, - )?; + Self::remove(&self.key_file, current_version_offset)?; let new_offset = Self::insert(&mut self.index_file, *key.id(), code_hash)?; *prev_val = *code_hash; v.get_mut().0 = new_offset; @@ -198,7 +189,7 @@ impl ContractStore { })?, }; if let Some((_, (offset, _))) = self.key_to_code_part.remove(key.id()) { - Self::remove(KEY_FILE_PATH.get().expect("infallible"), offset)?; + Self::remove(&self.key_file, offset)?; } let key_path = self .contracts_dir diff --git a/crates/core/src/runtime/delegate_store.rs b/crates/core/src/runtime/delegate_store.rs index eafde9a79..0309e5694 100644 --- a/crates/core/src/runtime/delegate_store.rs +++ b/crates/core/src/runtime/delegate_store.rs @@ -16,10 +16,9 @@ pub struct DelegateStore { delegate_cache: Cache>, key_to_code_part: Arc>, index_file: BufWriter, + key_file: PathBuf, } -static KEY_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); - impl StoreFsManagement for DelegateStore { type MemContainer = Arc>; type Key = DelegateKey; @@ -40,13 +39,7 @@ impl DelegateStore { pub fn new(delegates_dir: PathBuf, max_size: i64) -> RuntimeResult { const ERR: &str = "failed to build mem cache"; let mut key_to_code_part = Arc::new(DashMap::new()); - let key_file = match KEY_FILE_PATH - .try_insert(delegates_dir.join("KEY_DATA")) - .map_err(|(e, _)| e) - { - Ok(f) => f, - Err(f) => f, - }; + let key_file = delegates_dir.join("KEY_DATA"); if !key_file.exists() { std::fs::create_dir_all(&delegates_dir).map_err(|err| { tracing::error!("error creating delegate dir: {err}"); @@ -54,17 +47,18 @@ impl DelegateStore { })?; File::create(delegates_dir.join("KEY_DATA"))?; } else { - Self::load_from_file(key_file, &mut key_to_code_part)?; + Self::load_from_file(&key_file, &mut key_to_code_part)?; } - Self::watch_changes(key_to_code_part.clone(), key_file)?; + Self::watch_changes(key_to_code_part.clone(), &key_file)?; let index_file = - std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(key_file)?); + std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(&key_file)?); Ok(Self { delegate_cache: Cache::new(100, max_size).expect(ERR), delegates_dir, key_to_code_part, index_file, + key_file, }) } @@ -138,10 +132,7 @@ impl DelegateStore { let current_version_offset = v.get().0; let prev_val = &mut v.get_mut().1; // first mark the old entry (if it exists) as removed - Self::remove( - KEY_FILE_PATH.get().expect("should be set"), - current_version_offset, - )?; + Self::remove(&self.key_file, current_version_offset)?; let new_offset = Self::insert(&mut self.index_file, key.clone(), code_hash)?; *prev_val = *code_hash; v.get_mut().0 = new_offset; @@ -159,7 +150,7 @@ impl DelegateStore { self.delegate_cache.remove(key.code_hash()); let cmp_path: PathBuf = self.delegates_dir.join(key.encode()).with_extension("wasm"); if let Some((_, (offset, _))) = self.key_to_code_part.remove(key) { - Self::remove(KEY_FILE_PATH.get().expect("infallible"), offset)?; + Self::remove(&self.key_file, offset)?; } match std::fs::remove_file(cmp_path) { Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), diff --git a/crates/core/src/runtime/secrets_store.rs b/crates/core/src/runtime/secrets_store.rs index b3404ecf4..93931318f 100644 --- a/crates/core/src/runtime/secrets_store.rs +++ b/crates/core/src/runtime/secrets_store.rs @@ -39,10 +39,9 @@ pub struct SecretsStore { ciphers: HashMap, key_to_secret_part: Arc)>>, index_file: BufWriter, + key_file: PathBuf, } -static KEY_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); - pub(super) struct ConcatenatedSecretKeys(Vec); impl AsRef<[u8]> for ConcatenatedSecretKeys { @@ -108,13 +107,7 @@ static DEFAULT_ENCRYPTION: Lazy = Lazy::new(|| Encryption { impl SecretsStore { pub fn new(secrets_dir: PathBuf) -> RuntimeResult { let mut key_to_secret_part = Arc::new(DashMap::new()); - let key_file = match KEY_FILE_PATH - .try_insert(secrets_dir.join("KEY_DATA")) - .map_err(|(e, _)| e) - { - Ok(f) => f, - Err(f) => f, - }; + let key_file = secrets_dir.join("KEY_DATA"); if !key_file.exists() { std::fs::create_dir_all(&secrets_dir).map_err(|err| { tracing::error!("error creating delegate dir: {err}"); @@ -122,17 +115,18 @@ impl SecretsStore { })?; File::create(secrets_dir.join("KEY_DATA"))?; } else { - Self::load_from_file(key_file, &mut key_to_secret_part)?; + Self::load_from_file(&key_file, &mut key_to_secret_part)?; } - Self::watch_changes(key_to_secret_part.clone(), key_file)?; + Self::watch_changes(key_to_secret_part.clone(), &key_file)?; let index_file = - std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(key_file)?); + std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(&key_file)?); Ok(Self { base_path: secrets_dir, ciphers: HashMap::new(), key_to_secret_part, index_file, + key_file, }) } @@ -183,10 +177,7 @@ impl SecretsStore { value.extend_from_slice(hash); } // first mark the old entry (if it exists) as removed - Self::remove( - KEY_FILE_PATH.get().expect("should be set"), - current_version_offset, - )?; + Self::remove(&self.key_file, current_version_offset)?; let new_offset = Self::insert( &mut self.index_file, delegate.clone(), From 60cd9adce7b1dbe39dd4dc49eaa9e5daa7c379eb Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 16 Nov 2023 12:59:55 +0100 Subject: [PATCH 05/12] Fix some issues around the test util setup --- crates/core/src/client_events.rs | 21 ++++-- crates/core/src/node.rs | 90 ++++++++++++----------- crates/core/src/node/tests.rs | 10 +-- crates/core/src/operations/get.rs | 33 +++++---- crates/core/src/operations/put.rs | 24 +++--- crates/core/src/operations/subscribe.rs | 24 +++--- crates/core/src/ring.rs | 6 +- crates/core/src/runtime/tests/contract.rs | 5 -- crates/core/src/runtime/tests/time.rs | 1 - crates/core/src/topology.rs | 2 +- crates/fdev/src/testing.rs | 46 +++++++++--- 11 files changed, 154 insertions(+), 108 deletions(-) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index 339e3c17a..d384e663e 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -207,13 +207,21 @@ pub(crate) mod test { self.internal_state = Some(internal_state); } - fn generate_rand_event(&mut self) -> Option> { - let (rng, state) = self + async fn generate_rand_event(&mut self) -> Option> { + let (mut rng, mut state) = self .rng - .as_mut() - .zip(self.internal_state.as_mut()) + .take() + .zip(self.internal_state.take()) .expect("rng should be set"); - rng.gen_event(state) + let (rng, state, res) = tokio::task::spawn_blocking(move || { + let res = rng.gen_event(&mut state); + (rng, state, res) + }) + .await + .expect("task shouldn't fail"); + self.rng = Some(rng); + self.internal_state = Some(state); + res } } @@ -258,6 +266,7 @@ pub(crate) mod test { client_id: ClientId::FIRST, request: self .generate_rand_event() + .await .ok_or_else(|| ClientError::from(ErrorKind::Disconnect))? .into(), notification_channel: None, @@ -307,7 +316,7 @@ pub(crate) mod test { existing_contracts: Vec, } - pub trait RandomEventGenerator { + pub trait RandomEventGenerator: Send + 'static { fn seed_from_u64(seed: u64) -> Self; fn gen_u8(&mut self) -> u8; diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index f47a8f475..50a148c1a 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -323,16 +323,16 @@ async fn client_event_handling( tokio::select! { client_request = client_events.recv() => { let req = match client_request { - Ok(req) => { - tracing::debug!(%req, "got client request event"); - req + Ok(request) => { + tracing::debug!(%request, "got client request event"); + request } - Err(err) if matches!(err.kind(), ErrorKind::Shutdown) => { + Err(error) if matches!(error.kind(), ErrorKind::Shutdown) => { node_controller.send(NodeEvent::Disconnect { cause: None }).await.ok(); break; } - Err(err) => { - tracing::debug!(error = %err, "client error"); + Err(error) => { + tracing::debug!(%error, "client error"); continue; } }; @@ -344,8 +344,8 @@ async fn client_event_handling( } res = client_responses.recv() => { if let Some((cli_id, res)) = res { - if let Ok(res) = &res { - tracing::debug!(%res, "sending client response"); + if let Ok(result) = &res { + tracing::debug!(%result, "sending client response"); } if let Err(err) = client_events.send(cli_id, res).await { tracing::debug!("channel closed: {err}"); @@ -372,8 +372,8 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { // Initialize a put op. tracing::debug!( - "Received put from user event @ {}", - &op_storage.ring.peer_key + this_peer = %op_storage.ring.peer_key, + "Received put from user event", ); let op = put::start_op( contract, @@ -406,41 +406,49 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { - // Initialize a subscribe op. - loop { - let op = subscribe::start_op(key.clone()); - match subscribe::request_subscribe(&op_storage, op, Some(client_id)).await { - Err(OpError::ContractError(ContractError::ContractNotFound(key))) - if !missing_contract => + const TIMEOUT: Duration = Duration::from_secs(10); + let timeout = tokio::time::timeout(TIMEOUT, async { + // Initialize a subscribe op. + loop { + let op = subscribe::start_op(key.clone()); + match subscribe::request_subscribe(&op_storage, op, Some(client_id)) + .await { - tracing::info!("Trying to subscribe to a contract not present: {key}, requesting it first"); - missing_contract = true; - let get_op = get::start_op(key.clone(), true); - if let Err(err) = - get::request_get(&op_storage, get_op, Some(client_id)).await - { - tracing::error!("Failed getting the contract `{key}` while previously trying to subscribe; bailing: {err}"); + Err(OpError::ContractError(ContractError::ContractNotFound( + key, + ))) if !missing_contract => { + tracing::info!(%key, "Trying to subscribe to a contract not present, requesting it first"); + missing_contract = true; + let get_op = get::start_op(key.clone(), true); + if let Err(error) = + get::request_get(&op_storage, get_op, Some(client_id)).await + { + tracing::error!(%key, %error, "Failed getting the contract while previously trying to subscribe; bailing"); + break; + } + } + Err(OpError::ContractError(ContractError::ContractNotFound(_))) => { + tracing::warn!("Still waiting for {key} contract"); + tokio::time::sleep(Duration::from_secs(2)).await + } + Err(err) => { + tracing::error!("{}", err); break; } - } - Err(OpError::ContractError(ContractError::ContractNotFound(_))) => { - tracing::warn!("Still waiting for {key} contract"); - tokio::time::sleep(Duration::from_secs(2)).await - } - Err(err) => { - tracing::error!("{}", err); - break; - } - Ok(()) => { - if missing_contract { - tracing::debug!( - "Got back the missing contract ({key}) while subscribing" - ); + Ok(()) => { + if missing_contract { + tracing::debug!(%key, + "Got back the missing contract while subscribing" + ); + } + tracing::debug!(%key, "Starting subscribe request"); + break; } - tracing::debug!("Starting subscribe request to {key}"); - break; } } + }); + if timeout.await.is_err() { + tracing::error!(%key, "Timeout while waiting for contract"); } } _ => { @@ -536,7 +544,7 @@ async fn report_result( if let Some(tx) = tx { op_storage.completed(tx); } - #[cfg(debug_assertions)] + #[cfg(any(debug_assertions, test))] { let OpError::InvalidStateTransition { tx, state, trace } = err else { tracing::error!("Finished transaction with error: {err}"); @@ -556,7 +564,7 @@ async fn report_result( tracing::error!(%tx, ?state, "Wrong state"); eprintln!("Operation error trace:\n{trace}"); } - #[cfg(not(debug_assertions))] + #[cfg(not(any(debug_assertions, test)))] { tracing::debug!("Finished transaction with error: {err}"); } diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index d896910d3..fa5a9a2c4 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -613,17 +613,17 @@ impl SimNetwork { tracing::info!("Number of simulated nodes: {num_nodes}"); - let missing_percent = 1.0 - (missing.len() as f64 / num_nodes as f64); - if missing_percent > percent { + let missing_percent = (num_nodes - missing.len()) as f64 / num_nodes as f64; + if missing_percent > (percent + 0.01/* 1% error tolerance */) { missing.sort(); let show_max = missing.len().min(100); + tracing::error!("Nodes without connection: {:?}(..)", &missing[..show_max],); tracing::error!( - "Nodes without connection: {:?}(..) ({}% > {}%)", - &missing[..show_max], + "Total nodes without connection: {:?}, ({}% > {}%)", + missing.len(), missing_percent * 100.0, percent * 100.0 ); - tracing::error!("Total nodes without connection: {:?}", missing.len()); anyhow::bail!("found disconnected nodes"); } diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 5c677db3e..0d0f4ad43 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -203,7 +203,7 @@ impl Operation for GetOp { self.state, Some(GetState::AwaitingResponse { .. }) )); - tracing::debug!(tx = %id, "Seek contract {} @ {}", key, target.peer); + tracing::info!(tx = %id, %key, target = %target.peer, "Seek contract"); new_state = self.state; stats = Some(GetStats { contract_location: Location::from(key), @@ -243,17 +243,17 @@ impl Operation for GetOp { if !is_cached_contract { tracing::warn!( tx = %id, - "Contract `{}` not found while processing a get request at node @ {}", - key, - target.peer + %key, + this_peer = %target.peer, + "Contract not found while processing a get request", ); if htl == 0 { tracing::warn!( tx = %id, - "The maximum hops has been exceeded, sending the error \ - back to the node @ {}", - sender.peer + sender = %sender.peer, + "The maximum hops has been exceeded, sending error \ + back to the node", ); return build_op_result( @@ -279,7 +279,12 @@ impl Operation for GetOp { .ring .closest_caching(&key, [&sender.peer].as_slice()) else { - tracing::warn!(tx = %id, "No other peers found while trying getting contract {key} @ {}", target.peer); + tracing::warn!( + tx = %id, + %key, + target = %target.peer, + "No other peers found while trying getting contract", + ); return Err(OpError::RingError(RingError::NoCachingPeers(key))); }; continue_seeking( @@ -373,10 +378,10 @@ impl Operation for GetOp { let this_peer = target; tracing::warn!( tx = %id, + %key, %this_peer, - "Neither contract or contract value for contract `{}` found at peer {}, \ + "Neither contract or contract value for contract found at peer {}, \ retrying with other peers", - key, sender.peer ); @@ -556,7 +561,7 @@ impl Operation for GetOp { contract: contract.clone(), }); } else { - tracing::debug!(tx = %id, "Get response received for contract {}", key); + tracing::info!(tx = %id, %key, "Get response received for contract"); new_state = None; return_msg = None; result = Some(GetResult { @@ -566,7 +571,7 @@ impl Operation for GetOp { } } Some(GetState::ReceivedRequest) => { - tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); + tracing::info!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; return_msg = Some(GetMsg::ReturnGet { id, @@ -719,8 +724,8 @@ pub(crate) async fn request_get( }; tracing::debug!( tx = %id, - "Preparing get contract request to {}", - target.peer, + target = %target.peer, + "Preparing get contract request", ); match get_op.state { diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 2ca1eaa0f..5a6008201 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -223,9 +223,9 @@ impl Operation for PutOp { tracing::debug!( tx = %id, - "Puttting contract {} at target peer {}", - key, - target.peer, + %key, + target = %target.peer, + "Puttting contract at target peer", ); if !is_cached_contract @@ -233,7 +233,7 @@ impl Operation for PutOp { .ring .within_caching_distance(&Location::from(&key)) { - tracing::debug!(tx = %id, "Contract `{}` not cached @ peer {}", key, target.peer); + tracing::debug!(tx = %id, %key, "Contract not cached @ peer {}", target.peer); match try_to_cache_contract(op_storage, contract, &key, client_id).await { Ok(_) => {} Err(err) => return Err(err), @@ -244,8 +244,8 @@ impl Operation for PutOp { // to give back to requesting peer tracing::warn!( tx = %id, - "Contract {} not found while processing info, forwarding", - key + %key, + "Contract not found while processing info, forwarding", ); } @@ -446,9 +446,9 @@ impl Operation for PutOp { } _ => return Err(OpError::invalid_transition(self.id)), }; - tracing::debug!( - "Peer {} completed contract value put", - op_storage.ring.peer_key + tracing::info!( + this_peer = %op_storage.ring.peer_key, + "Peer completed contract value put", ); } PutMsg::PutForward { @@ -461,9 +461,9 @@ impl Operation for PutOp { let peer_loc = op_storage.ring.own_location(); tracing::debug!( - "Forwarding changes at {}, trying put the contract {}", - peer_loc.peer, - key + %key, + this_peer = % peer_loc.peer, + "Forwarding changes, trying put the contract" ); let cached_contract = op_storage.ring.is_contract_cached(&key); diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index cce1bd6b6..0772f4fed 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -188,10 +188,11 @@ impl Operation for SubscribeOp { match self.state { Some(SubscribeState::ReceivedRequest) => { - tracing::debug!( + tracing::info!( tx = %id, - "Peer {} successfully subscribed to contract {key}", - subscriber.peer, + %key, + subscriber = % subscriber.peer, + "Peer successfully subscribed to contract", ); new_state = None; // TODO review behaviour, if the contract is not cached should return subscribed false? @@ -215,8 +216,9 @@ impl Operation for SubscribeOp { } => { tracing::warn!( tx = %id, - "Contract `{key}` not found at potential subscription provider {}", - sender.peer + %key, + potential_provider = %sender.peer, + "Contract not found at potential subscription provider", ); // will error out in case it has reached max number of retries match self.state { @@ -269,15 +271,15 @@ impl Operation for SubscribeOp { } => { match self.state { Some(SubscribeState::AwaitingResponse { .. }) => { - tracing::debug!( + tracing::info!( tx = %id, - target = ?target.peer, - this = ?op_storage.ring.own_location().peer, - "Subscribed to `{key}` at provider {}", sender.peer + %key, + this_peer = %target.peer, + provider = %sender.peer, + "Subscribed to contract" ); op_storage.ring.add_subscription(key.clone()); - // todo: should inform back to the network event loop in case a client - // is waiting for response + // fixme: should inform back to the network event loop in case a client is waiting for response let _ = client_id; new_state = Some(SubscribeState::Completed); return_msg = None; diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index e5e7c9b56..85b3875fb 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -628,7 +628,7 @@ impl Ring { const REMOVAL_TICK_DURATION: Duration = Duration::from_secs(60 * 5); #[cfg(test)] const REMOVAL_TICK_DURATION: Duration = Duration::from_secs(1); - const ACQUIRE_CONNS_TICK_DURATION: Duration = Duration::from_secs(1); + const ACQUIRE_CONNS_TICK_DURATION: Duration = Duration::from_secs(2); const REGENERATE_DENSITY_MAP_INTERVAL: Duration = Duration::from_secs(60); let mut check_interval = tokio::time::interval(REMOVAL_TICK_DURATION); @@ -645,6 +645,10 @@ impl Ring { #[cfg(test)] let retry_interval = Duration::from_secs(5); + // if the peer is just starting wait a bit before + // we even attempt acquiring more connections + tokio::time::sleep(ACQUIRE_CONNS_TICK_DURATION).await; + let mut live_tx = None; 'outer: loop { loop { diff --git a/crates/core/src/runtime/tests/contract.rs b/crates/core/src/runtime/tests/contract.rs index 057eb4777..914a20133 100644 --- a/crates/core/src/runtime/tests/contract.rs +++ b/crates/core/src/runtime/tests/contract.rs @@ -8,7 +8,6 @@ use super::super::Runtime; const TEST_CONTRACT_1: &str = "test_contract_1"; #[test] -#[ignore] fn validate_state() -> Result<(), Box> { let TestSetup { contract_store, @@ -39,7 +38,6 @@ fn validate_state() -> Result<(), Box> { } #[test] -#[ignore] fn validate_delta() -> Result<(), Box> { let TestSetup { contract_store, @@ -68,7 +66,6 @@ fn validate_delta() -> Result<(), Box> { } #[test] -#[ignore] fn update_state() -> Result<(), Box> { let TestSetup { contract_store, @@ -94,7 +91,6 @@ fn update_state() -> Result<(), Box> { } #[test] -#[ignore] fn summarize_state() -> Result<(), Box> { let TestSetup { contract_store, @@ -116,7 +112,6 @@ fn summarize_state() -> Result<(), Box> { } #[test] -#[ignore] fn get_state_delta() -> Result<(), Box> { let TestSetup { contract_store, diff --git a/crates/core/src/runtime/tests/time.rs b/crates/core/src/runtime/tests/time.rs index 1972c5d8d..47dc4c6c3 100644 --- a/crates/core/src/runtime/tests/time.rs +++ b/crates/core/src/runtime/tests/time.rs @@ -5,7 +5,6 @@ use wasmer::TypedFunction; use super::{super::Runtime, TestSetup}; #[test] -#[ignore] fn now() -> Result<(), Box> { let TestSetup { contract_store, diff --git a/crates/core/src/topology.rs b/crates/core/src/topology.rs index 938b3bcdc..fd673109f 100644 --- a/crates/core/src/topology.rs +++ b/crates/core/src/topology.rs @@ -146,7 +146,7 @@ impl TopologyManager { location } Err(_) => { - tracing::warn!( + tracing::debug!( "An error occurred while getting max density, falling back to random location" ); self.random_location() diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index 6033bf3b8..6db3a33a2 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -34,10 +34,10 @@ pub struct TestConfig { /// Events are simulated get, puts and other operations. #[arg(long, default_value_t = usize::MAX)] events: usize, - /// Time in milliseconds to wait for the network to be sufficiently connected - /// to start requesting events. (20% of the network) - #[arg(long, default_value_t = 15_000)] - wait_duration: u64, + /// Time in milliseconds to wait for the network to be sufficiently connected to start requesting events. + /// (20% of the expected connections to be processed per gateway) + #[arg(long)] + wait_duration: Option, /// Time in milliseconds to wait for the next event to be executed. #[arg(long)] event_wait_time: Option, @@ -92,6 +92,12 @@ mod single_process { use tokio::signal; pub(super) async fn run(base_config: &super::TestConfig) -> Result<(), super::Error> { + if base_config.gateways == 0 { + anyhow::bail!("Gateways should be higher than 0"); + } + if base_config.nodes == 0 { + anyhow::bail!("Nodes should be higher than 0"); + } let name = &base_config .name .as_ref() @@ -121,18 +127,32 @@ mod single_process { .await; let events = base_config.events; - let connectivity_timeout = Duration::from_millis(base_config.wait_duration); let next_event_wait_time = base_config.event_wait_time.map(Duration::from_millis); + let (connectivity_timeout, network_connection_percent) = { + let conns_per_gw = (base_config.nodes / base_config.gateways) as f64; + let conn_percent = (conns_per_gw / base_config.nodes as f64).min(0.99); + let connectivity_timeout = + Duration::from_millis(base_config.wait_duration.unwrap_or_else(|| { + // expect a peer to take max 200ms to connect, this should happen in parallel + // but err on the side of safety + (conns_per_gw * 200.0).ceil() as u64 + })); + (connectivity_timeout, conn_percent) + }; let events_generated = tokio::task::spawn_blocking(move || { - // todo: come with a good time_out wait value experimentally, - // on avg how long should it take for the 20% of the network to be connected? - simulated_network.check_partial_connectivity(connectivity_timeout, 0.20)?; + println!( + "Waiting for network to be sufficiently connected ({}ms timeout, {}%)", + connectivity_timeout.as_millis(), + network_connection_percent * 100.0 + ); + simulated_network + .check_partial_connectivity(connectivity_timeout, network_connection_percent)?; for _ in simulated_network.event_chain(events) { if let Some(t) = next_event_wait_time { std::thread::sleep(t); } } - Ok::<_, super::Error>(()) + Ok::<_, super::Error>(simulated_network) }); let join_peer_tasks = async move { @@ -149,6 +169,7 @@ mod single_process { tokio::pin!(join_peer_tasks); tokio::pin!(ctrl_c); + let mut network_sim = None; loop { tokio::select! { _ = &mut ctrl_c /* SIGINT handling */ => { @@ -156,9 +177,10 @@ mod single_process { } res = &mut events_generated => { match res? { - Ok(_) => { + Ok(return_nw) => { println!("Test events generated successfully"); - *events_generated = tokio::task::spawn(futures::future::pending::>()); + network_sim = Some(return_nw); + *events_generated = tokio::task::spawn(futures::future::pending::>()); continue; } Err(err) => { @@ -181,6 +203,8 @@ mod single_process { } } + _ = network_sim; + Ok(()) } } From 931b9f7f19a55c83ea5973cc7cad989dfe0406ff Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 16 Nov 2023 14:35:42 +0100 Subject: [PATCH 06/12] Fix issue with concurrent idx writting --- Cargo.lock | 11 - crates/core/Cargo.toml | 1 - crates/core/src/runtime/contract_store.rs | 18 +- crates/core/src/runtime/delegate_store.rs | 9 +- crates/core/src/runtime/secrets_store.rs | 14 +- crates/core/src/runtime/store.rs | 261 +++++++++++++--------- 6 files changed, 174 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d0f58e57..1f6dc27d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1565,7 +1565,6 @@ dependencies = [ "either", "fastrand 2.0.1", "freenet-stdlib", - "fs4", "futures", "itertools", "libp2p", @@ -1641,16 +1640,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "fs4" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7" -dependencies = [ - "rustix 0.38.24", - "windows-sys 0.48.0", -] - [[package]] name = "fsevent-sys" version = "4.1.0" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index b3dd06990..ea2a92c39 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -35,7 +35,6 @@ delegate = "0.10" directories = "5" either = { features = ["serde"], workspace = true } fastrand = { workspace = true } -fs4 = "0.7" futures = "0.3.21" itertools = "0.11" libp2p = { default-features = false, features = ["autonat", "dns", "ed25519", "identify", "macros", "noise", "ping", "tcp", "tokio", "yamux"], version = "0.52.3" } diff --git a/crates/core/src/runtime/contract_store.rs b/crates/core/src/runtime/contract_store.rs index 4f2367fcf..3f2b5c094 100644 --- a/crates/core/src/runtime/contract_store.rs +++ b/crates/core/src/runtime/contract_store.rs @@ -1,15 +1,14 @@ -use std::{ - fs::{File, OpenOptions}, - io::{BufWriter, Write}, - path::PathBuf, - sync::Arc, -}; +use std::{fs::File, io::Write, path::PathBuf, sync::Arc}; use dashmap::DashMap; use freenet_stdlib::prelude::*; use stretto::Cache; -use super::{error::RuntimeInnerError, store::StoreFsManagement, RuntimeResult}; +use super::{ + error::RuntimeInnerError, + store::{SafeWriter, StoreFsManagement}, + RuntimeResult, +}; /// Handle contract blob storage on the file system. pub struct ContractStore { @@ -17,7 +16,7 @@ pub struct ContractStore { key_file: PathBuf, contract_cache: Cache>>, key_to_code_part: Arc>, - index_file: BufWriter, + index_file: SafeWriter, } // TODO: add functionality to delete old contracts which have not been used for a while // to keep the total space used under a configured threshold @@ -54,8 +53,7 @@ impl ContractStore { } Self::watch_changes(key_to_code_part.clone(), &key_file)?; - let index_file = - std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(&key_file)?); + let index_file = SafeWriter::new(&key_file, false)?; Ok(Self { contract_cache: Cache::new(100, max_size).expect(ERR), contracts_dir, diff --git a/crates/core/src/runtime/delegate_store.rs b/crates/core/src/runtime/delegate_store.rs index 0309e5694..5540ce428 100644 --- a/crates/core/src/runtime/delegate_store.rs +++ b/crates/core/src/runtime/delegate_store.rs @@ -3,11 +3,11 @@ use freenet_stdlib::prelude::{ APIVersion, CodeHash, Delegate, DelegateCode, DelegateContainer, DelegateKey, DelegateWasmAPIVersion, Parameters, }; -use std::fs::OpenOptions; -use std::io::BufWriter; use std::{fs::File, io::Write, path::PathBuf, sync::Arc}; use stretto::Cache; +use crate::runtime::store::SafeWriter; + use super::store::StoreFsManagement; use super::RuntimeResult; @@ -15,7 +15,7 @@ pub struct DelegateStore { delegates_dir: PathBuf, delegate_cache: Cache>, key_to_code_part: Arc>, - index_file: BufWriter, + index_file: SafeWriter, key_file: PathBuf, } @@ -51,8 +51,7 @@ impl DelegateStore { } Self::watch_changes(key_to_code_part.clone(), &key_file)?; - let index_file = - std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(&key_file)?); + let index_file = SafeWriter::new(&key_file, false)?; Ok(Self { delegate_cache: Cache::new(100, max_size).expect(ERR), delegates_dir, diff --git a/crates/core/src/runtime/secrets_store.rs b/crates/core/src/runtime/secrets_store.rs index 93931318f..d5694b82a 100644 --- a/crates/core/src/runtime/secrets_store.rs +++ b/crates/core/src/runtime/secrets_store.rs @@ -1,7 +1,7 @@ use std::{ collections::{HashMap, HashSet}, - fs::{self, File, OpenOptions}, - io::{BufWriter, Write}, + fs::{self, File}, + io::Write, path::PathBuf, sync::Arc, }; @@ -12,7 +12,10 @@ use dashmap::DashMap; use freenet_stdlib::{client_api::DelegateRequest, prelude::*}; use once_cell::sync::Lazy; -use super::{store::StoreFsManagement, RuntimeResult}; +use super::{ + store::{SafeWriter, StoreFsManagement}, + RuntimeResult, +}; type SecretKey = [u8; 32]; @@ -38,7 +41,7 @@ pub struct SecretsStore { base_path: PathBuf, ciphers: HashMap, key_to_secret_part: Arc)>>, - index_file: BufWriter, + index_file: SafeWriter, key_file: PathBuf, } @@ -119,8 +122,7 @@ impl SecretsStore { } Self::watch_changes(key_to_secret_part.clone(), &key_file)?; - let index_file = - std::io::BufWriter::new(OpenOptions::new().append(true).read(true).open(&key_file)?); + let index_file = SafeWriter::new(&key_file, false)?; Ok(Self { base_path: secrets_dir, ciphers: HashMap::new(), diff --git a/crates/core/src/runtime/store.rs b/crates/core/src/runtime/store.rs index 84f7406b2..a6fe62b15 100644 --- a/crates/core/src/runtime/store.rs +++ b/crates/core/src/runtime/store.rs @@ -3,8 +3,8 @@ use either::Either; use freenet_stdlib::prelude::{CodeHash, ContractInstanceId, DelegateKey}; use notify::Watcher; use std::fs::{self, OpenOptions}; -use std::io::{BufReader, BufWriter, Seek, Write}; -use std::path::Path; +use std::io::{self, BufReader, BufWriter, Seek, Write}; +use std::path::{Path, PathBuf}; use std::time::Duration; use std::{fs::File, io::Read}; @@ -13,6 +13,82 @@ use crate::DynError; const INTERNAL_KEY: usize = 32; const TOMBSTONE_MARKER: usize = 1; +pub(super) struct SafeWriter { + file: BufWriter, + lock_file_path: PathBuf, + compact: bool, + _marker: std::marker::PhantomData ()>, +} + +impl SafeWriter { + pub fn new(path: &Path, compact: bool) -> Result { + let file = if compact { + OpenOptions::new().create(true).write(true).open(path)? + } else { + OpenOptions::new() + .create(true) + .read(true) + .append(true) + .open(path)? + }; + let s = Self { + file: BufWriter::new(file), + _marker: std::marker::PhantomData, + compact, + lock_file_path: path.with_extension("lock"), + }; + Ok(s) + } + + /// Inserts a new record and returns the offset + fn insert_record(&mut self, key: StoreKey, value: &[u8]) -> std::io::Result { + self.check_lock(); + // The full key is the tombstone marker byte + kind + [internal key content] + size of value + self.file.write_u8(false as u8)?; + let mut traversed = 1; + match key { + StoreKey::ContractKey(key) => { + self.file.write_u8(KeyType::Contract as u8)?; + self.file.write_all(&key)?; + } + StoreKey::DelegateKey { key, code_hash } => { + self.file.write_u8(KeyType::Delegate as u8)?; + self.file.write_all(&key)?; + self.file.write_all(&code_hash)?; + traversed += 32; // additional code_hash bytes + } + } + traversed += 1 + 32; // key + type marker + self.file + .write_u32::(value.as_ref().len() as u32)?; + traversed += std::mem::size_of::(); + self.file.write_all(value)?; + traversed += value.len(); + self.file.flush()?; + let current_offset = self.file.stream_position()?; + Ok(current_offset - traversed as u64) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.check_lock(); + self.file.flush() + } +} + +impl SafeWriter { + fn check_lock(&self) { + while !self.compact && self.lock_file_path.exists() { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +impl Drop for SafeWriter { + fn drop(&mut self) { + self.check_lock(); + } +} + #[derive(Debug)] pub(super) enum StoreKey { ContractKey([u8; INTERNAL_KEY]), @@ -61,7 +137,7 @@ enum KeyType { Delegate = 1, } -pub(super) trait StoreFsManagement { +pub(super) trait StoreFsManagement: Sized { type MemContainer: Send + Sync + 'static; type Key: Clone + From; type Value: AsRef<[u8]> + for<'x> TryFrom<&'x [u8], Error = std::io::Error>; @@ -94,7 +170,7 @@ pub(super) trait StoreFsManagement { )?; std::thread::spawn(move || loop { std::thread::sleep(Duration::from_secs(5 * 60)); - if let Err(err) = compact_index_file(&key_path) { + if let Err(err) = compact_index_file::(&key_path) { tracing::warn!("Failed index file ({key_path:?}) compaction: {err}"); } }); @@ -104,7 +180,7 @@ pub(super) trait StoreFsManagement { /// Insert in index file and returns the offset at which this record resides. fn insert( - file: &mut BufWriter, + file: &mut SafeWriter, key: Self::Key, value: &Self::Value, ) -> std::io::Result @@ -113,7 +189,7 @@ pub(super) trait StoreFsManagement { { // The full key is the tombstone marker byte + kind + [internal key content] + size of value let internal_key: StoreKey = key.into(); - let offset = insert_record(file, internal_key, value.as_ref())?; + let offset = file.insert_record(internal_key, value.as_ref())?; Ok(offset) } @@ -149,33 +225,6 @@ pub(super) trait StoreFsManagement { } } -/// Inserts a new record and returns the offset -fn insert_record(file: &mut BufWriter, key: StoreKey, value: &[u8]) -> std::io::Result { - // The full key is the tombstone marker byte + kind + [internal key content] + size of value - file.write_u8(false as u8)?; - let mut traversed = 1; - match key { - StoreKey::ContractKey(key) => { - file.write_u8(KeyType::Contract as u8)?; - file.write_all(&key)?; - } - StoreKey::DelegateKey { key, code_hash } => { - file.write_u8(KeyType::Delegate as u8)?; - file.write_all(&key)?; - file.write_all(&code_hash)?; - traversed += 32; // additional code_hash bytes - } - } - traversed += 1 + 32; // key + type marker - file.write_u32::(value.as_ref().len() as u32)?; - traversed += std::mem::size_of::(); - file.write_all(value)?; - traversed += value.len(); - file.flush()?; - let current_offset = file.stream_position()?; - Ok(current_offset - traversed as u64) -} - #[allow(clippy::type_complexity)] fn process_record( reader: &mut BufReader, @@ -242,36 +291,44 @@ where } } -fn compact_index_file(key_file_path: &Path) -> std::io::Result<()> { - use fs4::FileExt; +fn compact_index_file(key_file_path: &Path) -> std::io::Result<()> { + // Define the path to the lock file + let lock_file_path = key_file_path.with_extension("lock"); + + // Attempt to create the lock file + let mut opts = fs::OpenOptions::new(); + opts.create_new(true).write(true); + match opts.open(&lock_file_path) { + Ok(_) => { + // The lock file was created successfully, so a compaction can proceed + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // The lock file already exists, so a compaction is in progress + println!("locked"); + return Ok(()); + } + Err(e) => { + // An unexpected error occurred + return Err(e); + } + } let original_file = OpenOptions::new() .truncate(false) .read(true) .open(key_file_path)?; - // Lock the original file exclusively - if original_file.try_lock_exclusive().is_err() { - return Ok(()); - } - // Create a new temporary file to write compacted data let temp_file_path = key_file_path.with_extension("tmp"); - let temp_file = OpenOptions::new() - .create(true) - .write(true) - .open(&temp_file_path) - .map_err(|e| { - let _ = original_file.unlock(); - e - })?; - - // Lock the temporary file exclusively - temp_file.try_lock_exclusive()?; // Read the original file and compact data into the temp file let mut original_reader = BufReader::new(original_file); - let mut temp_writer = BufWriter::new(temp_file); + let mut temp_writer = SafeWriter::::new(&temp_file_path, true).map_err(|e| { + if let Err(e) = fs::remove_file(&lock_file_path) { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + } + e + })?; let mut any_deleted = false; // Track if any deleted records were found @@ -282,11 +339,10 @@ fn compact_index_file(key_file_path: &Path) -> std::io::Result<()> { Either::Left(v) => v.as_slice(), Either::Right(v) => v.as_slice(), }; - if let Err(err) = insert_record(&mut temp_writer, store_key, value) { - // Handle the error gracefully - let _ = original_reader.into_inner().unlock(); - let _ = temp_writer.into_inner().map(|f| f.unlock()); - let _ = fs::remove_file(&temp_file_path); + if let Err(err) = temp_writer.insert_record(store_key, value) { + if let Err(e) = fs::remove_file(&lock_file_path) { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + } return Err(err); } } @@ -300,9 +356,9 @@ fn compact_index_file(key_file_path: &Path) -> std::io::Result<()> { } Err(other) => { // Handle other errors gracefully - let _ = original_reader.into_inner().unlock(); - let _ = temp_writer.into_inner().map(|f| f.unlock()); - let _ = fs::remove_file(&temp_file_path); + if let Err(e) = fs::remove_file(&lock_file_path) { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + } return Err(other); } } @@ -310,32 +366,40 @@ fn compact_index_file(key_file_path: &Path) -> std::io::Result<()> { // Check if any deleted records were found; if not, skip compaction if !any_deleted { - let _ = original_reader.into_inner().unlock(); - let _ = temp_writer.into_inner().map(|f| f.unlock()); - let _ = fs::remove_file(&temp_file_path); + if let Err(e) = fs::remove_file(&lock_file_path) { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + } return Ok(()); } // Clean up and finalize the compaction process - let original_file = original_reader.into_inner(); - original_file.unlock()?; - temp_writer.flush()?; - let _ = temp_writer.into_inner().map(|f| f.unlock()); - std::mem::drop(original_file); + if let Err(e) = temp_writer.flush() { + if let Err(e) = fs::remove_file(&lock_file_path) { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + } + return Err(e); + } // Replace the original file with the temporary file - fs::rename(&temp_file_path, key_file_path)?; - let _ = fs::remove_file(&temp_file_path); + if let Err(e) = fs::rename(&temp_file_path, key_file_path) { + if let Err(e) = fs::remove_file(&lock_file_path) { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + } + return Err(e); + } + + // Remove the lock file + fs::remove_file(&lock_file_path).map_err(|e| { + eprintln!("{}:{}: Failed to remove lock file: {e}", file!(), line!()); + e + })?; Ok(()) } #[cfg(test)] mod tests { - use std::{ - fs::OpenOptions, - sync::{Arc, Barrier}, - }; + use std::sync::{Arc, Barrier}; use crate::util::tests::get_temp_dir; @@ -387,22 +451,8 @@ mod tests { // Test the update function { - let mut file_1 = BufWriter::new( - OpenOptions::new() - .create(true) - .read(true) - .append(true) - .open(&contract_keys_file_path) - .expect("Failed to open key file"), - ); - let mut file_2 = BufWriter::new( - OpenOptions::new() - .create(true) - .read(true) - .append(true) - .open(&delegate_keys_file_path) - .expect("Failed to open key file"), - ); + let mut file_1 = SafeWriter::new(&contract_keys_file_path, false).expect("failed"); + let mut file_2 = SafeWriter::new(&delegate_keys_file_path, false).expect("failed"); let container_1 = ::MemContainer::default(); let container_2 = ::MemContainer::default(); @@ -465,13 +515,8 @@ mod tests { for i in [0, 10, 20, 30] { let shared_data = container.clone(); let barrier = barrier.clone(); - let mut file = BufWriter::new( - OpenOptions::new() - .read(true) - .append(true) - .open(&temp_dir.path().join("contract_keys")) - .expect("Failed to open key file"), - ); + let path = &temp_dir.path().join("contract_keys"); + let mut file = SafeWriter::new(path, false).expect("failed"); let key_file_path = contract_keys_file_path.clone(); let handle = std::thread::spawn(move || { // Wait for all threads to reach this point @@ -514,6 +559,12 @@ mod tests { #[test] fn test_concurrent_compaction() { + for _ in 0..100 { + concurrent_compaction(); + } + } + + fn concurrent_compaction() { let temp_dir = get_temp_dir(); let key_file_path = temp_dir.path().join("data.dat"); std::fs::File::create(&key_file_path).expect("Failed to create file"); @@ -530,25 +581,21 @@ mod tests { let key_file_path = key_file_path.clone(); let barrier = barrier.clone(); let shared_data = container.clone(); - let mut file = BufWriter::new( - OpenOptions::new() - .read(true) - .append(true) - .open(&key_file_path) - .expect("Failed to open key file"), - ); + let mut file = SafeWriter::new(&key_file_path, false).expect("failed"); std::thread::spawn(move || { barrier.wait(); // concurrently creates/removes some data and compacts if [10, 30].contains(&i) { create_test_data(&mut file, &key_file_path, shared_data, i); - } else if let Err(err) = super::compact_index_file(&key_file_path) { + } else if let Err(err) = super::compact_index_file::(&key_file_path) + { eprintln!("Thread encountered an error during compaction: {err}"); return Err(err); } barrier.wait(); + println!("next compaction (this should print 4 times"); // compact a last time so we know what data to compare against - super::compact_index_file(&key_file_path).map_err(|err| { + super::compact_index_file::(&key_file_path).map_err(|err| { eprintln!("Thread encountered an error during compaction: {err}"); err }) @@ -594,7 +641,7 @@ mod tests { } fn create_test_data( - file: &mut BufWriter, + file: &mut SafeWriter, test_path: &Path, shared_data: ::MemContainer, thread: u8, From 72b7529cd644c5636384b213510757ffb585224b Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 17 Nov 2023 11:33:46 +0100 Subject: [PATCH 07/12] Refactor so we can have multiprocess nodes for testing --- Cargo.lock | 14 +- crates/core/Cargo.toml | 2 +- crates/core/examples/gw_key | 2 - crates/core/examples/p2p_network.rs | 198 ---------------- crates/core/src/client_events.rs | 4 +- crates/core/src/config.rs | 8 +- crates/core/src/contract/executor.rs | 2 +- crates/core/src/lib.rs | 12 +- crates/core/src/message.rs | 22 +- crates/core/src/node.rs | 78 ++++--- crates/core/src/node/network_bridge.rs | 15 +- .../core/src/node/network_bridge/in_memory.rs | 11 +- .../src/node/network_bridge/inter_process.rs | 20 ++ .../src/node/network_bridge/p2p_protoc.rs | 71 +++--- crates/core/src/node/network_event_log.rs | 26 +-- crates/core/src/node/op_state_manager.rs | 12 +- crates/core/src/node/p2p_impl.rs | 36 ++- .../src/node/{tests.rs => testing_impl.rs} | 220 ++++++++++++------ .../in_memory.rs} | 172 +++++++------- .../src/node/testing_impl/inter_process.rs | 32 +++ crates/core/src/operations.rs | 7 +- crates/core/src/operations/connect.rs | 21 +- crates/core/src/operations/get.rs | 30 +-- crates/core/src/operations/put.rs | 20 +- crates/core/src/operations/subscribe.rs | 20 +- crates/core/src/operations/update.rs | 2 +- crates/core/src/ring.rs | 19 +- crates/core/src/router.rs | 1 - crates/core/src/router/util.rs | 10 - crates/core/src/runtime/store.rs | 2 +- crates/fdev/src/testing.rs | 122 ++++++---- crates/fdev/src/wasm_runtime/user_events.rs | 27 ++- 32 files changed, 620 insertions(+), 618 deletions(-) delete mode 100644 crates/core/examples/gw_key delete mode 100644 crates/core/examples/p2p_network.rs create mode 100644 crates/core/src/node/network_bridge/inter_process.rs rename crates/core/src/node/{tests.rs => testing_impl.rs} (81%) rename crates/core/src/node/{in_memory_impl.rs => testing_impl/in_memory.rs} (85%) create mode 100644 crates/core/src/node/testing_impl/inter_process.rs diff --git a/Cargo.lock b/Cargo.lock index 1f6dc27d1..47c9c74af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1566,7 +1566,8 @@ dependencies = [ "fastrand 2.0.1", "freenet-stdlib", "futures", - "itertools", + "itertools 0.11.0", + "itertools 0.12.0", "libp2p", "libp2p-identity", "notify", @@ -2289,6 +2290,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -4852,7 +4862,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" dependencies = [ - "itertools", + "itertools 0.11.0", "nom", "unicode_categories", ] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index ea2a92c39..a91942bb0 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -74,7 +74,7 @@ freenet-stdlib = { features = ["net"], workspace = true } arbitrary = { features = ["derive"], version = "1" } chrono = { features = ["arbitrary"], workspace = true } freenet-stdlib = { features = ["net", "testing"], workspace = true } -itertools = "0.11" +itertools = "0.12" pav_regression = "0.4.0" pico-args = "0.5" statrs = "0.16.0" diff --git a/crates/core/examples/gw_key b/crates/core/examples/gw_key deleted file mode 100644 index c0349160a..000000000 --- a/crates/core/examples/gw_key +++ /dev/null @@ -1,2 +0,0 @@ -Mç‘æß13ãÅ -Òïå × ùƒP™ûâÒP± ñ—ï+ñ–†]¹ÒÎs ÚhÛYE€ÅÏyTPí-M¦#}œ¥¼þm \ No newline at end of file diff --git a/crates/core/examples/p2p_network.rs b/crates/core/examples/p2p_network.rs deleted file mode 100644 index 66ea6ec8c..000000000 --- a/crates/core/examples/p2p_network.rs +++ /dev/null @@ -1,198 +0,0 @@ -use std::{net::Ipv4Addr, sync::Arc, time::Duration}; - -use anyhow::{anyhow, bail}; -use clap::Parser; -use freenet::network_sim::*; -use freenet_stdlib::{ - client_api::{ClientError, ClientRequest, ContractRequest, HostResponse}, - prelude::*, -}; -use futures::future::BoxFuture; -use libp2p::{ - identity::{ed25519, Keypair}, - PeerId, -}; -use tokio::sync::mpsc::{channel, Receiver, Sender}; - -const ENCODED_GW_KEY: &[u8] = include_bytes!("gw_key"); - -async fn start_gateway( - key: Keypair, - port: u16, - location: Location, - user_events: UserEvents, -) -> Result<(), Box> { - // todo: send user events though ws interface - let mut builder = NodeBuilder::new([Box::new(user_events)]); - builder - .with_ip(Ipv4Addr::LOCALHOST) - .with_port(port) - .with_key(key) - .with_location(location); - let config = NodeConfig::parse(); - builder.build(config).await?.run().await -} - -async fn start_new_peer( - gateway_config: InitPeerNode, - user_events: UserEvents, -) -> Result<(), Box> { - // todo: send user events though ws interface - let mut builder = NodeBuilder::new([Box::new(user_events)]); - builder.add_gateway(gateway_config); - let config = NodeConfig::parse(); - builder.build(config).await?.run().await -} - -async fn run_test(manager: EventManager) -> Result<(), anyhow::Error> { - let contract = ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(vec![7, 3, 9, 5])), - Parameters::from(vec![]), - ))); - let key = contract.key().clone(); - let init_val = WrappedState::new(vec![1, 2, 3, 4]); - - tokio::time::sleep(Duration::from_secs(10)).await; - manager - .tx_gw_ev - .send( - ContractRequest::Put { - state: init_val, - contract: contract.clone(), - related_contracts: Default::default(), - } - .into(), - ) - .await - .map_err(|_| anyhow!("channel closed"))?; - tokio::time::sleep(Duration::from_secs(10)).await; - - manager - .tx_gw_ev - .send( - ContractRequest::Get { - key, - fetch_contract: false, - } - .into(), - ) - .await - .map_err(|_| anyhow!("channel closed"))?; - tokio::time::sleep(Duration::from_secs(10)).await; - - let second_val = WrappedState::new(vec![2, 3, 1, 4]); - manager - .tx_node_ev - .send( - ContractRequest::Put { - state: second_val, - contract, - related_contracts: Default::default(), - } - .into(), - ) - .await - .map_err(|_| anyhow!("channel closed"))?; - tokio::time::sleep(Duration::from_secs(300)).await; - Ok(()) -} - -#[tokio::main(worker_threads = 2)] -async fn main() -> Result<(), anyhow::Error> { - let args = Args::parse_args()?; - - let gw_port = 64510; - let gw_key: Keypair = Keypair::try_from(ed25519::Keypair::try_from_bytes( - &mut ENCODED_GW_KEY.to_vec(), - )?)?; - let gw_id: PeerId = gw_key.public().into(); - let gw_loc = Location::random(); - let gw_config = InitPeerNode::new(gw_id, gw_loc) - .listening_ip(Ipv4Addr::LOCALHOST) - .listening_port(gw_port); - - let (tx_gw_ev, rx_gw_ev) = channel(100); - let (tx_node_ev, rx_node_ev) = channel(100); - let manager = EventManager { - tx_gw_ev, - tx_node_ev, - }; - - match (args.is_gw, args.is_peer) { - (true, true) => bail!("a node cannot be both a gateway and a normal peer"), - (true, _) => { - tokio::spawn(start_gateway( - gw_key, - gw_port, - gw_loc, - UserEvents { rx_ev: rx_gw_ev }, - )); - run_test(manager).await - } - (_, true) => { - tokio::spawn(start_new_peer(gw_config, UserEvents { rx_ev: rx_node_ev })); - run_test(manager).await - } - _ => { - tokio::spawn(start_gateway( - gw_key, - gw_port, - gw_loc, - UserEvents { rx_ev: rx_gw_ev }, - )); - tokio::time::sleep(Duration::from_millis(100)).await; - tokio::spawn(start_new_peer(gw_config, UserEvents { rx_ev: rx_node_ev })); - run_test(manager).await - } - } -} - -#[derive(Clone)] -struct EventManager { - tx_gw_ev: Sender>, - tx_node_ev: Sender>, -} - -struct UserEvents { - rx_ev: Receiver>, -} - -impl ClientEventsProxy for UserEvents { - /// # Cancellation Safety - /// This future must be safe to cancel. - fn recv(&mut self) -> BoxFuture<'_, Result, ClientError>> { - Box::pin(async move { - Ok(OpenRequest::new( - ClientId::FIRST, - Box::new(self.rx_ev.recv().await.expect("channel open")), - )) - }) - } - - /// Sends a response from the host to the client application. - fn send( - &mut self, - _id: ClientId, - _response: Result, - ) -> BoxFuture<'_, Result<(), ClientError>> { - Box::pin(async move { - tracing::info!("received response"); - Ok(()) - }) - } -} - -struct Args { - is_gw: bool, - is_peer: bool, -} - -impl Args { - fn parse_args() -> Result { - let mut pargs = pico_args::Arguments::from_env(); - Ok(Args { - is_gw: pargs.contains("--gateway"), - is_peer: pargs.contains("--peer"), - }) - } -} diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index d384e663e..8d87d5edd 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -164,11 +164,11 @@ pub(crate) mod test { use rand::{seq::SliceRandom, SeedableRng}; use tokio::sync::watch::Receiver; - use crate::node::{tests::EventId, PeerKey}; + use crate::node::{testing_impl::EventId, PeerKey}; use super::*; - pub(crate) struct MemoryEventsGen { + pub struct MemoryEventsGen { id: PeerKey, signal: Receiver<(EventId, PeerKey)>, events_to_gen: HashMap>, diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 74478077e..1c05a8703 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -1,4 +1,3 @@ -#![allow(unused)] // FIXME: remove unused use std::{ convert::TryFrom, fs::{self, File}, @@ -58,6 +57,7 @@ pub struct Config { local_mode: AtomicBool, #[cfg(feature = "websocket")] + #[allow(unused)] pub(crate) ws: WebSocketApiConfig, } @@ -102,7 +102,6 @@ pub struct ConfigPaths { delegates_dir: PathBuf, secrets_dir: PathBuf, db_dir: PathBuf, - app_data_dir: PathBuf, event_log: PathBuf, } @@ -158,7 +157,6 @@ impl ConfigPaths { delegates_dir, secrets_dir, db_dir, - app_data_dir, event_log, }) } @@ -375,8 +373,7 @@ pub fn set_logger() { #[cfg(feature = "trace")] pub(super) mod tracer { - use tracing::Subscriber; - use tracing_subscriber::{fmt, layer::Layered, Layer, Registry}; + use tracing_subscriber::{Layer, Registry}; use crate::DynError; @@ -396,7 +393,6 @@ pub(super) mod tracer { // use opentelemetry_sdk::propagation::TraceContextPropagator; use tracing_subscriber::layer::SubscriberExt; - use tracing_subscriber::Registry; let disabled_logs = std::env::var("FREENET_DISABLE_LOGS").is_ok(); let layers = { diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index f04cd0bc8..e0b0a7e71 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -260,7 +260,7 @@ pub struct Executor { ))] mode: OperationMode, runtime: R, - state_store: StateStore, + pub state_store: StateStore, update_notifications: HashMap)>>, subscriber_summaries: HashMap>>>, delegate_attested_ids: HashMap>, diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 214de36bc..d305b300a 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -23,19 +23,13 @@ pub mod local_node { pub use node::NodeConfig; } -/// Exports to build a running network simulation. -pub mod network_sim { - use super::*; - pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; - pub use node::{tests::SimNetwork, InitPeerNode, NodeBuilder, NodeConfig}; - pub use ring::Location; -} - /// Exports for the dev tool. pub mod dev_tool { use super::*; pub use crate::config::Config; - pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; + pub use client_events::{test::MemoryEventsGen, ClientEventsProxy, ClientId, OpenRequest}; pub use contract::{storages::Storage, Executor, OperationMode}; + pub use node::{testing_impl::SimNetwork, InitPeerNode, NodeBuilder, NodeConfig, PeerKey}; + pub use ring::Location; pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 4085170be..7c790b6ad 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -197,7 +197,7 @@ mod sealed_msg_type { macro_rules! transaction_type_enumeration { (decl struct { $( $var:tt -> $ty:tt),+ }) => { $( - impl From<$ty> for Message { + impl From<$ty> for NetMessage { fn from(msg: $ty) -> Self { Self::$var(msg) } @@ -222,7 +222,7 @@ mod sealed_msg_type { } #[derive(Debug, Serialize, Deserialize)] -pub(crate) enum Message { +pub(crate) enum NetMessage { Connect(ConnectMsg), Put(PutMsg), Get(GetMsg), @@ -232,7 +232,7 @@ pub(crate) enum Message { Aborted(Transaction), } -pub(crate) trait InnerMessage: Into { +pub(crate) trait InnerMessage: Into { fn id(&self) -> &Transaction; fn target(&self) -> Option<&PeerKeyLocation>; @@ -283,9 +283,9 @@ impl Display for NodeEvent { } } -impl Message { +impl NetMessage { pub fn id(&self) -> &Transaction { - use Message::*; + use NetMessage::*; match self { Connect(op) => op.id(), Put(op) => op.id(), @@ -297,7 +297,7 @@ impl Message { } pub fn target(&self) -> Option<&PeerKeyLocation> { - use Message::*; + use NetMessage::*; match self { Connect(op) => op.target(), Put(op) => op.target(), @@ -310,7 +310,7 @@ impl Message { /// Is the last expected message for this chain of messages. pub fn terminal(&self) -> bool { - use Message::*; + use NetMessage::*; match self { Connect(op) => op.terminal(), Put(op) => op.terminal(), @@ -322,7 +322,7 @@ impl Message { } pub fn requested_location(&self) -> Option { - use Message::*; + use NetMessage::*; match self { Connect(op) => op.requested_location(), Put(op) => op.requested_location(), @@ -334,14 +334,14 @@ impl Message { } pub fn track_stats(&self) -> bool { - use Message::*; + use NetMessage::*; !matches!(self, Connect(_) | Subscribe(_) | Aborted(_)) } } -impl Display for Message { +impl Display for NetMessage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use Message::*; + use NetMessage::*; write!(f, "Message {{")?; match self { Connect(msg) => msg.fmt(f)?, diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 50a148c1a..8c5661486 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -18,9 +18,9 @@ use std::{ use either::Either; use freenet_stdlib::client_api::{ClientRequest, ContractRequest, ErrorKind}; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; +use serde::{Deserialize, Serialize}; use tracing::Instrument; -use self::in_memory_impl::NodeInMemory; use self::{network_event_log::NetEventLog, p2p_impl::NodeP2P}; use crate::{ client_events::{BoxedClient, ClientEventsProxy, ClientId, OpenRequest}, @@ -30,7 +30,7 @@ use crate::{ ClientResponses, ClientResponsesSender, ContractError, ExecutorToEventLoopChannel, NetworkContractHandler, NetworkEventListenerHalve, OperationMode, }, - message::{Message, NodeEvent, Transaction, TransactionType}, + message::{NetMessage, NodeEvent, Transaction, TransactionType}, operations::{ connect::{self, ConnectMsg, ConnectOp}, get, put, subscribe, OpEnum, OpError, OpOutcome, @@ -43,15 +43,16 @@ use crate::{ use crate::operations::handle_op_request; pub(crate) use network_bridge::{ConnectionError, EventLoopNotificationsSender, NetworkBridge}; +#[cfg(feature = "trace-ot")] +pub(crate) use network_event_log::CombinedRegister; pub(crate) use network_event_log::{EventRegister, NetEventRegister}; pub(crate) use op_state_manager::{OpManager, OpNotAvailable}; -mod in_memory_impl; mod network_bridge; mod network_event_log; mod op_state_manager; mod p2p_impl; -pub(crate) mod tests; +pub(crate) mod testing_impl; #[derive(clap::Parser, Clone, Debug)] pub struct NodeConfig { @@ -89,9 +90,10 @@ impl Node { /// /// If both are provided but also additional peers are added via the [`Self::add_gateway()`] method, this node will /// be listening but also try to connect to an existing peer. -pub struct NodeBuilder { - /// local peer private key in - pub(crate) local_key: identity::Keypair, +#[derive(Serialize, Deserialize)] +pub struct NodeBuilder { + /// public key for the peer + pub(crate) public_key: PeerKey, // optional local info, in case this is an initial bootstrap node /// IP to bind to the listener pub(crate) local_ip: Option, @@ -110,14 +112,13 @@ pub struct NodeBuilder { pub(crate) rnd_if_htl_above: Option, pub(crate) max_number_conn: Option, pub(crate) min_number_conn: Option, - pub(crate) clients: [BoxedClient; CLIENTS], } -impl NodeBuilder { - pub fn new(clients: [BoxedClient; CLIENTS]) -> NodeBuilder { - let local_key = Config::conf().local_peer_keypair.clone(); +impl NodeBuilder { + pub fn new() -> NodeBuilder { + let local_key = Config::conf().local_peer_keypair.public().into(); NodeBuilder { - local_key, + public_key: local_key, remote_nodes: Vec::with_capacity(1), local_ip: None, local_port: None, @@ -128,7 +129,6 @@ impl NodeBuilder { rnd_if_htl_above: None, max_number_conn: None, min_number_conn: None, - clients, } } @@ -164,8 +164,8 @@ impl NodeBuilder { /// Optional identity key of this node. /// If not provided it will be either obtained from the configuration or freshly generated. - pub fn with_key(&mut self, key: identity::Keypair) -> &mut Self { - self.local_key = key; + pub fn with_key(&mut self, key: PeerKey) -> &mut Self { + self.public_key = key; self } @@ -181,11 +181,16 @@ impl NodeBuilder { } /// Builds a node using the default backend connection manager. - pub async fn build(self, config: NodeConfig) -> Result { + pub async fn build( + self, + config: NodeConfig, + clients: [BoxedClient; CLIENTS], + private_key: identity::Keypair, + ) -> Result { let event_register = { #[cfg(feature = "trace-ot")] { - use super::node::network_event_log::{CombinedRegister, OTEventRegister}; + use super::node::network_event_log::OTEventRegister; CombinedRegister::new([ Box::new(EventRegister::new()), Box::new(OTEventRegister::new()), @@ -196,23 +201,28 @@ impl NodeBuilder { EventRegister::new() } }; - let node = - NodeP2P::build::(self, event_register, config) - .await?; + let node = NodeP2P::build::( + self, + private_key, + clients, + event_register, + config, + ) + .await?; Ok(Node(node)) } /// Returns all specified gateways for this peer. Returns an error if the peer is not a gateway /// and no gateways are specified. fn get_gateways(&self) -> Result, anyhow::Error> { - let peer = PeerKey::from(self.local_key.public()); + let peer = self.public_key; let gateways: Vec<_> = self .remote_nodes .iter() .filter_map(|node| { if node.addr.is_some() { Some(PeerKeyLocation { - peer: PeerKey::from(node.identifier), + peer: node.identifier, location: Some(node.location), }) } else { @@ -231,11 +241,17 @@ impl NodeBuilder { } } +impl Default for NodeBuilder { + fn default() -> Self { + Self::new() + } +} + /// Gateway node to bootstrap the network. -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct InitPeerNode { addr: Option, - identifier: PeerId, + identifier: PeerKey, location: Location, } @@ -243,7 +259,7 @@ impl InitPeerNode { pub fn new(identifier: PeerId, location: Location) -> Self { Self { addr: None, - identifier, + identifier: PeerKey(identifier), location, } } @@ -587,7 +603,7 @@ macro_rules! handle_op_not_available { } async fn process_message( - msg: Message, + msg: NetMessage, op_storage: Arc, mut conn_manager: CB, mut event_listener: Box, @@ -605,7 +621,7 @@ async fn process_message( .await; loop { match &msg { - Message::Connect(op) => { + NetMessage::Connect(op) => { // log_handling_msg!("join", op.id(), op_storage); let op_result = handle_op_request::( &op_storage, @@ -625,7 +641,7 @@ async fn process_message( ) .await; } - Message::Put(op) => { + NetMessage::Put(op) => { // log_handling_msg!("put", *op.id(), op_storage); let op_result = handle_op_request::( &op_storage, @@ -645,7 +661,7 @@ async fn process_message( ) .await; } - Message::Get(op) => { + NetMessage::Get(op) => { // log_handling_msg!("get", op.id(), op_storage); let op_result = handle_op_request::( &op_storage, @@ -665,7 +681,7 @@ async fn process_message( ) .await; } - Message::Subscribe(op) => { + NetMessage::Subscribe(op) => { // log_handling_msg!("subscribe", op.id(), op_storage); let op_result = handle_op_request::( &op_storage, @@ -738,7 +754,7 @@ impl<'a> arbitrary::Arbitrary<'a> for PeerKey { } impl PeerKey { - pub(crate) fn random() -> Self { + pub fn random() -> Self { use libp2p::identity::Keypair; PeerKey::from(Keypair::generate_ed25519().public()) } diff --git a/crates/core/src/node/network_bridge.rs b/crates/core/src/node/network_bridge.rs index 5b479d4ea..7ea75b2ea 100644 --- a/crates/core/src/node/network_bridge.rs +++ b/crates/core/src/node/network_bridge.rs @@ -10,10 +10,11 @@ use tokio::sync::mpsc::{self, Receiver, Sender}; use super::PeerKey; use crate::{ client_events::ClientId, - message::{Message, NodeEvent}, + message::{NetMessage, NodeEvent}, }; pub(crate) mod in_memory; +pub(crate) mod inter_process; pub(crate) mod p2p_protoc; // TODO: use this constants when we do real net i/o @@ -30,7 +31,7 @@ pub(crate) trait NetworkBridge: Send + Sync { async fn drop_connection(&mut self, peer: &PeerKey) -> ConnResult<()>; - async fn send(&self, target: &PeerKey, msg: Message) -> ConnResult<()>; + async fn send(&self, target: &PeerKey, msg: NetMessage) -> ConnResult<()>; } #[derive(Debug, thiserror::Error, Serialize, Deserialize)] @@ -85,7 +86,9 @@ impl Clone for ConnectionError { } } -pub(super) struct EventLoopNotifications(Receiver), NodeEvent>>); +pub(super) struct EventLoopNotifications( + Receiver), NodeEvent>>, +); impl EventLoopNotifications { pub fn channel() -> (EventLoopNotifications, EventLoopNotificationsSender) { @@ -98,7 +101,7 @@ impl EventLoopNotifications { } impl Deref for EventLoopNotifications { - type Target = Receiver), NodeEvent>>; + type Target = Receiver), NodeEvent>>; fn deref(&self) -> &Self::Target { &self.0 @@ -113,11 +116,11 @@ impl DerefMut for EventLoopNotifications { #[derive(Clone)] pub(crate) struct EventLoopNotificationsSender( - Sender), NodeEvent>>, + Sender), NodeEvent>>, ); impl Deref for EventLoopNotificationsSender { - type Target = Sender), NodeEvent>>; + type Target = Sender), NodeEvent>>; fn deref(&self) -> &Self::Target { &self.0 diff --git a/crates/core/src/node/network_bridge/in_memory.rs b/crates/core/src/node/network_bridge/in_memory.rs index d69051b08..ae3085403 100644 --- a/crates/core/src/node/network_bridge/in_memory.rs +++ b/crates/core/src/node/network_bridge/in_memory.rs @@ -14,7 +14,7 @@ use tokio::sync::Mutex; use super::{ConnectionError, NetworkBridge, PeerKey}; use crate::{ config::GlobalExecutor, - message::Message, + message::NetMessage, node::{network_event_log::NetEventLog, NetEventRegister, OpManager}, }; @@ -25,7 +25,7 @@ pub(in crate::node) struct MemoryConnManager { pub transport: InMemoryTransport, log_register: Arc>>, op_manager: Arc, - msg_queue: Arc>>, + msg_queue: Arc>>, peer: PeerKey, } @@ -47,7 +47,8 @@ impl MemoryConnManager { let Some(msg) = transport_cp.msg_stack_queue.lock().await.pop() else { continue; }; - let msg_data: Message = bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); + let msg_data: NetMessage = + bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); msg_queue_cp.lock().await.push(msg_data); } }); @@ -61,7 +62,7 @@ impl MemoryConnManager { } } - pub async fn recv(&self) -> Result { + pub async fn recv(&self) -> Result { loop { let mut queue = self.msg_queue.lock().await; let Some(msg) = queue.pop() else { @@ -94,7 +95,7 @@ impl Clone for MemoryConnManager { #[async_trait::async_trait] impl NetworkBridge for MemoryConnManager { - async fn send(&self, target: &PeerKey, msg: Message) -> super::ConnResult<()> { + async fn send(&self, target: &PeerKey, msg: NetMessage) -> super::ConnResult<()> { self.log_register .try_lock() .expect("unique lock") diff --git a/crates/core/src/node/network_bridge/inter_process.rs b/crates/core/src/node/network_bridge/inter_process.rs new file mode 100644 index 000000000..50a399582 --- /dev/null +++ b/crates/core/src/node/network_bridge/inter_process.rs @@ -0,0 +1,20 @@ +use crate::{dev_tool::PeerKey, message::NetMessage}; + +use super::NetworkBridge; + +pub(in crate::node) struct InterProcessConnManager {} + +#[async_trait::async_trait] +impl NetworkBridge for InterProcessConnManager { + async fn send(&self, target: &PeerKey, msg: NetMessage) -> super::ConnResult<()> { + todo!() + } + + async fn add_connection(&mut self, _peer: PeerKey) -> super::ConnResult<()> { + Ok(()) + } + + async fn drop_connection(&mut self, _peer: &PeerKey) -> super::ConnResult<()> { + Ok(()) + } +} diff --git a/crates/core/src/node/network_bridge/p2p_protoc.rs b/crates/core/src/node/network_bridge/p2p_protoc.rs index 977692d6a..af7d69fec 100644 --- a/crates/core/src/node/network_bridge/p2p_protoc.rs +++ b/crates/core/src/node/network_bridge/p2p_protoc.rs @@ -44,7 +44,7 @@ use crate::{ client_events::ClientId, config::{self, GlobalExecutor}, contract::{ClientResponsesSender, ExecutorToEventLoopChannel, NetworkEventListenerHalve}, - message::{Message, NodeEvent, Transaction, TransactionType}, + message::{NetMessage, NodeEvent, Transaction, TransactionType}, node::{ handle_cancelled_op, join_ring_request, network_event_log::NetEventLog, process_message, InitPeerNode, NetEventRegister, NodeBuilder, OpManager, PeerKey, @@ -63,7 +63,7 @@ const CURRENT_PROTOC_VER_STR: &str = "/freenet/0.1.0"; const CURRENT_IDENTIFY_PROTOC_VER: &str = "/id/1.0.0"; fn config_behaviour( - local_key: &Keypair, + private_key: &Keypair, gateways: &[InitPeerNode], _private_addr: &Option, op_manager: Arc, @@ -73,17 +73,19 @@ fn config_behaviour( .filter_map(|p| { p.addr .as_ref() - .map(|addr| (p.identifier, HashSet::from_iter([addr.clone()]))) + .map(|addr| (p.identifier.0, HashSet::from_iter([addr.clone()]))) }) .collect(); - let ident_config = - identify::Config::new(CURRENT_IDENTIFY_PROTOC_VER.to_string(), local_key.public()) - .with_agent_version(CURRENT_AGENT_VER.to_string()); + let ident_config = identify::Config::new( + CURRENT_IDENTIFY_PROTOC_VER.to_string(), + private_key.public(), + ) + .with_agent_version(CURRENT_AGENT_VER.to_string()); let ping = ping::Behaviour::default(); - let peer_id = local_key.public().to_peer_id(); + let peer_id = private_key.public().to_peer_id(); let auto_nat = { let config = autonat::Config { ..Default::default() @@ -91,7 +93,7 @@ fn config_behaviour( let mut behaviour = autonat::Behaviour::new(peer_id, config); for (peer, addr) in gateways.iter().map(|p| (&p.identifier, &p.addr)) { - behaviour.add_server(*peer, addr.clone()); + behaviour.add_server(peer.0, addr.clone()); } behaviour }; @@ -120,7 +122,7 @@ fn multiaddr_from_connection(conn: (IpAddr, u16)) -> Multiaddr { addr } -type P2pBridgeEvent = Either<(PeerKey, Box), NodeEvent>; +type P2pBridgeEvent = Either<(PeerKey, Box), NodeEvent>; pub(crate) struct P2pBridge { active_net_connections: Arc>, @@ -220,7 +222,7 @@ impl NetworkBridge for P2pBridge { Ok(()) } - async fn send(&self, target: &PeerKey, msg: Message) -> super::ConnResult<()> { + async fn send(&self, target: &PeerKey, msg: NetMessage) -> super::ConnResult<()> { self.log_register .try_lock() .expect("single reference") @@ -246,11 +248,12 @@ pub(in crate::node) struct P2pConnManager { } impl P2pConnManager { - pub fn build( + pub fn build( transport: transport::Boxed<(PeerId, muxing::StreamMuxerBox)>, - config: &NodeBuilder, + config: &NodeBuilder, op_manager: Arc, event_listener: impl NetEventRegister + Clone, + private_key: Keypair, ) -> Result { // We set a global executor which is virtually the Tokio multi-threaded executor // to reuse it's thread pool and scheduler in order to drive futures. @@ -271,7 +274,7 @@ impl P2pConnManager { }; let behaviour = config_behaviour( - &config.local_key, + &private_key, &config.remote_nodes, &private_addr, op_manager.clone(), @@ -279,7 +282,7 @@ impl P2pConnManager { let mut swarm = Swarm::new( transport, behaviour, - PeerId::from(config.local_key.public()), + config.public_key.0, SwarmConfig::with_executor(global_executor) .with_idle_connection_timeout(config::PEER_TIMEOUT), ); @@ -446,7 +449,7 @@ impl P2pConnManager { Ok(Left((msg, client_id))) => { let cb = self.bridge.clone(); match msg { - Message::Aborted(tx) => { + NetMessage::Aborted(tx) => { let tx_type = tx.transaction_type(); let res = handle_cancelled_op( tx, @@ -604,7 +607,7 @@ enum ConnMngrActions { /// Outbound message SendMessage { peer: PeerKey, - msg: Box, + msg: Box, }, /// Update self own public address, useful when communicating for first time UpdatePublicAddr(Multiaddr), @@ -618,9 +621,9 @@ enum ConnMngrActions { /// Manages network connections with different peers and event routing within the swarm. pub(in crate::node) struct FreenetBehaviour { // FIFO queue for outbound messages - outbound: VecDeque<(PeerId, Either)>, + outbound: VecDeque<(PeerId, Either)>, // FIFO queue for inbound messages - inbound: VecDeque>, + inbound: VecDeque>, routing_table: HashMap>, connected: HashMap, openning_connection: HashSet, @@ -630,7 +633,7 @@ pub(in crate::node) struct FreenetBehaviour { impl NetworkBehaviour for FreenetBehaviour { type ConnectionHandler = Handler; - type ToSwarm = Message; + type ToSwarm = NetMessage; fn handle_established_inbound_connection( &mut self, @@ -743,8 +746,8 @@ type UniqConnId = usize; #[derive(Debug)] pub(in crate::node) enum HandlerEvent { - Inbound(Either), - Outbound(Either), + Inbound(Either), + Outbound(Either), } /// Handles the connection with a given peer. @@ -752,7 +755,7 @@ pub(in crate::node) struct Handler { substreams: Vec, uniq_conn_id: UniqConnId, protocol_status: ProtocolStatus, - pending: Vec, + pending: Vec, op_manager: Arc, } @@ -769,7 +772,7 @@ enum SubstreamState { /// We haven't started opening the outgoing substream yet. /// Contains the initial request we want to send. OutPendingOpen { - msg: Box, + msg: Box, conn_id: UniqConnId, }, /// Waiting for the first message after requesting an outbound open connection. @@ -782,7 +785,7 @@ enum SubstreamState { PendingSend { conn_id: UniqConnId, substream: FreenetStream, - msg: Box>, + msg: Box>, }, /// Waiting to flush the substream so that the data arrives to the remote. PendingFlush { @@ -811,7 +814,7 @@ impl Handler { } #[inline] - fn send_to_free_substream(&mut self, msg: Message) -> Option { + fn send_to_free_substream(&mut self, msg: NetMessage) -> Option { let pos = self .substreams .iter() @@ -1180,12 +1183,12 @@ pub(crate) type FreenetStream = stream::AndThen< sink::With< stream::ErrInto>>>, ConnectionError>, io::Cursor>, - Message, + NetMessage, future::Ready>, ConnectionError>>, - fn(Message) -> future::Ready>, ConnectionError>>, + fn(NetMessage) -> future::Ready>, ConnectionError>>, >, - future::Ready>, - fn(BytesMut) -> future::Ready>, + future::Ready>, + fn(BytesMut) -> future::Ready>, >; impl InboundUpgrade for FreenetProtocol @@ -1231,12 +1234,12 @@ where } #[inline(always)] -fn encode_msg(msg: Message) -> Result, ConnectionError> { +fn encode_msg(msg: NetMessage) -> Result, ConnectionError> { bincode::serialize(&msg).map_err(|err| ConnectionError::Serialization(Some(err))) } #[inline(always)] -fn decode_msg(buf: BytesMut) -> Result { +fn decode_msg(buf: BytesMut) -> Result { let cursor = std::io::Cursor::new(buf); bincode::deserialize_from(cursor).map_err(|err| ConnectionError::Serialization(Some(err))) } @@ -1259,7 +1262,7 @@ pub(in crate::node) struct NetBehaviour { #[derive(Debug)] pub(in crate::node) enum NetEvent { - Freenet(Box), + Freenet(Box), Identify(Box), Ping(ping::Event), Autonat(autonat::Event), @@ -1283,8 +1286,8 @@ impl From for NetEvent { } } -impl From for NetEvent { - fn from(event: Message) -> NetEvent { +impl From for NetEvent { + fn from(event: NetMessage) -> NetEvent { Self::Freenet(Box::new(event)) } } diff --git a/crates/core/src/node/network_event_log.rs b/crates/core/src/node/network_event_log.rs index 56dd37502..1f7c0a552 100644 --- a/crates/core/src/node/network_event_log.rs +++ b/crates/core/src/node/network_event_log.rs @@ -18,7 +18,7 @@ use super::PeerKey; use crate::{ config::GlobalExecutor, contract::StoreResponse, - message::{Message, Transaction}, + message::{NetMessage, Transaction}, operations::{connect, get::GetMsg, put::PutMsg, subscribe::SubscribeMsg}, ring::{Location, PeerKeyLocation}, router::RouteEvent, @@ -130,11 +130,11 @@ impl<'a> NetEventLog<'a> { } pub fn from_outbound_msg( - msg: &'a Message, + msg: &'a NetMessage, op_storage: &'a OpManager, ) -> Either> { let kind = match msg { - Message::Connect(connect::ConnectMsg::Response { + NetMessage::Connect(connect::ConnectMsg::Response { msg: connect::ConnectResponse::AcceptedBy { peers, @@ -156,7 +156,7 @@ impl<'a> NetEventLog<'a> { EventKind::Ignored } } - Message::Connect(connect::ConnectMsg::Response { + NetMessage::Connect(connect::ConnectMsg::Response { msg: connect::ConnectResponse::Proxy { accepted_by, @@ -187,11 +187,11 @@ impl<'a> NetEventLog<'a> { } pub fn from_inbound_msg( - msg: &'a Message, + msg: &'a NetMessage, op_storage: &'a OpManager, ) -> Either> { let kind = match msg { - Message::Connect(connect::ConnectMsg::Response { + NetMessage::Connect(connect::ConnectMsg::Response { msg: connect::ConnectResponse::AcceptedBy { peers, @@ -230,7 +230,7 @@ impl<'a> NetEventLog<'a> { } return Either::Right(events); } - Message::Put(PutMsg::RequestPut { + NetMessage::Put(PutMsg::RequestPut { contract, target, .. }) => { let key = contract.key(); @@ -239,13 +239,13 @@ impl<'a> NetEventLog<'a> { key, }) } - Message::Put(PutMsg::SuccessfulUpdate { new_value, .. }) => { + NetMessage::Put(PutMsg::SuccessfulUpdate { new_value, .. }) => { EventKind::Put(PutEvent::PutSuccess { requester: op_storage.ring.peer_key, value: new_value.clone(), }) } - Message::Put(PutMsg::Broadcasting { + NetMessage::Put(PutMsg::Broadcasting { new_value, broadcast_to, key, @@ -255,7 +255,7 @@ impl<'a> NetEventLog<'a> { key: key.clone(), value: new_value.clone(), }), - Message::Put(PutMsg::BroadcastTo { + NetMessage::Put(PutMsg::BroadcastTo { sender, new_value, key, @@ -265,12 +265,12 @@ impl<'a> NetEventLog<'a> { key: key.clone(), value: new_value.clone(), }), - Message::Get(GetMsg::ReturnGet { + NetMessage::Get(GetMsg::ReturnGet { key, value: StoreResponse { state: Some(_), .. }, .. }) => EventKind::Get { key: key.clone() }, - Message::Subscribe(SubscribeMsg::ReturnSub { + NetMessage::Subscribe(SubscribeMsg::ReturnSub { subscribed: true, key, sender, @@ -1003,7 +1003,7 @@ pub(super) mod test { use parking_lot::Mutex; use super::*; - use crate::{node::tests::NodeLabel, ring::Distance}; + use crate::{node::testing_impl::NodeLabel, ring::Distance}; static LOG_ID: AtomicUsize = AtomicUsize::new(0); diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 47f24358c..8e5b6c0c7 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -9,7 +9,7 @@ use crate::{ config::GlobalExecutor, contract::{ContractError, ContractHandlerChannel, ContractHandlerEvent, SenderHalve}, dev_tool::ClientId, - message::{Message, Transaction, TransactionType}, + message::{NetMessage, Transaction, TransactionType}, operations::{ connect::ConnectOp, get::GetOp, put::PutOp, subscribe::SubscribeOp, update::UpdateOp, OpEnum, OpError, @@ -59,14 +59,14 @@ pub(crate) struct OpManager { } impl OpManager { - pub(super) fn new( + pub(super) fn new( notification_channel: EventLoopNotificationsSender, contract_handler: ContractHandlerChannel, - builder: &NodeBuilder, + builder: &NodeBuilder, gateways: &[PeerKeyLocation], event_register: ER, ) -> Result { - let ring = Ring::new::(builder, gateways, notification_channel.clone())?; + let ring = Ring::new::(builder, gateways, notification_channel.clone())?; let ops = Arc::new(Ops::default()); let (new_transactions, rx) = tokio::sync::mpsc::channel(100); @@ -97,7 +97,7 @@ impl OpManager { /// with other nodes, like intermediate states before returning. pub async fn notify_op_change( &self, - msg: Message, + msg: NetMessage, op: OpEnum, client_id: Option, ) -> Result<(), OpError> { @@ -208,7 +208,7 @@ impl OpManager { } /// Notify the operation manager that a transaction is being transacted over the network. - pub fn sending_transaction(&self, peer: &PeerKey, msg: &Message) { + pub fn sending_transaction(&self, peer: &PeerKey, msg: &NetMessage) { let transaction = msg.id(); if let Some(loc) = msg.requested_location() { self.ring diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index 4d071e374..2c21cd23c 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -17,7 +17,7 @@ use super::{ NetEventRegister, PeerKey, }; use crate::{ - client_events::combinator::ClientEventsCombinator, + client_events::{combinator::ClientEventsCombinator, BoxedClient}, config::{self, GlobalExecutor}, contract::{ self, ClientResponsesSender, ContractHandler, ExecutorToEventLoopChannel, @@ -76,7 +76,9 @@ impl NodeP2P { } pub(crate) async fn build( - builder: NodeBuilder, + builder: NodeBuilder, + private_key: Keypair, + clients: [BoxedClient; CLIENTS], event_register: ER, ch_builder: CH::Builder, ) -> Result @@ -84,7 +86,7 @@ impl NodeP2P { CH: ContractHandler + Send + 'static, ER: NetEventRegister + Clone, { - let peer_key = PeerKey::from(builder.local_key.public()); + let peer_key = builder.public_key; let gateways = builder.get_gateways()?; let (notification_channel, notification_tx) = EventLoopNotifications::channel(); @@ -104,8 +106,14 @@ impl NodeP2P { .map_err(|e| anyhow::anyhow!(e))?; let conn_manager = { - let transport = Self::config_transport(&builder.local_key)?; - P2pConnManager::build(transport, &builder, op_storage.clone(), event_register)? + let transport = Self::config_transport(&private_key)?; + P2pConnManager::build( + transport, + &builder, + op_storage.clone(), + event_register, + private_key, + )? }; let parent_span = tracing::Span::current(); @@ -113,7 +121,7 @@ impl NodeP2P { contract::contract_handling(contract_handler) .instrument(tracing::info_span!(parent: parent_span.clone(), "contract_handling")), ); - let clients = ClientEventsCombinator::new(builder.clients); + let clients = ClientEventsCombinator::new(clients); let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); GlobalExecutor::spawn( client_event_handling( @@ -169,7 +177,7 @@ mod test { client_events::test::MemoryEventsGen, config::GlobalExecutor, contract::MemoryContractHandler, - node::{network_event_log, tests::get_free_port, InitPeerNode}, + node::{network_event_log, testing_impl::get_free_port, InitPeerNode}, ring::Location, }; @@ -220,14 +228,16 @@ mod test { // Start up the initial node. GlobalExecutor::spawn(async move { let user_events = MemoryEventsGen::new(receiver1, PeerKey::from(peer1_id)); - let mut config = NodeBuilder::new([Box::new(user_events)]); + let mut config = NodeBuilder::new(); config .with_ip(Ipv4Addr::LOCALHOST) .with_port(peer1_port) - .with_key(peer1_key); + .with_key(peer1_key.public().into()); let mut peer1 = Box::new( NodeP2P::build::( config, + peer1_key, + [Box::new(user_events)], network_event_log::TestEventListener::new(), "ping-listener".into(), ) @@ -241,10 +251,14 @@ mod test { // Start up the dialing node let dialer = GlobalExecutor::spawn(async move { let user_events = MemoryEventsGen::new(receiver2, PeerKey::from(peer2_id)); - let mut config = NodeBuilder::new([Box::new(user_events)]); - config.add_gateway(peer1_config.clone()); + let mut config = NodeBuilder::new(); + config + .add_gateway(peer1_config.clone()) + .with_key(peer2_key.public().into()); let mut peer2 = NodeP2P::build::( config, + peer2_key, + [Box::new(user_events)], network_event_log::TestEventListener::new(), "ping-dialer".into(), ) diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/testing_impl.rs similarity index 81% rename from crates/core/src/node/tests.rs rename to crates/core/src/node/testing_impl.rs index fa5a9a2c4..a8c1fd08a 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/testing_impl.rs @@ -14,14 +14,19 @@ use tokio::sync::watch::{channel, Receiver, Sender}; use tracing::{info, Instrument}; #[cfg(feature = "trace-ot")] -use crate::node::network_event_log::{CombinedRegister, NetEventRegister}; +use crate::node::network_event_log::CombinedRegister; use crate::{ client_events::test::{MemoryEventsGen, RandomEventGenerator}, config::GlobalExecutor, - node::{network_event_log::TestEventListener, InitPeerNode, NodeBuilder, NodeInMemory}, + node::{network_event_log::TestEventListener, InitPeerNode, NetEventRegister, NodeBuilder}, ring::{Distance, Location, PeerKeyLocation}, }; +mod in_memory; +mod inter_process; + +use self::inter_process::SimPeer; + use super::PeerKey; pub fn get_free_port() -> Result { @@ -123,14 +128,14 @@ struct GatewayConfig { location: Location, } -pub struct EventChain<'a> { - network: &'a SimNetwork, +pub struct EventChain { + network: SimNetwork, total_events: usize, count: usize, rng: rand::rngs::SmallRng, } -impl Iterator for EventChain<'_> { +impl Iterator for EventChain { type Item = usize; fn next(&mut self) -> Option { @@ -150,6 +155,43 @@ impl Iterator for EventChain<'_> { } } +#[cfg(feature = "trace-ot")] +type DefaultRegistry = CombinedRegister<2>; + +#[cfg(not(feature = "trace-ot"))] +type DefaultRegistry = TestEventListener; + +pub(super) struct Builder { + pub(super) peer_key: PeerKey, + builder: NodeBuilder, + ch_builder: String, + add_noise: bool, + event_register: ER, + contracts: Vec<(ContractContainer, WrappedState)>, + contract_subscribers: HashMap>, +} + +impl Builder { + /// Buils an in-memory node. Does nothing upon construction, + pub fn build( + builder: NodeBuilder, + event_register: ER, + ch_builder: String, + add_noise: bool, + ) -> Builder { + let peer_key = builder.public_key; + Builder { + peer_key, + builder, + ch_builder, + add_noise, + event_register, + contracts: Vec::new(), + contract_subscribers: HashMap::new(), + } + } +} + /// A simulated in-memory network topology. pub struct SimNetwork { name: String, @@ -159,9 +201,9 @@ pub struct SimNetwork { user_ev_controller: Sender<(EventId, PeerKey)>, receiver_ch: Receiver<(EventId, PeerKey)>, number_of_gateways: usize, - gateways: Vec<(NodeInMemory, GatewayConfig)>, + gateways: Vec<(Builder, GatewayConfig)>, number_of_nodes: usize, - nodes: Vec<(NodeInMemory, NodeLabel)>, + nodes: Vec<(Builder, NodeLabel)>, ring_max_htl: usize, rnd_if_htl_above: usize, max_connections: usize, @@ -222,7 +264,9 @@ impl SimNetwork { net.build_nodes(nodes).await; net } +} +impl SimNetwork { pub fn with_start_backoff(&mut self, value: Duration) { self.init_backoff = value; } @@ -248,11 +292,11 @@ impl SimNetwork { let port = get_free_port().unwrap(); let location = Location::random(); - let mut config = NodeBuilder::new([]); + let mut config = NodeBuilder::new(); config .with_ip(Ipv6Addr::LOCALHOST) .with_port(port) - .with_key(pair) + .with_key(pair.public().into()) .with_location(location) .max_hops_to_live(self.ring_max_htl) .max_number_of_connections(self.max_connections) @@ -286,17 +330,26 @@ impl SimNetwork { .listening_port(*port), ); } - let peer = PeerKey::from(this_node.local_key.public()); - let parent_span = tracing::info_span!("in_mem_gateway", %peer); - let gateway = NodeInMemory::build( + let event_listener = { + #[cfg(feature = "trace-ot")] + { + use super::network_event_log::OTEventRegister; + CombinedRegister::new([ + self.event_listener.trait_clone(), + Box::new(OTEventRegister::new()), + ]) + } + #[cfg(not(feature = "trace-ot"))] + { + self.event_listener.clone() + } + }; + let gateway = Builder::build( this_node, - self.event_listener.clone(), + event_listener, format!("{}-{label}", self.name, label = this_config.label), self.add_noise, - ) - .instrument(parent_span) - .await - .unwrap(); + ); self.gateways.push((gateway, this_config)); } } @@ -315,7 +368,7 @@ impl SimNetwork { let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); - let mut config = NodeBuilder::new([]); + let mut config = NodeBuilder::new(); for GatewayConfig { port, id, location, .. } in &gateways @@ -330,12 +383,11 @@ impl SimNetwork { .max_hops_to_live(self.ring_max_htl) .rnd_if_htl_above(self.rnd_if_htl_above) .max_number_of_connections(self.max_connections) - .with_key(pair); + .with_key(pair.public().into()); let peer = PeerKey::from(id); self.event_listener.add_node(label.clone(), peer); - let parent_span = tracing::info_span!("in_mem_node", %peer); let event_listener = { #[cfg(feature = "trace-ot")] { @@ -350,15 +402,12 @@ impl SimNetwork { self.event_listener.clone() } }; - let node = NodeInMemory::build( + let node = Builder::build( config, event_listener, format!("{}-{label}", self.name), self.add_noise, - ) - .instrument(parent_span) - .await - .unwrap(); + ); self.nodes.push((node, label)); } } @@ -374,38 +423,31 @@ impl SimNetwork { mut specs: HashMap, ) { let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); - for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { + for (mut node, label) in gw.chain(self.nodes.drain(..)).collect::>() { + tracing::debug!(peer = %label, "initializing"); let node_spec = specs.remove(&label); - self.initialize_peer(node, label, node_spec); + let mut user_events = MemoryEventsGen::new(self.receiver_ch.clone(), node.peer_key); + if let Some(specs) = node_spec.clone() { + user_events.generate_events(specs.events_to_generate); + } + let span = if label.is_gateway() { + tracing::info_span!("in_mem_gateway", %node.peer_key) + } else { + tracing::info_span!("in_mem_node", %node.peer_key) + }; + if let Some(specs) = node_spec { + node.append_contracts(specs.owned_contracts, specs.contract_subscribers); + } + self.labels.push((label, node.peer_key)); + + let node_task = async move { node.run_node(user_events).await }; + GlobalExecutor::spawn(node_task.instrument(span)); + tokio::time::sleep(self.init_backoff).await; } self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); } - #[cfg(test)] - fn initialize_peer( - &mut self, - mut peer: NodeInMemory, - label: NodeLabel, - node_specs: Option, - ) { - let mut user_events = MemoryEventsGen::new(self.receiver_ch.clone(), peer.peer_key); - if let Some(specs) = node_specs.clone() { - user_events.generate_events(specs.events_to_generate); - } - tracing::debug!(peer = %label, "initializing"); - self.labels.push((label, peer.peer_key)); - let node_task = async move { - if let Some(specs) = node_specs { - peer.append_contracts(specs.owned_contracts, specs.contract_subscribers) - .await - .map_err(|_| anyhow::anyhow!("failed inserting test owned contracts"))?; - } - peer.run_node(user_events).await - }; - GlobalExecutor::spawn(node_task); - } - pub async fn start_with_rand_gen( &mut self, seed: u64, @@ -419,34 +461,38 @@ impl SimNetwork { let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); let mut peers = vec![]; for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { - let handle = self.initialize_peer_with_rnd_gen::( - node, - label, - (total_peer_num, max_contract_num, seed, iterations), - ); + tracing::debug!(peer = %label, "initializing"); + let mut user_events = + MemoryEventsGen::::new_with_seed(self.receiver_ch.clone(), node.peer_key, seed); + user_events.rng_params(label.number(), total_peer_num, max_contract_num, iterations); + let span = if label.is_gateway() { + tracing::info_span!("in_mem_gateway", %node.peer_key) + } else { + tracing::info_span!("in_mem_node", %node.peer_key) + }; + self.labels.push((label, node.peer_key)); + + let node_task = async move { node.run_node(user_events).await }; + let handle = GlobalExecutor::spawn(node_task.instrument(span)); peers.push(handle); + tokio::time::sleep(self.init_backoff).await; } self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); peers } - fn initialize_peer_with_rnd_gen( - &mut self, - mut peer: NodeInMemory, - label: NodeLabel, - (total_peer_num, max_contract_num, seed, iterations): (usize, usize, u64, usize), - ) -> tokio::task::JoinHandle> - where - R: RandomEventGenerator + Send + 'static, - { - tracing::debug!(peer = %label, "initializing"); - let mut user_events = - MemoryEventsGen::::new_with_seed(self.receiver_ch.clone(), peer.peer_key, seed); - user_events.rng_params(label.number(), total_peer_num, max_contract_num, iterations); - self.labels.push((label, peer.peer_key)); - let node_task = async move { peer.run_node(user_events).await }; - GlobalExecutor::spawn(node_task) + /// Builds peer nodes and returns the controller to trigger events. + pub fn build_peers(&mut self) -> Vec<(NodeLabel, SimPeer)> { + let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); + let mut peers = vec![]; + for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { + self.labels.push((label.clone(), node.peer_key)); + peers.push((label, SimPeer(node))); + } + self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); + peers.sort_by(|(a, _), (b, _)| a.cmp(b)); + peers } pub fn get_locations_by_node(&self) -> HashMap { @@ -454,10 +500,22 @@ impl SimNetwork { // Get node and gateways location by label for (node, label) in &self.nodes { - locations_by_node.insert(label.clone(), node.op_storage.ring.own_location()); + locations_by_node.insert( + label.clone(), + PeerKeyLocation { + peer: node.peer_key, + location: None, + }, + ); } for (node, config) in &self.gateways { - locations_by_node.insert(config.label.clone(), node.op_storage.ring.own_location()); + locations_by_node.insert( + config.label.clone(), + PeerKeyLocation { + peer: node.peer_key, + location: config.location.into(), + }, + ); } locations_by_node } @@ -564,9 +622,19 @@ impl SimNetwork { Ok(()) } - /// Start an event chain for this simulation. - pub fn event_chain(&self, total_events: usize) -> EventChain<'_> { + /// Start an event chain for this simulation. Allows passing a different controller for the peers. + /// + /// If done make sure you set the proper receiving side for the controller. For example in the + /// nodes built through the [`build_peers`](`Self::build_peers`) method. + pub fn event_chain( + mut self, + total_events: usize, + controller: Option>, + ) -> EventChain { const SEED: u64 = 0xdeadbeef; + if let Some(controller) = controller { + self.user_ev_controller = controller; + } EventChain { network: self, total_events, diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/testing_impl/in_memory.rs similarity index 85% rename from crates/core/src/node/in_memory_impl.rs rename to crates/core/src/node/testing_impl/in_memory.rs index 6941fab0a..e1a925ac7 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/testing_impl/in_memory.rs @@ -1,10 +1,10 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use either::Either; use freenet_stdlib::prelude::*; use tracing::Instrument; -use super::{ +use crate::node::{ client_event_handling, handle_cancelled_op, join_ring_request, network_bridge::{in_memory::MemoryConnManager, EventLoopNotifications}, network_event_log::NetEventLog, @@ -18,37 +18,21 @@ use crate::{ self, executor_channel, ClientResponsesSender, ContractHandler, ExecutorToEventLoopChannel, MemoryContractHandler, NetworkEventListenerHalve, }, - message::{Message, NodeEvent, TransactionType}, - node::NodeBuilder, + message::{NetMessage, NodeEvent, TransactionType}, operations::OpError, ring::PeerKeyLocation, util::IterExt, }; -pub(super) struct NodeInMemory { - pub peer_key: PeerKey, - pub op_storage: Arc, - gateways: Vec, - notification_channel: EventLoopNotifications, - conn_manager: MemoryConnManager, - event_register: Box, - is_gateway: bool, - _executor_listener: ExecutorToEventLoopChannel, - /// Span to use for this node for tracing purposes - pub parent_span: tracing::Span, -} +use super::Builder; -impl NodeInMemory { - /// Buils an in-memory node. Does nothing upon construction, - pub async fn build( - builder: NodeBuilder<0>, - event_register: ER, - ch_builder: String, - add_noise: bool, - ) -> Result { - let peer_key = PeerKey::from(builder.local_key.public()); - let gateways = builder.get_gateways()?; - let is_gateway = builder.local_ip.zip(builder.local_port).is_some(); +impl Builder { + pub async fn run_node(self, user_events: UsrEv) -> Result<(), anyhow::Error> + where + UsrEv: ClientEventsProxy + Send + 'static, + { + let gateways = self.builder.get_gateways()?; + let is_gateway = self.builder.local_ip.zip(self.builder.local_port).is_some(); let (notification_channel, notification_tx) = EventLoopNotifications::channel(); let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); @@ -56,22 +40,21 @@ impl NodeInMemory { let op_storage = Arc::new(OpManager::new( notification_tx, ops_ch_channel, - &builder, + &self.builder, &gateways, - event_register.clone(), + self.event_register.clone(), )?); let (_executor_listener, executor_sender) = executor_channel(op_storage.clone()); let contract_handler = - MemoryContractHandler::build(ch_channel, executor_sender, ch_builder) + MemoryContractHandler::build(ch_channel, executor_sender, self.ch_builder) .await .map_err(|e| anyhow::anyhow!(e))?; - let event_register = Box::new(event_register); let conn_manager = MemoryConnManager::new( - peer_key, - event_register.trait_clone(), + self.peer_key, + self.event_register.trait_clone(), op_storage.clone(), - add_noise, + self.add_noise, ); let parent_span = tracing::Span::current(); @@ -79,62 +62,53 @@ impl NodeInMemory { contract::contract_handling(contract_handler) .instrument(tracing::info_span!(parent: parent_span.clone(), "contract_handling")), ); - - Ok(NodeInMemory { - peer_key, - conn_manager, + let mut running_node = InMemoryNode { + peer_key: self.peer_key, op_storage, gateways, notification_channel, - event_register, + conn_manager, + event_register: self.event_register.trait_clone(), is_gateway, _executor_listener, parent_span, - }) - } - - pub async fn run_node(&mut self, user_events: UsrEv) -> Result<(), anyhow::Error> - where - UsrEv: ClientEventsProxy + Send + 'static, - { - if !self.is_gateway { - if let Some(gateway) = self.gateways.iter().shuffle().take(1).next() { - join_ring_request( - None, - self.peer_key, - gateway, - &self.op_storage, - &mut self.conn_manager, - ) - .await?; - } else { - anyhow::bail!("requires at least one gateway"); - } - } - let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); - let parent_span = self.parent_span.clone(); - let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); - GlobalExecutor::spawn( - client_event_handling( - self.op_storage.clone(), - user_events, - client_responses, - node_controller_tx, - ) - .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), - ); - let parent_span: tracing::Span = self.parent_span.clone(); - self.run_event_listener(cli_response_sender, node_controller_rx) - .instrument(parent_span) - .await + }; + running_node + .append_contracts(self.contracts, self.contract_subscribers) + .await?; + running_node.run_node(user_events).await } #[cfg(test)] - pub async fn append_contracts<'a>( - &self, + pub fn append_contracts( + &mut self, contracts: Vec<(ContractContainer, WrappedState)>, contract_subscribers: std::collections::HashMap>, - ) -> Result<(), crate::contract::ContractError> { + ) { + self.contracts.extend(contracts); + self.contract_subscribers.extend(contract_subscribers); + } +} + +struct InMemoryNode { + peer_key: PeerKey, + op_storage: Arc, + gateways: Vec, + notification_channel: EventLoopNotifications, + conn_manager: MemoryConnManager, + event_register: Box, + is_gateway: bool, + _executor_listener: ExecutorToEventLoopChannel, + /// Span to use for this node for tracing purposes + parent_span: tracing::Span, +} + +impl InMemoryNode { + async fn append_contracts( + &mut self, + contracts: Vec<(ContractContainer, WrappedState)>, + contract_subscribers: HashMap>, + ) -> Result<(), anyhow::Error> { use crate::contract::ContractHandlerEvent; for (contract, state) in contracts { let key: ContractKey = contract.key(); @@ -177,6 +151,42 @@ impl NodeInMemory { Ok(()) } + pub async fn run_node(mut self, user_events: UsrEv) -> Result<(), anyhow::Error> + where + UsrEv: ClientEventsProxy + Send + 'static, + { + if !self.is_gateway { + if let Some(gateway) = self.gateways.iter().shuffle().take(1).next() { + join_ring_request( + None, + self.peer_key, + gateway, + &self.op_storage, + &mut self.conn_manager, + ) + .await?; + } else { + anyhow::bail!("requires at least one gateway"); + } + } + let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); + let parent_span = self.parent_span.clone(); + let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); + GlobalExecutor::spawn( + client_event_handling( + self.op_storage.clone(), + user_events, + client_responses, + node_controller_tx, + ) + .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), + ); + let parent_span: tracing::Span = self.parent_span.clone(); + self.run_event_listener(cli_response_sender, node_controller_rx) + .instrument(parent_span) + .await + } + /// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. async fn run_event_listener( &mut self, @@ -202,7 +212,7 @@ impl NodeInMemory { } }; - if let Ok(Either::Left(Message::Aborted(tx))) = msg { + if let Ok(Either::Left(NetMessage::Aborted(tx))) = msg { let tx_type = tx.transaction_type(); let res = handle_cancelled_op( tx, @@ -259,7 +269,7 @@ impl NodeInMemory { } }, Err(err) => { - super::report_result( + super::super::report_result( None, Err(err.into()), &self.op_storage, diff --git a/crates/core/src/node/testing_impl/inter_process.rs b/crates/core/src/node/testing_impl/inter_process.rs new file mode 100644 index 000000000..cf4750570 --- /dev/null +++ b/crates/core/src/node/testing_impl/inter_process.rs @@ -0,0 +1,32 @@ +use std::sync::Arc; + +use crate::{ + contract::{ExecutorToEventLoopChannel, NetworkEventListenerHalve}, + dev_tool::PeerKey, + node::{ + network_bridge::{inter_process::InterProcessConnManager, EventLoopNotifications}, + NetEventRegister, OpManager, + }, + ring::PeerKeyLocation, +}; + +use super::{Builder, DefaultRegistry}; + +pub struct SimPeer(pub(super) Builder); + +impl SimPeer { + pub fn peer_key(&self) -> PeerKey { + self.0.peer_key + } +} + +struct InterProcessNode { + peer_key: PeerKey, + op_storage: Arc, + gateways: Vec, + notification_channel: EventLoopNotifications, + conn_manager: InterProcessConnManager, + event_register: Box, + is_gateway: bool, + _executor_listener: ExecutorToEventLoopChannel, +} diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index 44a0a5aeb..b363538c2 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -8,7 +8,7 @@ use tokio::sync::mpsc::error::SendError; use crate::{ client_events::{ClientId, HostResult}, contract::ContractError, - message::{InnerMessage, Message, Transaction, TransactionType}, + message::{InnerMessage, NetMessage, Transaction, TransactionType}, node::{ConnectionError, NetworkBridge, OpManager, OpNotAvailable, PeerKey}, ring::{Location, PeerKeyLocation, RingError}, DynError, @@ -22,7 +22,7 @@ pub(crate) mod update; pub(crate) struct OperationResult { /// Inhabited if there is a message to return to the other peer. - pub return_msg: Option, + pub return_msg: Option, /// None if the operation has been completed. pub state: Option, } @@ -63,7 +63,6 @@ async fn handle_op_result( where CB: NetworkBridge, { - // FIXME: register changes in the future op commit log match result { Err(OpError::StatePushed) => { // do nothing and continue, the operation will just continue later on @@ -72,7 +71,7 @@ where Err(err) => { if let Some(sender) = sender { network_bridge - .send(&sender, Message::Aborted(tx_id)) + .send(&sender, NetMessage::Aborted(tx_id)) .await?; } return Err(err); diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index a3f42de2b..d63e54a73 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -7,7 +7,7 @@ use std::{collections::HashSet, time::Duration}; use super::{OpError, OpInitialization, OpOutcome, Operation, OperationResult}; use crate::{ client_events::ClientId, - message::{InnerMessage, Message, Transaction}, + message::{InnerMessage, NetMessage, Transaction}, node::{ConnectionError, NetworkBridge, OpManager, PeerKey}, operations::OpEnum, ring::{Location, PeerKeyLocation, Ring}, @@ -460,7 +460,7 @@ impl Operation for ConnectOp { }; op_storage .notify_op_change( - Message::Aborted(*id), + NetMessage::Aborted(*id), OpEnum::Connect(op.into()), None, ) @@ -682,7 +682,7 @@ fn build_op_result( backoff, }); Ok(OperationResult { - return_msg: msg.map(Message::from), + return_msg: msg.map(NetMessage::from), state: output_op.map(|op| OpEnum::Connect(Box::new(op))), }) } @@ -858,7 +858,7 @@ where conn_bridge.add_connection(gateway.peer).await?; let assigned_location = op_storage.ring.own_location().location; - let join_req = Message::from(messages::ConnectMsg::Request { + let join_req = NetMessage::from(messages::ConnectMsg::Request { id: tx, msg: messages::ConnectRequest::StartReq { target: gateway, @@ -931,16 +931,13 @@ where "Selecting close peer to forward request", ); // FIXME: target the `desired_location` - ring.routing( - joiner.location.unwrap(), - Some(&req_peer.peer), - skip_list.as_slice(), - ) - .and_then(|pkl| (pkl.peer != joiner.peer).then_some(pkl)) + let desired_location = joiner.location.unwrap(); + ring.routing(desired_location, Some(&req_peer.peer), skip_list.as_slice()) + .and_then(|pkl| (pkl.peer != joiner.peer).then_some(pkl)) }; if let Some(forward_to) = forward_to { - let forwarded = Message::from(ConnectMsg::Request { + let forwarded = NetMessage::from(ConnectMsg::Request { id, msg: ConnectRequest::Proxy { joiner, @@ -1142,7 +1139,7 @@ mod messages { mod test { use std::time::Duration; - use crate::node::tests::SimNetwork; + use crate::node::testing_impl::SimNetwork; /// Given a network of one node and one gateway test that both are connected. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 0d0f4ad43..173c587a9 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -8,7 +8,7 @@ use futures::FutureExt; use crate::{ client_events::ClientId, contract::{ContractError, ContractHandlerEvent, StoreResponse}, - message::{InnerMessage, Message, Transaction}, + message::{InnerMessage, NetMessage, Transaction}, node::{NetworkBridge, OpManager, PeerKey}, operations::{OpInitialization, Operation}, ring::{Location, PeerKeyLocation, RingError}, @@ -506,7 +506,7 @@ impl Operation for GetOp { op_storage .notify_op_change( - Message::from(GetMsg::ReturnGet { + NetMessage::from(GetMsg::ReturnGet { id, key, value: StoreResponse { @@ -609,7 +609,7 @@ fn build_op_result( stats, }); Ok(OperationResult { - return_msg: msg.map(Message::from), + return_msg: msg.map(NetMessage::from), state: output_op.map(OpEnum::Get), }) } @@ -617,7 +617,7 @@ fn build_op_result( async fn continue_seeking( conn_manager: &mut NB, new_target: &PeerKeyLocation, - retry_msg: Message, + retry_msg: NetMessage, ) -> Result<(), OpError> { tracing::debug!( tx = %retry_msg.id(), @@ -759,7 +759,7 @@ pub(crate) async fn request_get( }; op_storage - .notify_op_change(Message::from(msg), OpEnum::Get(op), client_id) + .notify_op_change(NetMessage::from(msg), OpEnum::Get(op), client_id) .await?; } _ => return Err(OpError::invalid_transition(get_op.id)), @@ -859,7 +859,7 @@ mod test { use std::{collections::HashMap, time::Duration}; use super::*; - use crate::node::tests::{NodeSpecification, SimNetwork}; + use crate::node::testing_impl::{NodeSpecification, SimNetwork}; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_get_op_between_nodes() -> Result<(), anyhow::Error> { @@ -876,7 +876,7 @@ mod test { fetch_contract: true, } .into(); - let node_0 = NodeSpecification { + let node_1 = NodeSpecification { owned_contracts: vec![], events_to_generate: HashMap::from_iter([(1, get_event)]), contract_subscribers: HashMap::new(), @@ -891,7 +891,7 @@ mod test { contract_subscribers: HashMap::new(), }; - let get_specs = HashMap::from_iter([("node-0".into(), node_0), ("gateway-0".into(), gw_0)]); + let get_specs = HashMap::from_iter([("node-1".into(), node_1), ("gateway-0".into(), gw_0)]); // establish network let mut sim_nw = SimNetwork::new( @@ -909,9 +909,9 @@ mod test { // trigger get @ node-0, which does not own the contract sim_nw - .trigger_event("node-0", 1, Some(Duration::from_secs(1))) + .trigger_event("node-1", 1, Some(Duration::from_secs(1))) .await?; - assert!(sim_nw.has_got_contract("node-0", &key)); + assert!(sim_nw.has_got_contract("node-1", &key)); Ok(()) } @@ -970,13 +970,13 @@ mod test { } .into(); - let node_0 = NodeSpecification { + let node_1 = NodeSpecification { owned_contracts: vec![], events_to_generate: HashMap::from_iter([(1, get_event)]), contract_subscribers: HashMap::new(), }; - let node_1 = NodeSpecification { + let node_2 = NodeSpecification { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, @@ -992,8 +992,8 @@ mod test { }; let get_specs = HashMap::from_iter([ - ("node-0".into(), node_0), ("node-1".into(), node_1), + ("node-2".into(), node_2), ("gateway-0".into(), gw_0), ]); @@ -1011,9 +1011,9 @@ mod test { sim_nw.start_with_spec(get_specs).await; sim_nw.check_connectivity(Duration::from_secs(3))?; sim_nw - .trigger_event("node-0", 1, Some(Duration::from_secs(1))) + .trigger_event("node-1", 1, Some(Duration::from_secs(1))) .await?; - assert!(sim_nw.has_got_contract("node-0", &key)); + assert!(sim_nw.has_got_contract("node-1", &key)); Ok(()) } } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 5a6008201..1f83c25c0 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -15,7 +15,7 @@ use super::{OpEnum, OpError, OpInitialization, OpOutcome, Operation, OperationRe use crate::{ client_events::ClientId, contract::ContractHandlerEvent, - message::{InnerMessage, Message, Transaction}, + message::{InnerMessage, NetMessage, Transaction}, node::{NetworkBridge, OpManager, PeerKey}, ring::{Location, PeerKeyLocation, RingError}, }; @@ -524,7 +524,7 @@ fn build_op_result( ) -> Result { let output_op = Some(PutOp { id, state, stats }); Ok(OperationResult { - return_msg: msg.map(Message::from), + return_msg: msg.map(NetMessage::from), state: output_op.map(OpEnum::Put), }) } @@ -592,7 +592,7 @@ async fn try_to_broadcast( }; op_storage .notify_op_change( - Message::from(return_msg.unwrap()), + NetMessage::from(return_msg.unwrap()), OpEnum::Put(op), client_id, ) @@ -710,7 +710,7 @@ pub(crate) async fn request_put( }; op_storage - .notify_op_change(Message::from(msg), OpEnum::Put(op), client_id) + .notify_op_change(NetMessage::from(msg), OpEnum::Put(op), client_id) .await?; } _ => return Err(OpError::invalid_transition(put_op.id)), @@ -940,7 +940,7 @@ mod test { use freenet_stdlib::client_api::ContractRequest; use freenet_stdlib::prelude::*; - use crate::node::tests::{NodeSpecification, SimNetwork}; + use crate::node::testing_impl::{NodeSpecification, SimNetwork}; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_put_op_between_nodes() -> Result<(), anyhow::Error> { @@ -965,11 +965,11 @@ mod test { ) .await; let mut locations = sim_nw.get_locations_by_node(); - let node0_loc = locations.remove(&"node-0".into()).unwrap(); - let node1_loc = locations.remove(&"node-1".into()).unwrap(); + let node0_loc = locations.remove(&"node-1".into()).unwrap(); + let node1_loc = locations.remove(&"node-2".into()).unwrap(); // both own the contract, and one triggers an update - let node_0 = NodeSpecification { + let node_1 = NodeSpecification { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val.clone(), @@ -978,7 +978,7 @@ mod test { contract_subscribers: HashMap::from_iter([(key.clone(), vec![node1_loc])]), }; - let node_1 = NodeSpecification { + let node_2 = NodeSpecification { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val.clone(), @@ -1005,8 +1005,8 @@ mod test { // establish network let put_specs = HashMap::from_iter([ - ("node-0".into(), node_0), ("node-1".into(), node_1), + ("node-2".into(), node_2), ("gateway-0".into(), gw_0), ]); diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 0772f4fed..dfb08dfb3 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -9,7 +9,7 @@ use super::{OpEnum, OpError, OpInitialization, OpOutcome, Operation, OperationRe use crate::{ client_events::ClientId, contract::ContractError, - message::{InnerMessage, Message, Transaction}, + message::{InnerMessage, NetMessage, Transaction}, node::{NetworkBridge, OpManager, PeerKey}, ring::{Location, PeerKeyLocation, RingError}, }; @@ -135,7 +135,7 @@ impl Operation for SubscribeOp { let sender = op_storage.ring.own_location(); let return_err = || -> OperationResult { OperationResult { - return_msg: Some(Message::from(SubscribeMsg::ReturnSub { + return_msg: Some(NetMessage::from(SubscribeMsg::ReturnSub { key: key.clone(), id: *id, subscribed: false, @@ -307,7 +307,7 @@ fn build_op_result( state: Some(state), }); Ok(OperationResult { - return_msg: msg.map(Message::from), + return_msg: msg.map(NetMessage::from), state: output_op.map(OpEnum::Subscribe), }) } @@ -373,7 +373,7 @@ pub(crate) async fn request_subscribe( state: new_state, }; op_storage - .notify_op_change(Message::from(msg), OpEnum::Subscribe(op), client_id) + .notify_op_change(NetMessage::from(msg), OpEnum::Subscribe(op), client_id) .await?; } _ => return Err(OpError::invalid_transition(sub_op.id)), @@ -478,7 +478,7 @@ mod test { use freenet_stdlib::client_api::ContractRequest; use super::*; - use crate::node::tests::{NodeSpecification, SimNetwork}; + use crate::node::testing_impl::{NodeSpecification, SimNetwork}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn successful_subscribe_op_between_nodes() -> Result<(), anyhow::Error> { @@ -511,8 +511,8 @@ mod test { }; let subscribe_specs = HashMap::from_iter([ - ("node-0".into(), first_node), - ("node-1".into(), second_node), + ("node-1".into(), first_node), + ("node-2".into(), second_node), ]); let mut sim_nw = SimNetwork::new( "successful_subscribe_op_between_nodes", @@ -527,11 +527,11 @@ mod test { sim_nw.start_with_spec(subscribe_specs).await; sim_nw.check_connectivity(Duration::from_secs(3))?; sim_nw - .trigger_event("node-1", 1, Some(Duration::from_secs(1))) + .trigger_event("node-2", 1, Some(Duration::from_secs(1))) .await?; - assert!(sim_nw.has_got_contract("node-1", &contract_key)); + assert!(sim_nw.has_got_contract("node-2", &contract_key)); tokio::time::sleep(Duration::from_secs(3)).await; - assert!(sim_nw.is_subscribed_to_contract("node-1", &contract_key)); + assert!(sim_nw.is_subscribed_to_contract("node-2", &contract_key)); Ok(()) } } diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index e17a056a1..e0ebce09d 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -85,7 +85,7 @@ mod messages { todo!() } - fn requested_location(&self) -> Option { + fn requested_location(&self) -> Option { todo!() } } diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index 85b3875fb..e42a69942 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -224,14 +224,14 @@ impl Ring { /// connection of a peer in the network). const MAX_HOPS_TO_LIVE: usize = 10; - pub fn new( - config: &NodeBuilder, + pub fn new( + config: &NodeBuilder, gateways: &[PeerKeyLocation], event_loop_notifier: EventLoopNotificationsSender, ) -> Result, anyhow::Error> { let (live_tx_tracker, missing_candidate_rx) = LiveTransactionTracker::new(); - let peer_key = PeerKey::from(config.local_key.public()); + let peer_key = config.public_key; // for location here consider -1 == None let own_location = AtomicU64::new(u64::from_le_bytes((-1f64).to_le_bytes())); @@ -306,13 +306,20 @@ impl Ring { Ok(ring) } - async fn refresh_router(router: Arc>) { + async fn refresh_router(router: Arc>) { let mut interval = tokio::time::interval(Duration::from_secs(60 * 5)); interval.tick().await; loop { interval.tick().await; - // fixme - let history = if std::any::type_name::() == std::any::type_name::() { + #[cfg(feature = "trace-ot")] + let should_route = std::any::type_name::() + == std::any::type_name::() + || std::any::type_name::() + == std::any::type_name::>(); + #[cfg(not(feature = "trace-ot"))] + let should_route = + std::any::type_name::() == std::any::type_name::(); + let history = if should_route { EventRegister::get_router_events(10_000) .await .map_err(|error| { diff --git a/crates/core/src/router.rs b/crates/core/src/router.rs index 7ea381042..cf495998e 100644 --- a/crates/core/src/router.rs +++ b/crates/core/src/router.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] // FIXME: remove this when this module is integrated with the rest of the codebase mod isotonic_estimator; mod util; diff --git a/crates/core/src/router/util.rs b/crates/core/src/router/util.rs index 7aeca20a6..11252fbca 100644 --- a/crates/core/src/router/util.rs +++ b/crates/core/src/router/util.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use serde::Serialize; #[derive(Debug, Clone, Copy, Serialize)] @@ -38,11 +36,3 @@ impl Default for Mean { pub(super) struct TransferSpeed { pub bytes_per_second: f64, } - -impl TransferSpeed { - pub fn new(bytes: usize, duration: Duration) -> Self { - TransferSpeed { - bytes_per_second: bytes as f64 / duration.as_secs_f64(), - } - } -} diff --git a/crates/core/src/runtime/store.rs b/crates/core/src/runtime/store.rs index a6fe62b15..09c372249 100644 --- a/crates/core/src/runtime/store.rs +++ b/crates/core/src/runtime/store.rs @@ -17,7 +17,7 @@ pub(super) struct SafeWriter { file: BufWriter, lock_file_path: PathBuf, compact: bool, - _marker: std::marker::PhantomData ()>, + _marker: std::marker::PhantomData S>, } impl SafeWriter { diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index 6db3a33a2..7b155dd26 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -1,4 +1,7 @@ +use std::time::Duration; + use anyhow::Error; +use freenet::dev_tool::SimNetwork; /// Testing framework for running Freenet network simulations. #[derive(clap::Parser, Clone)] @@ -46,6 +49,24 @@ pub struct TestConfig { pub command: TestMode, } +impl TestConfig { + fn get_connection_check_params(&self) -> (Duration, f64) { + let conns_per_gw = (self.nodes / self.gateways) as f64; + let conn_percent = (conns_per_gw / self.nodes as f64).min(0.99); + let connectivity_timeout = Duration::from_millis(self.wait_duration.unwrap_or_else(|| { + // expect a peer to take max 200ms to connect, this should happen in parallel + // but err on the side of safety + (conns_per_gw * 200.0).ceil() as u64 + })); + (connectivity_timeout, conn_percent) + } + + fn seed(&self) -> u64 { + use rand::RngCore; + self.seed.unwrap_or_else(|| rand::rngs::OsRng.next_u64()) + } +} + fn randomize_test_name() -> String { const ALPHABET: &str = "abcdefghijklmnopqrstuvwxyz"; use rand::seq::IteratorRandom; @@ -83,42 +104,66 @@ pub(crate) async fn test_framework(base_config: TestConfig) -> Result<(), Error> Ok(()) } +async fn config_sim_network(base_config: &TestConfig) -> Result { + if base_config.gateways == 0 { + anyhow::bail!("Gateways should be higher than 0"); + } + if base_config.nodes == 0 { + anyhow::bail!("Nodes should be higher than 0"); + } + let name = &base_config + .name + .as_ref() + .cloned() + .unwrap_or_else(randomize_test_name); + let sim = SimNetwork::new( + name, + base_config.gateways, + base_config.nodes, + base_config.ring_max_htl, + base_config.rnd_if_htl_above, + base_config.max_connections, + base_config.min_connections, + ) + .await; + Ok(sim) +} + +mod multiple_process { + use freenet::dev_tool::{MemoryEventsGen, PeerKey}; + + struct MultiProcessEventSender {} + + struct MultiProcessEventReceiver {} + + pub(super) async fn run(base_config: &super::TestConfig) -> Result<(), super::Error> { + let mut simulated_network = super::config_sim_network(base_config).await?; + let peers = simulated_network.build_peers(); + let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerKey::random())); + for (label, node) in peers { + let mut user_events = MemoryEventsGen::::new_with_seed( + receiver_ch.clone(), + node.peer_key(), + base_config.seed(), + ); + // user_events.rng_params(label.number(), total_peer_num, max_contract_num, iterations); + } + todo!() + } +} + mod single_process { use std::time::Duration; - use freenet::network_sim::SimNetwork; use futures::StreamExt; - use rand::RngCore; use tokio::signal; pub(super) async fn run(base_config: &super::TestConfig) -> Result<(), super::Error> { - if base_config.gateways == 0 { - anyhow::bail!("Gateways should be higher than 0"); - } - if base_config.nodes == 0 { - anyhow::bail!("Nodes should be higher than 0"); - } - let name = &base_config - .name - .as_ref() - .cloned() - .unwrap_or_else(super::randomize_test_name); - let mut simulated_network = SimNetwork::new( - name, - base_config.gateways, - base_config.nodes, - base_config.ring_max_htl, - base_config.rnd_if_htl_above, - base_config.max_connections, - base_config.min_connections, - ) - .await; + let mut simulated_network = super::config_sim_network(base_config).await?; let join_handles = simulated_network .start_with_rand_gen::( - base_config - .seed - .unwrap_or_else(|| rand::rngs::OsRng.next_u64()), + base_config.seed(), base_config .max_contract_number .unwrap_or(base_config.nodes * 10), @@ -128,17 +173,8 @@ mod single_process { let events = base_config.events; let next_event_wait_time = base_config.event_wait_time.map(Duration::from_millis); - let (connectivity_timeout, network_connection_percent) = { - let conns_per_gw = (base_config.nodes / base_config.gateways) as f64; - let conn_percent = (conns_per_gw / base_config.nodes as f64).min(0.99); - let connectivity_timeout = - Duration::from_millis(base_config.wait_duration.unwrap_or_else(|| { - // expect a peer to take max 200ms to connect, this should happen in parallel - // but err on the side of safety - (conns_per_gw * 200.0).ceil() as u64 - })); - (connectivity_timeout, conn_percent) - }; + let (connectivity_timeout, network_connection_percent) = + base_config.get_connection_check_params(); let events_generated = tokio::task::spawn_blocking(move || { println!( "Waiting for network to be sufficiently connected ({}ms timeout, {}%)", @@ -147,12 +183,12 @@ mod single_process { ); simulated_network .check_partial_connectivity(connectivity_timeout, network_connection_percent)?; - for _ in simulated_network.event_chain(events) { + for _ in simulated_network.event_chain(events, None) { if let Some(t) = next_event_wait_time { std::thread::sleep(t); } } - Ok::<_, super::Error>(simulated_network) + Ok::<_, super::Error>(()) }); let join_peer_tasks = async move { @@ -169,7 +205,6 @@ mod single_process { tokio::pin!(join_peer_tasks); tokio::pin!(ctrl_c); - let mut network_sim = None; loop { tokio::select! { _ = &mut ctrl_c /* SIGINT handling */ => { @@ -177,10 +212,9 @@ mod single_process { } res = &mut events_generated => { match res? { - Ok(return_nw) => { + Ok(()) => { println!("Test events generated successfully"); - network_sim = Some(return_nw); - *events_generated = tokio::task::spawn(futures::future::pending::>()); + *events_generated = tokio::task::spawn(futures::future::pending::>()); continue; } Err(err) => { @@ -203,8 +237,6 @@ mod single_process { } } - _ = network_sim; - Ok(()) } } diff --git a/crates/fdev/src/wasm_runtime/user_events.rs b/crates/fdev/src/wasm_runtime/user_events.rs index 267c5a741..35990b77d 100644 --- a/crates/fdev/src/wasm_runtime/user_events.rs +++ b/crates/fdev/src/wasm_runtime/user_events.rs @@ -309,14 +309,25 @@ impl ClientEventsProxy for StdInput { )); } Ok(Command::GetParams) => { - // FIXME: related to issue 272 - let _node = &*self.app_state.local_node.read().await; - // let p = node - // .contract_state - // .get_params(self.contract.key()) - // .await - // .unwrap(); - // self.app_state.printout_deser(&p); + let node = &*self.app_state.local_node.read().await; + let key = self.contract.key(); + let p = node + .state_store + .get_params(&key) + .await + .map_err(|e| { + ClientError::from(ErrorKind::Unhandled { + cause: format!("{e}"), + }) + })? + .ok_or_else(|| { + ClientError::from(ErrorKind::Unhandled { + cause: format!("missing contract parameters: {key}",), + }) + })?; + if let Err(e) = self.app_state.printout_deser(&p) { + tracing::error!("error printing params: {e}"); + } } Ok(cmd) => { return Ok(Either::Left( From a3eb65a886b2da94f903d19ba069d7e6789dfa29 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 17 Nov 2023 13:27:58 +0100 Subject: [PATCH 08/12] Share node runner for both test impls --- crates/core/src/bin/freenet.rs | 8 +- crates/core/src/client_events.rs | 10 +- crates/core/src/contract/executor.rs | 6 +- crates/core/src/contract/handler.rs | 4 +- crates/core/src/contract/storages/rocks_db.rs | 4 +- crates/core/src/lib.rs | 4 +- crates/core/src/message.rs | 6 +- crates/core/src/node.rs | 83 ++--- crates/core/src/node/network_bridge.rs | 8 +- .../core/src/node/network_bridge/in_memory.rs | 50 +-- .../src/node/network_bridge/inter_process.rs | 28 +- .../src/node/network_bridge/p2p_protoc.rs | 54 ++-- crates/core/src/node/network_event_log.rs | 46 ++- crates/core/src/node/op_state_manager.rs | 8 +- crates/core/src/node/p2p_impl.rs | 30 +- crates/core/src/node/testing_impl.rs | 286 ++++++++++++++++-- .../core/src/node/testing_impl/in_memory.rs | 228 ++------------ .../src/node/testing_impl/inter_process.rs | 79 +++-- crates/core/src/operations.rs | 6 +- crates/core/src/operations/connect.rs | 24 +- crates/core/src/operations/get.rs | 8 +- crates/core/src/operations/put.rs | 8 +- crates/core/src/operations/subscribe.rs | 10 +- crates/core/src/ring.rs | 56 ++-- crates/core/src/runtime/store.rs | 4 +- crates/core/src/util.rs | 10 +- crates/fdev/src/testing.rs | 4 +- 27 files changed, 599 insertions(+), 473 deletions(-) diff --git a/crates/core/src/bin/freenet.rs b/crates/core/src/bin/freenet.rs index da78c34eb..b74c27614 100644 --- a/crates/core/src/bin/freenet.rs +++ b/crates/core/src/bin/freenet.rs @@ -1,17 +1,17 @@ use clap::Parser; -use freenet::local_node::{Executor, NodeConfig, OperationMode}; +use freenet::local_node::{Executor, OperationMode, PeerCliConfig}; use std::net::SocketAddr; type DynError = Box; -async fn run(config: NodeConfig) -> Result<(), DynError> { +async fn run(config: PeerCliConfig) -> Result<(), DynError> { match config.mode { OperationMode::Local => run_local(config).await, OperationMode::Network => Err("network mode not yet enabled".into()), } } -async fn run_local(config: NodeConfig) -> Result<(), DynError> { +async fn run_local(config: PeerCliConfig) -> Result<(), DynError> { let port = config.port; let ip = config.address; freenet::config::Config::set_op_mode(OperationMode::Local); @@ -22,7 +22,7 @@ async fn run_local(config: NodeConfig) -> Result<(), DynError> { fn main() -> Result<(), DynError> { freenet::config::set_logger(); - let config = NodeConfig::parse(); + let config = PeerCliConfig::parse(); let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(4) .enable_all() diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index 8d87d5edd..92ec15d09 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -164,13 +164,13 @@ pub(crate) mod test { use rand::{seq::SliceRandom, SeedableRng}; use tokio::sync::watch::Receiver; - use crate::node::{testing_impl::EventId, PeerKey}; + use crate::node::{testing_impl::EventId, PeerId}; use super::*; pub struct MemoryEventsGen { - id: PeerKey, - signal: Receiver<(EventId, PeerKey)>, + id: PeerId, + signal: Receiver<(EventId, PeerId)>, events_to_gen: HashMap>, rng: Option, internal_state: Option, @@ -180,7 +180,7 @@ pub(crate) mod test { where R: RandomEventGenerator, { - pub fn new_with_seed(signal: Receiver<(EventId, PeerKey)>, id: PeerKey, seed: u64) -> Self { + pub fn new_with_seed(signal: Receiver<(EventId, PeerId)>, id: PeerId, seed: u64) -> Self { Self { signal, id, @@ -227,7 +227,7 @@ pub(crate) mod test { impl MemoryEventsGen { #[cfg(test)] - pub fn new(signal: Receiver<(EventId, PeerKey)>, id: PeerKey) -> Self { + pub fn new(signal: Receiver<(EventId, PeerId)>, id: PeerId) -> Self { Self { signal, id, diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index e0b0a7e71..d02734f81 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -31,7 +31,7 @@ use crate::runtime::{ }; use crate::{ client_events::{ClientId, HostResult}, - node::NodeConfig, + node::PeerCliConfig, operations::{self, Operation}, DynError, }; @@ -377,7 +377,7 @@ impl Executor { } async fn get_stores( - config: &NodeConfig, + config: &PeerCliConfig, ) -> Result< ( ContractStore, @@ -419,7 +419,7 @@ impl Executor { } impl Executor { - pub async fn from_config(config: NodeConfig) -> Result { + pub async fn from_config(config: PeerCliConfig) -> Result { let (contract_store, delegate_store, secret_store, state_store) = Self::get_stores(&config).await?; let rt = Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(); diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 45fd57119..3d237156b 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -18,7 +18,7 @@ use super::{ }; use crate::client_events::HostResult; use crate::message::Transaction; -use crate::{client_events::ClientId, node::NodeConfig, runtime::Runtime, DynError}; +use crate::{client_events::ClientId, node::PeerCliConfig, runtime::Runtime, DynError}; pub(crate) struct ClientResponses(UnboundedReceiver<(ClientId, HostResult)>); @@ -86,7 +86,7 @@ pub(crate) struct NetworkContractHandler { } impl ContractHandler for NetworkContractHandler { - type Builder = NodeConfig; + type Builder = PeerCliConfig; type ContractExecutor = Executor; fn build( diff --git a/crates/core/src/contract/storages/rocks_db.rs b/crates/core/src/contract/storages/rocks_db.rs index d3777f771..4192e73f1 100644 --- a/crates/core/src/contract/storages/rocks_db.rs +++ b/crates/core/src/contract/storages/rocks_db.rs @@ -23,8 +23,8 @@ impl RocksDb { } impl RocksDb { - const STATE_SUFFIX: &[u8] = "_key".as_bytes(); - const PARAMS_SUFFIX: &[u8] = "_params".as_bytes(); + const STATE_SUFFIX: &'static [u8] = "_key".as_bytes(); + const PARAMS_SUFFIX: &'static [u8] = "_params".as_bytes(); } #[async_trait::async_trait] diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index d305b300a..dea062df5 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -20,7 +20,7 @@ pub mod local_node { use super::*; pub use contract::Executor; pub use contract::OperationMode; - pub use node::NodeConfig; + pub use node::PeerCliConfig; } /// Exports for the dev tool. @@ -29,7 +29,7 @@ pub mod dev_tool { pub use crate::config::Config; pub use client_events::{test::MemoryEventsGen, ClientEventsProxy, ClientId, OpenRequest}; pub use contract::{storages::Storage, Executor, OperationMode}; - pub use node::{testing_impl::SimNetwork, InitPeerNode, NodeBuilder, NodeConfig, PeerKey}; + pub use node::{testing_impl::SimNetwork, InitPeerNode, NodeConfig, PeerCliConfig, PeerId}; pub use ring::Location; pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 7c790b6ad..f5a07dc4f 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use ulid::Ulid; use crate::{ - node::{ConnectionError, PeerKey}, + node::{ConnectionError, PeerId}, operations::{ connect::ConnectMsg, get::GetMsg, put::PutMsg, subscribe::SubscribeMsg, update::UpdateMsg, }, @@ -250,9 +250,9 @@ pub(crate) enum NodeEvent { /// Received a confirmation from a peer that a physical connection was established. ConfirmedInbound, /// Drop the given peer connection. - DropConnection(PeerKey), + DropConnection(PeerId), /// Accept the connections from the given peer. - AcceptConnection(PeerKey), + AcceptConnection(PeerId), /// Error while sending a message by the connection bridge from within the ops. #[serde(skip)] Error(ConnectionError), diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 8c5661486..c69ae424b 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -17,7 +17,7 @@ use std::{ use either::Either; use freenet_stdlib::client_api::{ClientRequest, ContractRequest, ErrorKind}; -use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; +use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId as Libp2pPeerId}; use serde::{Deserialize, Serialize}; use tracing::Instrument; @@ -55,7 +55,7 @@ mod p2p_impl; pub(crate) mod testing_impl; #[derive(clap::Parser, Clone, Debug)] -pub struct NodeConfig { +pub struct PeerCliConfig { /// Node operation mode. #[clap(value_enum, default_value_t=OperationMode::Local)] pub mode: OperationMode, @@ -91,9 +91,9 @@ impl Node { /// If both are provided but also additional peers are added via the [`Self::add_gateway()`] method, this node will /// be listening but also try to connect to an existing peer. #[derive(Serialize, Deserialize)] -pub struct NodeBuilder { - /// public key for the peer - pub(crate) public_key: PeerKey, +pub struct NodeConfig { + /// public identifier for the peer + pub(crate) peer_id: PeerId, // optional local info, in case this is an initial bootstrap node /// IP to bind to the listener pub(crate) local_ip: Option, @@ -114,11 +114,11 @@ pub struct NodeBuilder { pub(crate) min_number_conn: Option, } -impl NodeBuilder { - pub fn new() -> NodeBuilder { +impl NodeConfig { + pub fn new() -> NodeConfig { let local_key = Config::conf().local_peer_keypair.public().into(); - NodeBuilder { - public_key: local_key, + NodeConfig { + peer_id: local_key, remote_nodes: Vec::with_capacity(1), local_ip: None, local_port: None, @@ -164,8 +164,8 @@ impl NodeBuilder { /// Optional identity key of this node. /// If not provided it will be either obtained from the configuration or freshly generated. - pub fn with_key(&mut self, key: PeerKey) -> &mut Self { - self.public_key = key; + pub fn with_key(&mut self, key: PeerId) -> &mut Self { + self.peer_id = key; self } @@ -183,7 +183,7 @@ impl NodeBuilder { /// Builds a node using the default backend connection manager. pub async fn build( self, - config: NodeConfig, + config: PeerCliConfig, clients: [BoxedClient; CLIENTS], private_key: identity::Keypair, ) -> Result { @@ -215,7 +215,7 @@ impl NodeBuilder { /// Returns all specified gateways for this peer. Returns an error if the peer is not a gateway /// and no gateways are specified. fn get_gateways(&self) -> Result, anyhow::Error> { - let peer = self.public_key; + let peer = self.peer_id; let gateways: Vec<_> = self .remote_nodes .iter() @@ -241,7 +241,7 @@ impl NodeBuilder { } } -impl Default for NodeBuilder { +impl Default for NodeConfig { fn default() -> Self { Self::new() } @@ -251,15 +251,15 @@ impl Default for NodeBuilder { #[derive(Clone, Serialize, Deserialize)] pub struct InitPeerNode { addr: Option, - identifier: PeerKey, + identifier: PeerId, location: Location, } impl InitPeerNode { - pub fn new(identifier: PeerId, location: Location) -> Self { + pub fn new(identifier: Libp2pPeerId, location: Location) -> Self { Self { addr: None, - identifier: PeerKey(identifier), + identifier: PeerId(identifier), location, } } @@ -268,8 +268,8 @@ impl InitPeerNode { /// /// # Panic /// Will panic if is not a valid representation. - pub fn decode_peer_id>(mut bytes: T) -> PeerId { - PeerId::from_public_key( + pub fn decode_peer_id>(mut bytes: T) -> Libp2pPeerId { + Libp2pPeerId::from_public_key( &identity::Keypair::try_from( identity::ed25519::Keypair::try_from_bytes(bytes.as_mut()).unwrap(), ) @@ -302,7 +302,7 @@ impl InitPeerNode { async fn join_ring_request( backoff: Option, - peer_key: PeerKey, + peer_key: PeerId, gateway: &PeerKeyLocation, op_storage: &OpManager, conn_manager: &mut CM, @@ -708,7 +708,7 @@ async fn process_message( async fn handle_cancelled_op( tx: Transaction, - peer_key: PeerKey, + peer_key: PeerId, op_storage: &OpManager, conn_manager: &mut CM, ) -> Result<(), OpError> @@ -740,23 +740,24 @@ where } #[derive(PartialEq, Eq, Hash, Clone, Copy, PartialOrd, Ord)] -pub struct PeerKey(PeerId); +pub struct PeerId(Libp2pPeerId); #[cfg(test)] -impl<'a> arbitrary::Arbitrary<'a> for PeerKey { +impl<'a> arbitrary::Arbitrary<'a> for PeerId { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { let data: [u8; 32] = u.arbitrary()?; - let id = - PeerId::from_multihash(libp2p::multihash::Multihash::wrap(0, data.as_slice()).unwrap()) - .unwrap(); + let id = Libp2pPeerId::from_multihash( + libp2p::multihash::Multihash::wrap(0, data.as_slice()).unwrap(), + ) + .unwrap(); Ok(Self(id)) } } -impl PeerKey { +impl PeerId { pub fn random() -> Self { use libp2p::identity::Keypair; - PeerKey::from(Keypair::generate_ed25519().public()) + PeerId::from(Keypair::generate_ed25519().public()) } #[cfg(test)] @@ -765,37 +766,37 @@ impl PeerKey { } } -impl std::fmt::Debug for PeerKey { +impl std::fmt::Debug for PeerId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { ::fmt(self, f) } } -impl Display for PeerKey { +impl Display for PeerId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } } -impl From for PeerKey { +impl From for PeerId { fn from(val: identity::PublicKey) -> Self { - PeerKey(PeerId::from(val)) + PeerId(Libp2pPeerId::from(val)) } } -impl From for PeerKey { - fn from(val: PeerId) -> Self { - PeerKey(val) +impl From for PeerId { + fn from(val: Libp2pPeerId) -> Self { + PeerId(val) } } mod serialization { - use libp2p::PeerId; + use libp2p::PeerId as Libp2pPeerId; use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use super::PeerKey; + use super::PeerId; - impl Serialize for PeerKey { + impl Serialize for PeerId { fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -804,14 +805,14 @@ mod serialization { } } - impl<'de> Deserialize<'de> for PeerKey { + impl<'de> Deserialize<'de> for PeerId { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let bytes: Vec = Deserialize::deserialize(deserializer)?; - Ok(PeerKey( - PeerId::from_bytes(&bytes).expect("failed deserialization of PeerKey"), + Ok(PeerId( + Libp2pPeerId::from_bytes(&bytes).expect("failed deserialization of PeerKey"), )) } } diff --git a/crates/core/src/node/network_bridge.rs b/crates/core/src/node/network_bridge.rs index 7ea75b2ea..19506ce5d 100644 --- a/crates/core/src/node/network_bridge.rs +++ b/crates/core/src/node/network_bridge.rs @@ -7,7 +7,7 @@ use libp2p::swarm::StreamUpgradeError; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc::{self, Receiver, Sender}; -use super::PeerKey; +use super::PeerId; use crate::{ client_events::ClientId, message::{NetMessage, NodeEvent}, @@ -27,11 +27,11 @@ pub(crate) type ConnResult = std::result::Result; /// to other peers in the network with whom connection has been established. #[async_trait::async_trait] pub(crate) trait NetworkBridge: Send + Sync { - async fn add_connection(&mut self, peer: PeerKey) -> ConnResult<()>; + async fn add_connection(&mut self, peer: PeerId) -> ConnResult<()>; - async fn drop_connection(&mut self, peer: &PeerKey) -> ConnResult<()>; + async fn drop_connection(&mut self, peer: &PeerId) -> ConnResult<()>; - async fn send(&self, target: &PeerKey, msg: NetMessage) -> ConnResult<()>; + async fn send(&self, target: &PeerId, msg: NetMessage) -> ConnResult<()>; } #[derive(Debug, thiserror::Error, Serialize, Deserialize)] diff --git a/crates/core/src/node/network_bridge/in_memory.rs b/crates/core/src/node/network_bridge/in_memory.rs index ae3085403..0d1923d86 100644 --- a/crates/core/src/node/network_bridge/in_memory.rs +++ b/crates/core/src/node/network_bridge/in_memory.rs @@ -7,15 +7,18 @@ use std::{ }; use crossbeam::channel::{self, Receiver, Sender}; +use futures::{future::BoxFuture, FutureExt}; use once_cell::sync::OnceCell; use rand::{prelude::StdRng, seq::SliceRandom, Rng, SeedableRng}; use tokio::sync::Mutex; -use super::{ConnectionError, NetworkBridge, PeerKey}; +use super::{ConnectionError, NetworkBridge, PeerId}; use crate::{ config::GlobalExecutor, message::NetMessage, - node::{network_event_log::NetEventLog, NetEventRegister, OpManager}, + node::{ + network_event_log::NetEventLog, testing_impl::NetworkBridgeExt, NetEventRegister, OpManager, + }, }; static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = @@ -26,12 +29,12 @@ pub(in crate::node) struct MemoryConnManager { log_register: Arc>>, op_manager: Arc, msg_queue: Arc>>, - peer: PeerKey, + peer: PeerId, } impl MemoryConnManager { pub fn new( - peer: PeerKey, + peer: PeerId, log_register: Box, op_manager: Arc, add_noise: bool, @@ -61,17 +64,22 @@ impl MemoryConnManager { peer, } } +} - pub async fn recv(&self) -> Result { - loop { - let mut queue = self.msg_queue.lock().await; - let Some(msg) = queue.pop() else { - std::mem::drop(queue); - tokio::time::sleep(Duration::from_millis(10)).await; - continue; - }; - return Ok(msg); +impl NetworkBridgeExt for MemoryConnManager { + fn recv(&mut self) -> BoxFuture<'_, Result> { + async { + loop { + let mut queue = self.msg_queue.lock().await; + let Some(msg) = queue.pop() else { + std::mem::drop(queue); + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + }; + return Ok(msg); + } } + .boxed() } } @@ -95,7 +103,7 @@ impl Clone for MemoryConnManager { #[async_trait::async_trait] impl NetworkBridge for MemoryConnManager { - async fn send(&self, target: &PeerKey, msg: NetMessage) -> super::ConnResult<()> { + async fn send(&self, target: &PeerId, msg: NetMessage) -> super::ConnResult<()> { self.log_register .try_lock() .expect("unique lock") @@ -107,25 +115,25 @@ impl NetworkBridge for MemoryConnManager { Ok(()) } - async fn add_connection(&mut self, _peer: PeerKey) -> super::ConnResult<()> { + async fn add_connection(&mut self, _peer: PeerId) -> super::ConnResult<()> { Ok(()) } - async fn drop_connection(&mut self, _peer: &PeerKey) -> super::ConnResult<()> { + async fn drop_connection(&mut self, _peer: &PeerId) -> super::ConnResult<()> { Ok(()) } } #[derive(Clone, Debug)] struct MessageOnTransit { - origin: PeerKey, - target: PeerKey, + origin: PeerId, + target: PeerId, data: Vec, } #[derive(Clone, Debug)] pub struct InMemoryTransport { - interface_peer: PeerKey, + interface_peer: PeerId, /// received messages per each peer awaiting processing msg_stack_queue: Arc>>, /// all messages 'traversing' the network at a given time @@ -133,7 +141,7 @@ pub struct InMemoryTransport { } impl InMemoryTransport { - fn new(interface_peer: PeerKey, add_noise: bool) -> Self { + fn new(interface_peer: PeerId, add_noise: bool) -> Self { let msg_stack_queue = Arc::new(Mutex::new(Vec::new())); let (network_tx, network_rx) = NETWORK_WIRES.get_or_init(crossbeam::channel::unbounded); @@ -199,7 +207,7 @@ impl InMemoryTransport { } } - fn send(&self, peer: PeerKey, message: Vec) { + fn send(&self, peer: PeerId, message: Vec) { let send_res = self.network.send(MessageOnTransit { origin: self.interface_peer, target: peer, diff --git a/crates/core/src/node/network_bridge/inter_process.rs b/crates/core/src/node/network_bridge/inter_process.rs index 50a399582..ab83fec90 100644 --- a/crates/core/src/node/network_bridge/inter_process.rs +++ b/crates/core/src/node/network_bridge/inter_process.rs @@ -1,20 +1,38 @@ -use crate::{dev_tool::PeerKey, message::NetMessage}; +use futures::future::BoxFuture; -use super::NetworkBridge; +use crate::{ + message::NetMessage, + node::{testing_impl::NetworkBridgeExt, PeerId}, +}; +use super::{ConnectionError, NetworkBridge}; + +#[derive(Clone)] pub(in crate::node) struct InterProcessConnManager {} +impl InterProcessConnManager { + pub fn new() -> Self { + Self {} + } +} + +impl NetworkBridgeExt for InterProcessConnManager { + fn recv(&mut self) -> BoxFuture> { + todo!() + } +} + #[async_trait::async_trait] impl NetworkBridge for InterProcessConnManager { - async fn send(&self, target: &PeerKey, msg: NetMessage) -> super::ConnResult<()> { + async fn send(&self, target: &PeerId, msg: NetMessage) -> super::ConnResult<()> { todo!() } - async fn add_connection(&mut self, _peer: PeerKey) -> super::ConnResult<()> { + async fn add_connection(&mut self, _peer: PeerId) -> super::ConnResult<()> { Ok(()) } - async fn drop_connection(&mut self, _peer: &PeerKey) -> super::ConnResult<()> { + async fn drop_connection(&mut self, _peer: &PeerId) -> super::ConnResult<()> { Ok(()) } } diff --git a/crates/core/src/node/network_bridge/p2p_protoc.rs b/crates/core/src/node/network_bridge/p2p_protoc.rs index af7d69fec..9296e6898 100644 --- a/crates/core/src/node/network_bridge/p2p_protoc.rs +++ b/crates/core/src/node/network_bridge/p2p_protoc.rs @@ -30,7 +30,7 @@ use libp2p::{ KeepAlive, NetworkBehaviour, NotifyHandler, Stream as NegotiatedSubstream, SubstreamProtocol, SwarmEvent, ToSwarm, }, - InboundUpgrade, Multiaddr, OutboundUpgrade, PeerId, Swarm, + InboundUpgrade, Multiaddr, OutboundUpgrade, PeerId as Libp2pPeerId, Swarm, }; use tokio::sync::{ mpsc::{self, Receiver, Sender}, @@ -47,7 +47,7 @@ use crate::{ message::{NetMessage, NodeEvent, Transaction, TransactionType}, node::{ handle_cancelled_op, join_ring_request, network_event_log::NetEventLog, process_message, - InitPeerNode, NetEventRegister, NodeBuilder, OpManager, PeerKey, + InitPeerNode, NetEventRegister, NodeConfig, OpManager, PeerId as FreenetPeerId, }, operations::OpError, ring::PeerKeyLocation, @@ -122,11 +122,11 @@ fn multiaddr_from_connection(conn: (IpAddr, u16)) -> Multiaddr { addr } -type P2pBridgeEvent = Either<(PeerKey, Box), NodeEvent>; +type P2pBridgeEvent = Either<(FreenetPeerId, Box), NodeEvent>; pub(crate) struct P2pBridge { - active_net_connections: Arc>, - accepted_peers: Arc>, + active_net_connections: Arc>, + accepted_peers: Arc>, ev_listener_tx: Sender, op_manager: Arc, log_register: Arc>>, @@ -197,7 +197,7 @@ impl Clone for P2pBridge { #[async_trait::async_trait] impl NetworkBridge for P2pBridge { - async fn add_connection(&mut self, peer: PeerKey) -> super::ConnResult<()> { + async fn add_connection(&mut self, peer: FreenetPeerId) -> super::ConnResult<()> { if self.active_net_connections.contains_key(&peer) { self.accepted_peers.insert(peer); } @@ -208,7 +208,7 @@ impl NetworkBridge for P2pBridge { Ok(()) } - async fn drop_connection(&mut self, peer: &PeerKey) -> super::ConnResult<()> { + async fn drop_connection(&mut self, peer: &FreenetPeerId) -> super::ConnResult<()> { self.accepted_peers.remove(peer); self.ev_listener_tx .send(Right(NodeEvent::DropConnection(*peer))) @@ -222,7 +222,7 @@ impl NetworkBridge for P2pBridge { Ok(()) } - async fn send(&self, target: &PeerKey, msg: NetMessage) -> super::ConnResult<()> { + async fn send(&self, target: &FreenetPeerId, msg: NetMessage) -> super::ConnResult<()> { self.log_register .try_lock() .expect("single reference") @@ -249,8 +249,8 @@ pub(in crate::node) struct P2pConnManager { impl P2pConnManager { pub fn build( - transport: transport::Boxed<(PeerId, muxing::StreamMuxerBox)>, - config: &NodeBuilder, + transport: transport::Boxed<(Libp2pPeerId, muxing::StreamMuxerBox)>, + config: &NodeConfig, op_manager: Arc, event_listener: impl NetEventRegister + Clone, private_key: Keypair, @@ -282,7 +282,7 @@ impl P2pConnManager { let mut swarm = Swarm::new( transport, behaviour, - config.public_key.0, + config.peer_id.0, SwarmConfig::with_executor(global_executor) .with_idle_connection_timeout(config::PEER_TIMEOUT), ); @@ -327,7 +327,7 @@ impl P2pConnManager { let mut pending_from_executor = HashSet::new(); let mut tx_to_client: HashMap = HashMap::new(); - let this_peer = super::PeerKey( + let this_peer = FreenetPeerId::from( crate::config::Config::conf() .local_peer_keypair .public() @@ -342,7 +342,7 @@ impl P2pConnManager { } SwarmEvent::ConnectionClosed { peer_id, .. } => { Ok(Right(ConnMngrActions::ConnectionClosed { - peer: PeerKey::from(peer_id), + peer: FreenetPeerId::from(peer_id), })) } SwarmEvent::Dialing { peer_id, .. } => { @@ -355,13 +355,13 @@ impl P2pConnManager { if let identify::Event::Received { peer_id, info } = *id { if Self::is_compatible_peer(&info) { Ok(Right(ConnMngrActions::ConnectionEstablished { - peer: PeerKey(peer_id), + peer: FreenetPeerId::from(peer_id), address: info.observed_addr, })) } else { tracing::warn!("Incompatible peer: {}, disconnecting", peer_id); Ok(Right(ConnMngrActions::ConnectionClosed { - peer: PeerKey::from(peer_id), + peer: FreenetPeerId::from(peer_id), })) } } else { @@ -378,7 +378,7 @@ impl P2pConnManager { "Successful autonat probe, established conn with {peer} @ {address}" ); Ok(Right(ConnMngrActions::ConnectionEstablished { - peer: PeerKey(peer), + peer: FreenetPeerId::from(peer), address, })) } @@ -597,22 +597,22 @@ impl P2pConnManager { enum ConnMngrActions { /// Received a new connection ConnectionEstablished { - peer: PeerKey, + peer: FreenetPeerId, address: Multiaddr, }, /// Closed a connection with the peer ConnectionClosed { - peer: PeerKey, + peer: FreenetPeerId, }, /// Outbound message SendMessage { - peer: PeerKey, + peer: FreenetPeerId, msg: Box, }, /// Update self own public address, useful when communicating for first time UpdatePublicAddr(Multiaddr), /// This is private, so when establishing connections hole-punching should be performed - IsPrivatePeer(PeerId), + IsPrivatePeer(Libp2pPeerId), NodeAction(NodeEvent), ClosedChannel, NoAction, @@ -621,12 +621,12 @@ enum ConnMngrActions { /// Manages network connections with different peers and event routing within the swarm. pub(in crate::node) struct FreenetBehaviour { // FIFO queue for outbound messages - outbound: VecDeque<(PeerId, Either)>, + outbound: VecDeque<(Libp2pPeerId, Either)>, // FIFO queue for inbound messages inbound: VecDeque>, - routing_table: HashMap>, - connected: HashMap, - openning_connection: HashSet, + routing_table: HashMap>, + connected: HashMap, + openning_connection: HashSet, op_manager: Arc, } @@ -638,7 +638,7 @@ impl NetworkBehaviour for FreenetBehaviour { fn handle_established_inbound_connection( &mut self, connection_id: ConnectionId, - peer_id: PeerId, + peer_id: Libp2pPeerId, _local_addr: &Multiaddr, remote_addr: &Multiaddr, ) -> Result, libp2p::swarm::ConnectionDenied> { @@ -655,7 +655,7 @@ impl NetworkBehaviour for FreenetBehaviour { fn handle_established_outbound_connection( &mut self, connection_id: ConnectionId, - peer_id: PeerId, + peer_id: Libp2pPeerId, addr: &Multiaddr, _role_override: libp2p::core::Endpoint, ) -> Result, libp2p::swarm::ConnectionDenied> { @@ -671,7 +671,7 @@ impl NetworkBehaviour for FreenetBehaviour { fn on_connection_handler_event( &mut self, - peer_id: PeerId, + peer_id: Libp2pPeerId, _connection: ConnectionId, event: ::ToBehaviour, ) { diff --git a/crates/core/src/node/network_event_log.rs b/crates/core/src/node/network_event_log.rs index 1f7c0a552..8a41d5a8d 100644 --- a/crates/core/src/node/network_event_log.rs +++ b/crates/core/src/node/network_event_log.rs @@ -14,7 +14,7 @@ use tokio::{ }, }; -use super::PeerKey; +use super::PeerId; use crate::{ config::GlobalExecutor, contract::StoreResponse, @@ -104,7 +104,7 @@ impl Clone for CombinedRegister { #[derive(Clone)] pub(crate) struct NetEventLog<'a> { tx: &'a Transaction, - peer_id: &'a PeerKey, + peer_id: &'a PeerId, kind: EventKind, } @@ -121,7 +121,7 @@ impl<'a> NetEventLog<'a> { } } - pub fn disconnected(from: &'a PeerKey) -> Self { + pub fn disconnected(from: &'a PeerId) -> Self { NetEventLog { tx: Transaction::NULL, peer_id: from, @@ -294,7 +294,7 @@ impl<'a> NetEventLog<'a> { struct NetLogMessage { tx: Transaction, datetime: DateTime, - peer_id: PeerKey, + peer_id: PeerId, kind: EventKind, } @@ -949,14 +949,14 @@ enum EventKind { #[cfg_attr(test, derive(arbitrary::Arbitrary))] enum ConnectEvent { StartConnection { - from: PeerKey, + from: PeerId, }, Connected { this: PeerKeyLocation, connected: PeerKeyLocation, }, Finished { - initiator: PeerKey, + initiator: PeerId, location: Location, }, } @@ -965,11 +965,11 @@ enum ConnectEvent { #[cfg_attr(test, derive(arbitrary::Arbitrary))] enum PutEvent { Request { - performer: PeerKey, + performer: PeerId, key: ContractKey, }, PutSuccess { - requester: PeerKey, + requester: PeerId, value: WrappedState, }, BroadcastEmitted { @@ -982,7 +982,7 @@ enum PutEvent { }, BroadcastReceived { /// peer who started the broadcast op - requester: PeerKey, + requester: PeerId, /// key of the contract which value was being updated key: ContractKey, /// value that was put @@ -1025,7 +1025,7 @@ pub(super) mod test { for _ in 0..TEST_LOGS { let tx: Transaction = gen.arbitrary()?; transactions.push(tx); - let peer: PeerKey = gen.arbitrary()?; + let peer: PeerId = gen.arbitrary()?; peers.push(peer); } for _ in 0..TEST_LOGS { @@ -1048,7 +1048,7 @@ pub(super) mod test { #[derive(Clone)] pub(crate) struct TestEventListener { - node_labels: Arc>, + node_labels: Arc>, tx_log: Arc>>, logs: Arc>>, } @@ -1062,11 +1062,11 @@ pub(super) mod test { } } - pub fn add_node(&mut self, label: NodeLabel, peer: PeerKey) { + pub fn add_node(&mut self, label: NodeLabel, peer: PeerId) { self.node_labels.insert(label, peer); } - pub fn is_connected(&self, peer: &PeerKey) -> bool { + pub fn is_connected(&self, peer: &PeerId) -> bool { let logs = self.logs.lock(); logs.iter().any(|log| { &log.peer_id == peer @@ -1076,7 +1076,7 @@ pub(super) mod test { pub fn has_put_contract( &self, - peer: &PeerKey, + peer: &PeerId, for_key: &ContractKey, expected_value: &WrappedState, ) -> bool { @@ -1151,7 +1151,7 @@ pub(super) mod test { false } - pub fn has_got_contract(&self, peer: &PeerKey, expected_key: &ContractKey) -> bool { + pub fn has_got_contract(&self, peer: &PeerId, expected_key: &ContractKey) -> bool { let logs = self.logs.lock(); logs.iter().any(|log| { &log.peer_id == peer @@ -1159,11 +1159,7 @@ pub(super) mod test { }) } - pub fn is_subscribed_to_contract( - &self, - peer: &PeerKey, - expected_key: &ContractKey, - ) -> bool { + pub fn is_subscribed_to_contract(&self, peer: &PeerId, expected_key: &ContractKey) -> bool { let logs = self.logs.lock(); logs.iter().any(|log| { &log.peer_id == peer @@ -1172,7 +1168,7 @@ pub(super) mod test { } /// Unique connections for a given peer and their relative distance to other peers. - pub fn connections(&self, peer: PeerKey) -> impl Iterator { + pub fn connections(&self, peer: PeerId) -> impl Iterator { let logs = self.logs.lock(); let disconnects = logs .iter() @@ -1253,13 +1249,13 @@ pub(super) mod test { #[test] fn test_get_connections() -> Result<(), anyhow::Error> { use crate::ring::Location; - let peer_id = PeerKey::random(); + let peer_id = PeerId::random(); let loc = Location::try_from(0.5)?; let tx = Transaction::new::(); let locations = [ - (PeerKey::random(), Location::try_from(0.5)?), - (PeerKey::random(), Location::try_from(0.75)?), - (PeerKey::random(), Location::try_from(0.25)?), + (PeerId::random(), Location::try_from(0.5)?), + (PeerId::random(), Location::try_from(0.75)?), + (PeerId::random(), Location::try_from(0.25)?), ]; let mut listener = TestEventListener::new(); diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 8e5b6c0c7..d92c5ea22 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -17,7 +17,7 @@ use crate::{ ring::{LiveTransactionTracker, PeerKeyLocation, Ring}, }; -use super::{network_bridge::EventLoopNotificationsSender, NetEventRegister, NodeBuilder, PeerKey}; +use super::{network_bridge::EventLoopNotificationsSender, NetEventRegister, NodeConfig, PeerId}; #[cfg(debug_assertions)] macro_rules! check_id_op { @@ -62,11 +62,11 @@ impl OpManager { pub(super) fn new( notification_channel: EventLoopNotificationsSender, contract_handler: ContractHandlerChannel, - builder: &NodeBuilder, + config: &NodeConfig, gateways: &[PeerKeyLocation], event_register: ER, ) -> Result { - let ring = Ring::new::(builder, gateways, notification_channel.clone())?; + let ring = Ring::new::(config, gateways, notification_channel.clone())?; let ops = Arc::new(Ops::default()); let (new_transactions, rx) = tokio::sync::mpsc::channel(100); @@ -208,7 +208,7 @@ impl OpManager { } /// Notify the operation manager that a transaction is being transacted over the network. - pub fn sending_transaction(&self, peer: &PeerKey, msg: &NetMessage) { + pub fn sending_transaction(&self, peer: &PeerId, msg: &NetMessage) { let transaction = msg.id(); if let Some(loc) = msg.requested_location() { self.ring diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index 2c21cd23c..9ce45d30c 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -7,14 +7,14 @@ use libp2p::{ }, dns, identity::Keypair, - noise, tcp, yamux, PeerId, Transport, + noise, tcp, yamux, PeerId as Libp2pPeerId, Transport, }; use tracing::Instrument; use super::{ client_event_handling, join_ring_request, network_bridge::{p2p_protoc::P2pConnManager, EventLoopNotifications}, - NetEventRegister, PeerKey, + NetEventRegister, PeerId as FreenetPeerId, }; use crate::{ client_events::{combinator::ClientEventsCombinator, BoxedClient}, @@ -24,14 +24,14 @@ use crate::{ NetworkEventListenerHalve, }, message::NodeEvent, - node::NodeBuilder, + node::NodeConfig, util::IterExt, }; use super::OpManager; pub(super) struct NodeP2P { - pub(crate) peer_key: PeerKey, + pub(crate) peer_key: FreenetPeerId, pub(crate) op_manager: Arc, notification_channel: EventLoopNotifications, pub(super) conn_manager: P2pConnManager, @@ -76,7 +76,7 @@ impl NodeP2P { } pub(crate) async fn build( - builder: NodeBuilder, + builder: NodeConfig, private_key: Keypair, clients: [BoxedClient; CLIENTS], event_register: ER, @@ -86,7 +86,7 @@ impl NodeP2P { CH: ContractHandler + Send + 'static, ER: NetEventRegister + Clone, { - let peer_key = builder.public_key; + let peer_key = builder.peer_id; let gateways = builder.get_gateways()?; let (notification_channel, notification_tx) = EventLoopNotifications::channel(); @@ -154,7 +154,7 @@ impl NodeP2P { /// - Multiplexing using [Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). fn config_transport( local_key: &Keypair, - ) -> std::io::Result> { + ) -> std::io::Result> { let tcp = tcp::tokio::Transport::new(tcp::Config::new().nodelay(true).port_reuse(true)); let with_dns = dns::tokio::Transport::system(tcp)?; Ok(with_dns @@ -214,21 +214,21 @@ mod test { async fn ping() -> Result<(), ()> { let peer1_port = get_free_port().unwrap(); let peer1_key = Keypair::generate_ed25519(); - let peer1_id: PeerId = peer1_key.public().into(); + let peer1_id: Libp2pPeerId = peer1_key.public().into(); let peer1_config = InitPeerNode::new(peer1_id, Location::random()) .listening_ip(Ipv4Addr::LOCALHOST) .listening_port(peer1_port); let peer2_key = Keypair::generate_ed25519(); - let peer2_id: PeerId = peer2_key.public().into(); + let peer2_id: Libp2pPeerId = peer2_key.public().into(); - let (_, receiver1) = channel((0, PeerKey::from(peer1_id))); - let (_, receiver2) = channel((0, PeerKey::from(peer2_id))); + let (_, receiver1) = channel((0, FreenetPeerId::from(peer1_id))); + let (_, receiver2) = channel((0, FreenetPeerId::from(peer2_id))); // Start up the initial node. GlobalExecutor::spawn(async move { - let user_events = MemoryEventsGen::new(receiver1, PeerKey::from(peer1_id)); - let mut config = NodeBuilder::new(); + let user_events = MemoryEventsGen::new(receiver1, FreenetPeerId::from(peer1_id)); + let mut config = NodeConfig::new(); config .with_ip(Ipv4Addr::LOCALHOST) .with_port(peer1_port) @@ -250,8 +250,8 @@ mod test { // Start up the dialing node let dialer = GlobalExecutor::spawn(async move { - let user_events = MemoryEventsGen::new(receiver2, PeerKey::from(peer2_id)); - let mut config = NodeBuilder::new(); + let user_events = MemoryEventsGen::new(receiver2, FreenetPeerId::from(peer2_id)); + let mut config = NodeConfig::new(); config .add_gateway(peer1_config.clone()) .with_key(peer2_key.public().into()); diff --git a/crates/core/src/node/testing_impl.rs b/crates/core/src/node/testing_impl.rs index a8c1fd08a..38dade048 100644 --- a/crates/core/src/node/testing_impl.rs +++ b/crates/core/src/node/testing_impl.rs @@ -6,9 +6,11 @@ use std::{ time::{Duration, Instant}, }; +use either::Either; use freenet_stdlib::prelude::*; +use futures::future::BoxFuture; use itertools::Itertools; -use libp2p::{identity, PeerId}; +use libp2p::{identity, PeerId as Libp2pPeerId}; use rand::{seq::SliceRandom, Rng}; use tokio::sync::watch::{channel, Receiver, Sender}; use tracing::{info, Instrument}; @@ -18,7 +20,9 @@ use crate::node::network_event_log::CombinedRegister; use crate::{ client_events::test::{MemoryEventsGen, RandomEventGenerator}, config::GlobalExecutor, - node::{network_event_log::TestEventListener, InitPeerNode, NetEventRegister, NodeBuilder}, + contract, + message::{NetMessage, NodeEvent}, + node::{network_event_log::TestEventListener, InitPeerNode, NetEventRegister, NodeConfig}, ring::{Distance, Location, PeerKeyLocation}, }; @@ -27,7 +31,7 @@ mod inter_process; use self::inter_process::SimPeer; -use super::PeerKey; +use super::{network_bridge::EventLoopNotifications, ConnectionError, NetworkBridge, PeerId}; pub fn get_free_port() -> Result { let mut port; @@ -124,7 +128,7 @@ pub(crate) struct NodeSpecification { struct GatewayConfig { label: NodeLabel, port: u16, - id: PeerId, + id: Libp2pPeerId, location: Location, } @@ -162,8 +166,8 @@ type DefaultRegistry = CombinedRegister<2>; type DefaultRegistry = TestEventListener; pub(super) struct Builder { - pub(super) peer_key: PeerKey, - builder: NodeBuilder, + pub(super) peer_key: PeerId, + config: NodeConfig, ch_builder: String, add_noise: bool, event_register: ER, @@ -174,15 +178,15 @@ pub(super) struct Builder { impl Builder { /// Buils an in-memory node. Does nothing upon construction, pub fn build( - builder: NodeBuilder, + builder: NodeConfig, event_register: ER, ch_builder: String, add_noise: bool, ) -> Builder { - let peer_key = builder.public_key; + let peer_key = builder.peer_id; Builder { peer_key, - builder, + config: builder, ch_builder, add_noise, event_register, @@ -196,10 +200,10 @@ impl Builder { pub struct SimNetwork { name: String, debug: bool, - labels: Vec<(NodeLabel, PeerKey)>, + labels: Vec<(NodeLabel, PeerId)>, pub(crate) event_listener: TestEventListener, - user_ev_controller: Sender<(EventId, PeerKey)>, - receiver_ch: Receiver<(EventId, PeerKey)>, + user_ev_controller: Sender<(EventId, PeerId)>, + receiver_ch: Receiver<(EventId, PeerId)>, number_of_gateways: usize, gateways: Vec<(Builder, GatewayConfig)>, number_of_nodes: usize, @@ -241,7 +245,7 @@ impl SimNetwork { min_connections: usize, ) -> Self { assert!(gateways > 0 && nodes > 0); - let (user_ev_controller, receiver_ch) = channel((0, PeerKey::random())); + let (user_ev_controller, receiver_ch) = channel((0, PeerId::random())); let mut net = Self { name: name.into(), debug: false, @@ -292,7 +296,7 @@ impl SimNetwork { let port = get_free_port().unwrap(); let location = Location::random(); - let mut config = NodeBuilder::new(); + let mut config = NodeConfig::new(); config .with_ip(Ipv6Addr::LOCALHOST) .with_port(port) @@ -304,7 +308,7 @@ impl SimNetwork { .rnd_if_htl_above(self.rnd_if_htl_above); self.event_listener - .add_node(label.clone(), PeerKey::from(id)); + .add_node(label.clone(), PeerId::from(id)); configs.push(( config, GatewayConfig { @@ -368,7 +372,7 @@ impl SimNetwork { let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); - let mut config = NodeBuilder::new(); + let mut config = NodeConfig::new(); for GatewayConfig { port, id, location, .. } in &gateways @@ -385,7 +389,7 @@ impl SimNetwork { .max_number_of_connections(self.max_connections) .with_key(pair.public().into()); - let peer = PeerKey::from(id); + let peer = PeerId::from(id); self.event_listener.add_node(label.clone(), peer); let event_listener = { @@ -440,8 +444,8 @@ impl SimNetwork { } self.labels.push((label, node.peer_key)); - let node_task = async move { node.run_node(user_events).await }; - GlobalExecutor::spawn(node_task.instrument(span)); + let node_task = async move { node.run_node(user_events, span).await }; + GlobalExecutor::spawn(node_task); tokio::time::sleep(self.init_backoff).await; } @@ -472,8 +476,8 @@ impl SimNetwork { }; self.labels.push((label, node.peer_key)); - let node_task = async move { node.run_node(user_events).await }; - let handle = GlobalExecutor::spawn(node_task.instrument(span)); + let node_task = async move { node.run_node(user_events, span).await }; + let handle = GlobalExecutor::spawn(node_task); peers.push(handle); tokio::time::sleep(self.init_backoff).await; @@ -486,9 +490,14 @@ impl SimNetwork { pub fn build_peers(&mut self) -> Vec<(NodeLabel, SimPeer)> { let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); let mut peers = vec![]; - for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { - self.labels.push((label.clone(), node.peer_key)); - peers.push((label, SimPeer(node))); + for (builder, label) in gw.chain(self.nodes.drain(..)).collect::>() { + self.labels.push((label.clone(), builder.peer_key)); + peers.push(( + label, + SimPeer { + config: builder.config, + }, + )); } self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); peers.sort_by(|(a, _), (b, _)| a.cmp(b)); @@ -581,7 +590,7 @@ impl SimNetwork { /// Returns the connectivity in the network per peer (that is all the connections /// this peers has registered). - pub fn node_connectivity(&self) -> HashMap)> { + pub fn node_connectivity(&self) -> HashMap)> { let mut peers_connections = HashMap::with_capacity(self.labels.len()); let key_to_label: HashMap<_, _> = self.labels.iter().map(|(k, v)| (v, k)).collect(); for (label, key) in &self.labels { @@ -629,7 +638,7 @@ impl SimNetwork { pub fn event_chain( mut self, total_events: usize, - controller: Option>, + controller: Option>, ) -> EventChain { const SEED: u64 = 0xdeadbeef; if let Some(controller) = controller { @@ -797,7 +806,7 @@ fn group_locations_in_buckets( } fn pretty_print_connections( - conns: &HashMap)>, + conns: &HashMap)>, ) -> String { let mut connections = String::from("Node connections:\n"); let mut conns = conns.iter().collect::>(); @@ -836,3 +845,226 @@ fn group_locations_test() -> Result<(), anyhow::Error> { Ok(()) } + +use super::op_state_manager::OpManager; +use crate::client_events::ClientEventsProxy; + +pub(super) trait NetworkBridgeExt: Clone + 'static { + fn recv(&mut self) -> BoxFuture>; +} + +struct RunnerConfig +where + NB: NetworkBridge, + UsrEv: ClientEventsProxy + Send + 'static, +{ + peer_key: PeerId, + is_gateway: bool, + gateways: Vec, + parent_span: Option, + op_storage: Arc, + conn_manager: NB, + /// Set on creation, taken on run + user_events: Option, + notification_channel: EventLoopNotifications, + event_register: Box, +} + +async fn run_node(mut config: RunnerConfig) -> Result<(), anyhow::Error> +where + NB: NetworkBridge + NetworkBridgeExt, + UsrEv: ClientEventsProxy + Send + 'static, +{ + use crate::util::IterExt; + if !config.is_gateway { + if let Some(gateway) = config.gateways.iter().shuffle().take(1).next() { + super::join_ring_request( + None, + config.peer_key, + gateway, + &config.op_storage, + &mut config.conn_manager, + ) + .await?; + } else { + anyhow::bail!("requires at least one gateway"); + } + } + let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); + let span = { + config + .parent_span + .clone() + .map(|parent_span| { + tracing::info_span!( + parent: parent_span, + "client_event_handling", + peer = %config.peer_key + ) + }) + .unwrap_or_else( + || tracing::info_span!("client_event_handling", peer = %config.peer_key), + ) + }; + let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); + GlobalExecutor::spawn( + super::client_event_handling( + config.op_storage.clone(), + config.user_events.take().expect("should be set"), + client_responses, + node_controller_tx, + ) + .instrument(span), + ); + let parent_span: tracing::Span = config + .parent_span + .clone() + .unwrap_or_else(|| tracing::info_span!("event_listener", peer = %config.peer_key)); + run_event_listener(cli_response_sender, node_controller_rx, config) + .instrument(parent_span) + .await +} + +/// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. +async fn run_event_listener( + _client_responses: contract::ClientResponsesSender, + mut node_controller_rx: tokio::sync::mpsc::Receiver, + RunnerConfig { + peer_key, + is_gateway, + gateways, + parent_span, + op_storage, + mut conn_manager, + mut notification_channel, + mut event_register, + .. + }: RunnerConfig, +) -> Result<(), anyhow::Error> +where + NB: NetworkBridge + NetworkBridgeExt, + UsrEv: ClientEventsProxy + Send + 'static, +{ + use crate::util::IterExt; + loop { + let msg = tokio::select! { + msg = conn_manager.recv() => { msg.map(Either::Left) } + msg = notification_channel.recv() => { + if let Some(msg) = msg { + Ok(msg.map_left(|(msg, _cli_id)| msg)) + } else { + anyhow::bail!("notification channel shutdown, fatal error"); + } + } + msg = node_controller_rx.recv() => { + if let Some(msg) = msg { + Ok(Either::Right(msg)) + } else { + anyhow::bail!("node controller channel shutdown, fatal error"); + } + } + }; + + if let Ok(Either::Left(NetMessage::Aborted(tx))) = msg { + let tx_type = tx.transaction_type(); + let res = + super::handle_cancelled_op(tx, peer_key, &op_storage, &mut conn_manager).await; + match res { + Err(crate::operations::OpError::MaxRetriesExceeded(_, _)) + if tx_type == crate::message::TransactionType::Connect && !is_gateway => + { + tracing::warn!("Retrying joining the ring with an other peer"); + if let Some(gateway) = gateways.iter().shuffle().next() { + super::join_ring_request( + None, + peer_key, + gateway, + &op_storage, + &mut conn_manager, + ) + .await? + } else { + anyhow::bail!("requires at least one gateway"); + } + } + Err(err) => return Err(anyhow::anyhow!(err)), + Ok(_) => {} + } + continue; + } + + let msg = match msg { + Ok(Either::Left(msg)) => msg, + Ok(Either::Right(action)) => match action { + NodeEvent::ShutdownNode => break Ok(()), + NodeEvent::DropConnection(peer) => { + tracing::info!("Dropping connection to {peer}"); + event_register.register_events(Either::Left( + crate::node::network_event_log::NetEventLog::disconnected(&peer), + )); + op_storage.ring.prune_connection(peer); + continue; + } + NodeEvent::Disconnect { cause: Some(cause) } => { + tracing::info!(peer = %peer_key, "Shutting down node, reason: {cause}"); + return Ok(()); + } + NodeEvent::Disconnect { cause: None } => { + tracing::info!(peer = %peer_key, "Shutting down node"); + return Ok(()); + } + other => { + unreachable!("event {other:?}, shouldn't happen in the in-memory impl") + } + }, + Err(err) => { + super::report_result( + None, + Err(err.into()), + &op_storage, + None, + None, + &mut *event_register as &mut _, + ) + .await; + continue; + } + }; + + let op_storage = op_storage.clone(); + let conn_manager = conn_manager.clone(); + let event_listener = event_register.trait_clone(); + + let span = { + parent_span + .clone() + .map(|parent_span| { + tracing::info_span!( + parent: parent_span.clone(), + "process_network_message", + peer = %peer_key, transaction = %msg.id(), + tx_type = %msg.id().transaction_type() + ) + }) + .unwrap_or_else(|| { + tracing::info_span!( + "process_network_message", + peer = %peer_key, transaction = %msg.id(), + tx_type = %msg.id().transaction_type() + ) + }) + }; + + let msg = super::process_message( + msg, + op_storage, + conn_manager, + event_listener, + None, + None, + None, + ) + .instrument(span); + GlobalExecutor::spawn(msg); + } +} diff --git a/crates/core/src/node/testing_impl/in_memory.rs b/crates/core/src/node/testing_impl/in_memory.rs index e1a925ac7..170120571 100644 --- a/crates/core/src/node/testing_impl/in_memory.rs +++ b/crates/core/src/node/testing_impl/in_memory.rs @@ -1,38 +1,34 @@ use std::{collections::HashMap, sync::Arc}; -use either::Either; use freenet_stdlib::prelude::*; use tracing::Instrument; -use crate::node::{ - client_event_handling, handle_cancelled_op, join_ring_request, - network_bridge::{in_memory::MemoryConnManager, EventLoopNotifications}, - network_event_log::NetEventLog, - op_state_manager::OpManager, - process_message, NetEventRegister, PeerKey, -}; use crate::{ client_events::ClientEventsProxy, config::GlobalExecutor, - contract::{ - self, executor_channel, ClientResponsesSender, ContractHandler, ExecutorToEventLoopChannel, - MemoryContractHandler, NetworkEventListenerHalve, + contract::{self, executor_channel, ContractHandler, MemoryContractHandler}, + node::{ + network_bridge::{in_memory::MemoryConnManager, EventLoopNotifications}, + op_state_manager::OpManager, + NetEventRegister, NetworkBridge, }, - message::{NetMessage, NodeEvent, TransactionType}, - operations::OpError, ring::PeerKeyLocation, - util::IterExt, }; -use super::Builder; +use super::{Builder, RunnerConfig}; -impl Builder { - pub async fn run_node(self, user_events: UsrEv) -> Result<(), anyhow::Error> +impl Builder { + pub async fn run_node( + self, + user_events: UsrEv, + parent_span: tracing::Span, + ) -> Result<(), anyhow::Error> where UsrEv: ClientEventsProxy + Send + 'static, + ER: NetEventRegister + Clone, { - let gateways = self.builder.get_gateways()?; - let is_gateway = self.builder.local_ip.zip(self.builder.local_port).is_some(); + let gateways = self.config.get_gateways()?; + let is_gateway = self.config.local_ip.zip(self.config.local_port).is_some(); let (notification_channel, notification_tx) = EventLoopNotifications::channel(); let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); @@ -40,7 +36,7 @@ impl Builder { let op_storage = Arc::new(OpManager::new( notification_tx, ops_ch_channel, - &self.builder, + &self.config, &gateways, self.event_register.clone(), )?); @@ -57,26 +53,25 @@ impl Builder { self.add_noise, ); - let parent_span = tracing::Span::current(); GlobalExecutor::spawn( contract::contract_handling(contract_handler) .instrument(tracing::info_span!(parent: parent_span.clone(), "contract_handling")), ); - let mut running_node = InMemoryNode { + let mut config = super::RunnerConfig { peer_key: self.peer_key, - op_storage, + is_gateway, gateways, - notification_channel, + parent_span: Some(parent_span), + op_storage, conn_manager, + user_events: Some(user_events), + notification_channel, event_register: self.event_register.trait_clone(), - is_gateway, - _executor_listener, - parent_span, }; - running_node + config .append_contracts(self.contracts, self.contract_subscribers) .await?; - running_node.run_node(user_events).await + super::run_node(config).await } #[cfg(test)] @@ -90,20 +85,11 @@ impl Builder { } } -struct InMemoryNode { - peer_key: PeerKey, - op_storage: Arc, - gateways: Vec, - notification_channel: EventLoopNotifications, - conn_manager: MemoryConnManager, - event_register: Box, - is_gateway: bool, - _executor_listener: ExecutorToEventLoopChannel, - /// Span to use for this node for tracing purposes - parent_span: tracing::Span, -} - -impl InMemoryNode { +impl RunnerConfig +where + NB: NetworkBridge, + UsrEv: ClientEventsProxy + Send + 'static, +{ async fn append_contracts( &mut self, contracts: Vec<(ContractContainer, WrappedState)>, @@ -150,160 +136,4 @@ impl InMemoryNode { } Ok(()) } - - pub async fn run_node(mut self, user_events: UsrEv) -> Result<(), anyhow::Error> - where - UsrEv: ClientEventsProxy + Send + 'static, - { - if !self.is_gateway { - if let Some(gateway) = self.gateways.iter().shuffle().take(1).next() { - join_ring_request( - None, - self.peer_key, - gateway, - &self.op_storage, - &mut self.conn_manager, - ) - .await?; - } else { - anyhow::bail!("requires at least one gateway"); - } - } - let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); - let parent_span = self.parent_span.clone(); - let (node_controller_tx, node_controller_rx) = tokio::sync::mpsc::channel(1); - GlobalExecutor::spawn( - client_event_handling( - self.op_storage.clone(), - user_events, - client_responses, - node_controller_tx, - ) - .instrument(tracing::info_span!(parent: parent_span, "client_event_handling")), - ); - let parent_span: tracing::Span = self.parent_span.clone(); - self.run_event_listener(cli_response_sender, node_controller_rx) - .instrument(parent_span) - .await - } - - /// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. - async fn run_event_listener( - &mut self, - _client_responses: ClientResponsesSender, - mut node_controller_rx: tokio::sync::mpsc::Receiver, - ) -> Result<(), anyhow::Error> { - loop { - let msg = tokio::select! { - msg = self.conn_manager.recv() => { msg.map(Either::Left) } - msg = self.notification_channel.recv() => { - if let Some(msg) = msg { - Ok(msg.map_left(|(msg, _cli_id)| msg)) - } else { - anyhow::bail!("notification channel shutdown, fatal error"); - } - } - msg = node_controller_rx.recv() => { - if let Some(msg) = msg { - Ok(Either::Right(msg)) - } else { - anyhow::bail!("node controller channel shutdown, fatal error"); - } - } - }; - - if let Ok(Either::Left(NetMessage::Aborted(tx))) = msg { - let tx_type = tx.transaction_type(); - let res = handle_cancelled_op( - tx, - self.peer_key, - &self.op_storage, - &mut self.conn_manager, - ) - .await; - match res { - Err(OpError::MaxRetriesExceeded(_, _)) - if tx_type == TransactionType::Connect && !self.is_gateway => - { - tracing::warn!("Retrying joining the ring with an other peer"); - if let Some(gateway) = self.gateways.iter().shuffle().next() { - join_ring_request( - None, - self.peer_key, - gateway, - &self.op_storage, - &mut self.conn_manager, - ) - .await? - } else { - anyhow::bail!("requires at least one gateway"); - } - } - Err(err) => return Err(anyhow::anyhow!(err)), - Ok(_) => {} - } - continue; - } - - let msg = match msg { - Ok(Either::Left(msg)) => msg, - Ok(Either::Right(action)) => match action { - NodeEvent::ShutdownNode => break Ok(()), - NodeEvent::DropConnection(peer) => { - tracing::info!("Dropping connection to {peer}"); - self.event_register - .register_events(Either::Left(NetEventLog::disconnected(&peer))); - self.op_storage.ring.prune_connection(peer); - continue; - } - NodeEvent::Disconnect { cause: Some(cause) } => { - tracing::info!(peer = %self.peer_key, "Shutting down node, reason: {cause}"); - return Ok(()); - } - NodeEvent::Disconnect { cause: None } => { - tracing::info!(peer = %self.peer_key, "Shutting down node"); - return Ok(()); - } - other => { - unreachable!("event {other:?}, shouldn't happen in the in-memory impl") - } - }, - Err(err) => { - super::super::report_result( - None, - Err(err.into()), - &self.op_storage, - None, - None, - &mut *self.event_register as &mut _, - ) - .await; - continue; - } - }; - - let op_storage = self.op_storage.clone(); - let conn_manager = self.conn_manager.clone(); - let event_listener = self.event_register.trait_clone(); - - let parent_span = tracing::Span::current(); - let span = tracing::info_span!( - parent: parent_span, - "process_network_message", - peer = %self.peer_key, transaction = %msg.id(), - tx_type = %msg.id().transaction_type() - ); - let msg = process_message( - msg, - op_storage, - conn_manager, - event_listener, - None, - None, - None, - ) - .instrument(span); - GlobalExecutor::spawn(msg); - } - } } diff --git a/crates/core/src/node/testing_impl/inter_process.rs b/crates/core/src/node/testing_impl/inter_process.rs index cf4750570..2cbc3b6c5 100644 --- a/crates/core/src/node/testing_impl/inter_process.rs +++ b/crates/core/src/node/testing_impl/inter_process.rs @@ -1,32 +1,77 @@ use std::sync::Arc; +use tracing::Instrument; + use crate::{ - contract::{ExecutorToEventLoopChannel, NetworkEventListenerHalve}, - dev_tool::PeerKey, + config::GlobalExecutor, + contract::{self, ContractHandler, MemoryContractHandler}, + dev_tool::{ClientEventsProxy, NodeConfig, PeerId}, node::{ network_bridge::{inter_process::InterProcessConnManager, EventLoopNotifications}, NetEventRegister, OpManager, }, - ring::PeerKeyLocation, }; -use super::{Builder, DefaultRegistry}; - -pub struct SimPeer(pub(super) Builder); +pub struct SimPeer { + pub(super) config: NodeConfig, +} impl SimPeer { - pub fn peer_key(&self) -> PeerKey { - self.0.peer_key + pub fn peer_key(&self) -> PeerId { + self.config.peer_id } } -struct InterProcessNode { - peer_key: PeerKey, - op_storage: Arc, - gateways: Vec, - notification_channel: EventLoopNotifications, - conn_manager: InterProcessConnManager, - event_register: Box, - is_gateway: bool, - _executor_listener: ExecutorToEventLoopChannel, +impl SimPeer { + pub(crate) async fn run_node( + self, + user_events: UsrEv, + event_register: ER, + ) -> Result<(), anyhow::Error> + where + UsrEv: ClientEventsProxy + Send + 'static, + ER: NetEventRegister + Send + Clone + 'static, + { + let gateways = self.config.get_gateways()?; + let is_gateway = self.config.local_ip.zip(self.config.local_port).is_some(); + + let (notification_channel, notification_tx) = EventLoopNotifications::channel(); + let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); + + let op_storage = Arc::new(OpManager::new( + notification_tx, + ops_ch_channel, + &self.config, + &gateways, + event_register.clone(), + )?); + let (_executor_listener, executor_sender) = contract::executor_channel(op_storage.clone()); + let contract_handler = MemoryContractHandler::build( + ch_channel, + executor_sender, + self.config.peer_id.to_string(), + ) + .await + .map_err(|e| anyhow::anyhow!(e))?; + + let conn_manager = InterProcessConnManager::new(); + + GlobalExecutor::spawn( + contract::contract_handling(contract_handler) + .instrument(tracing::info_span!("contract_handling")), + ); + + let running_node = super::RunnerConfig { + peer_key: self.config.peer_id, + op_storage, + gateways, + notification_channel, + conn_manager, + event_register: Box::new(event_register), + is_gateway, + user_events: Some(user_events), + parent_span: None, + }; + super::run_node(running_node).await + } } diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index b363538c2..175032dc9 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -9,7 +9,7 @@ use crate::{ client_events::{ClientId, HostResult}, contract::ContractError, message::{InnerMessage, NetMessage, Transaction, TransactionType}, - node::{ConnectionError, NetworkBridge, OpManager, OpNotAvailable, PeerKey}, + node::{ConnectionError, NetworkBridge, OpManager, OpNotAvailable, PeerId}, ring::{Location, PeerKeyLocation, RingError}, DynError, }; @@ -28,7 +28,7 @@ pub(crate) struct OperationResult { } pub(crate) struct OpInitialization { - sender: Option, + sender: Option, op: Op, } @@ -58,7 +58,7 @@ async fn handle_op_result( network_bridge: &mut CB, result: Result, tx_id: Transaction, - sender: Option, + sender: Option, ) -> Result, OpError> where CB: NetworkBridge, diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index d63e54a73..0738d81c7 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -8,7 +8,7 @@ use super::{OpError, OpInitialization, OpOutcome, Operation, OperationResult}; use crate::{ client_events::ClientId, message::{InnerMessage, NetMessage, Transaction}, - node::{ConnectionError, NetworkBridge, OpManager, PeerKey}, + node::{ConnectionError, NetworkBridge, OpManager, PeerId}, operations::OpEnum, ring::{Location, PeerKeyLocation, Ring}, util::ExponentialBackoff, @@ -692,7 +692,7 @@ fn try_returning_proxy_connection( sender: &PeerKeyLocation, own_loc: &PeerKeyLocation, accepted_by: HashSet, - joiner: PeerKey, + joiner: PeerId, ) -> (Option, Option) { let new_state = if accepted_by.contains(own_loc) { tracing::debug!( @@ -760,13 +760,13 @@ enum ConnectState { target: PeerKeyLocation, accepted_by: HashSet, new_location: Location, - new_peer_id: PeerKey, + new_peer_id: PeerId, }, AwaitingConnectionAcquisition { joiner: PeerKeyLocation, }, AwaitingNewConnection { - query_target: PeerKey, + query_target: PeerId, }, OCReceived, Connected, @@ -775,7 +775,7 @@ enum ConnectState { #[derive(Debug, Clone)] struct ConnectionInfo { gateway: PeerKeyLocation, - this_peer: PeerKey, + this_peer: PeerId, max_hops_to_live: usize, } @@ -802,7 +802,7 @@ impl ConnectState { } pub(crate) fn initial_request( - this_peer: PeerKey, + this_peer: PeerId, gateway: PeerKeyLocation, max_hops_to_live: usize, id: Transaction, @@ -894,7 +894,7 @@ async fn forward_conn( (req_peer, joiner): (PeerKeyLocation, PeerKeyLocation), left_htl: usize, accepted_by: HashSet, - skip_list: Vec, + skip_list: Vec, ) -> Result, OpError> where NB: NetworkBridge, @@ -1042,7 +1042,7 @@ mod messages { } impl ConnectMsg { - pub fn sender(&self) -> Option<&PeerKey> { + pub fn sender(&self) -> Option<&PeerId> { use ConnectMsg::*; match self { Response { sender, .. } => Some(&sender.peer), @@ -1093,7 +1093,7 @@ mod messages { pub(crate) enum ConnectRequest { StartReq { target: PeerKeyLocation, - joiner: PeerKey, + joiner: PeerId, assigned_location: Option, hops_to_live: usize, max_hops_to_live: usize, @@ -1111,7 +1111,7 @@ mod messages { sender: PeerKeyLocation, joiner: PeerKeyLocation, hops_to_live: usize, - skip_list: Vec, + skip_list: Vec, accepted_by: HashSet, }, ReceivedOC, @@ -1122,7 +1122,7 @@ mod messages { AcceptedBy { peers: HashSet, your_location: Location, - your_peer_id: PeerKey, + your_peer_id: PeerId, }, ReceivedOC { by_peer: PeerKeyLocation, @@ -1130,7 +1130,7 @@ mod messages { }, Proxy { accepted_by: HashSet, - joiner: PeerKey, + joiner: PeerId, }, } } diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 173c587a9..ce7ab17d6 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -9,7 +9,7 @@ use crate::{ client_events::ClientId, contract::{ContractError, ContractHandlerEvent, StoreResponse}, message::{InnerMessage, NetMessage, Transaction}, - node::{NetworkBridge, OpManager, PeerKey}, + node::{NetworkBridge, OpManager, PeerId}, operations::{OpInitialization, Operation}, ring::{Location, PeerKeyLocation, RingError}, DynError, @@ -142,7 +142,7 @@ impl Operation for GetOp { msg: &'a Self::Message, ) -> BoxFuture<'a, Result, OpError>> { async move { - let mut sender: Option = None; + let mut sender: Option = None; if let Some(peer_key_loc) = msg.sender().cloned() { sender = Some(peer_key_loc.peer); }; @@ -693,7 +693,7 @@ enum GetState { }, /// Awaiting response from petition. AwaitingResponse { - skip_list: Vec, + skip_list: Vec, retries: usize, fetch_contract: bool, }, @@ -706,7 +706,7 @@ pub(crate) async fn request_get( client_id: Option, ) -> Result<(), OpError> { let (target, id) = if let Some(GetState::PrepareRequest { key, id, .. }) = &get_op.state { - const EMPTY: &[PeerKey] = &[]; + const EMPTY: &[PeerId] = &[]; // the initial request must provide: // - a location in the network where the contract resides // - and the key of the contract value to get diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 1f83c25c0..6a7a7b3b5 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -16,7 +16,7 @@ use crate::{ client_events::ClientId, contract::ContractHandlerEvent, message::{InnerMessage, NetMessage, Transaction}, - node::{NetworkBridge, OpManager, PeerKey}, + node::{NetworkBridge, OpManager, PeerId}, ring::{Location, PeerKeyLocation, RingError}, }; @@ -128,7 +128,7 @@ impl Operation for PutOp { msg: &'a Self::Message, ) -> BoxFuture<'a, Result, OpError>> { async move { - let mut sender: Option = None; + let mut sender: Option = None; if let Some(peer_key_loc) = msg.sender().cloned() { sender = Some(peer_key_loc.peer); }; @@ -344,7 +344,7 @@ impl Operation for PutOp { .map(|i| { // Avoid already broadcast nodes and sender from broadcasting let mut subscribers: Vec = i.value().to_vec(); - let mut avoid_list: HashSet = + let mut avoid_list: HashSet = sender_subscribers.iter().map(|pl| pl.peer).collect(); avoid_list.insert(sender.peer); subscribers.retain(|s| !avoid_list.contains(&s.peer)); @@ -770,7 +770,7 @@ async fn forward_changes( { let key = contract.key(); let contract_loc = Location::from(&key); - const EMPTY: &[PeerKey] = &[]; + const EMPTY: &[PeerId] = &[]; let forward_to = op_storage.ring.closest_caching(&key, EMPTY); let own_loc = op_storage.ring.own_location().location.expect("infallible"); if let Some(peer) = forward_to { diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index dfb08dfb3..7bf7a5fdd 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -10,7 +10,7 @@ use crate::{ client_events::ClientId, contract::ContractError, message::{InnerMessage, NetMessage, Transaction}, - node::{NetworkBridge, OpManager, PeerKey}, + node::{NetworkBridge, OpManager, PeerId}, ring::{Location, PeerKeyLocation, RingError}, }; @@ -57,7 +57,7 @@ impl Operation for SubscribeOp { msg: &'a Self::Message, ) -> BoxFuture<'a, Result, OpError>> { async move { - let mut sender: Option = None; + let mut sender: Option = None; if let Some(peer_key_loc) = msg.sender().cloned() { sender = Some(peer_key_loc.peer); }; @@ -329,7 +329,7 @@ enum SubscribeState { ReceivedRequest, /// Awaitinh response from petition. AwaitingResponse { - skip_list: Vec, + skip_list: Vec, retries: usize, }, Completed, @@ -347,7 +347,7 @@ pub(crate) async fn request_subscribe( key.clone(), ))); } - const EMPTY: &[PeerKey] = &[]; + const EMPTY: &[PeerId] = &[]; ( op_storage .ring @@ -404,7 +404,7 @@ mod messages { key: ContractKey, target: PeerKeyLocation, subscriber: PeerKeyLocation, - skip_list: Vec, + skip_list: Vec, htl: usize, }, ReturnSub { diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index e42a69942..f284d0da5 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -44,7 +44,7 @@ use crate::{ config::GlobalExecutor, message::Transaction, node::{ - self, EventLoopNotificationsSender, EventRegister, NetEventRegister, NodeBuilder, PeerKey, + self, EventLoopNotificationsSender, EventRegister, NetEventRegister, NodeConfig, PeerId, }, operations::connect, router::Router, @@ -55,7 +55,7 @@ use crate::{ #[cfg_attr(test, derive(arbitrary::Arbitrary))] /// The location of a peer in the ring. This location allows routing towards the peer. pub struct PeerKeyLocation { - pub peer: PeerKey, + pub peer: PeerId, /// An unspecified location means that the peer hasn't been asigned a location, yet. pub location: Option, } @@ -64,14 +64,14 @@ impl PeerKeyLocation { #[cfg(test)] pub fn random() -> Self { PeerKeyLocation { - peer: PeerKey::random(), + peer: PeerId::random(), location: Some(Location::random()), } } } -impl From for PeerKeyLocation { - fn from(peer: PeerKey) -> Self { +impl From for PeerKeyLocation { + fn from(peer: PeerId) -> Self { PeerKeyLocation { peer, location: None, @@ -101,13 +101,13 @@ struct Connection { #[derive(Clone)] pub(crate) struct LiveTransactionTracker { - tx_per_peer: Arc>>, - missing_candidate_sender: sync::mpsc::Sender, + tx_per_peer: Arc>>, + missing_candidate_sender: sync::mpsc::Sender, } impl LiveTransactionTracker { /// The given peer does not have (good) candidates for acquiring new connections. - pub async fn missing_candidate_peers(&self, peer: PeerKey) { + pub async fn missing_candidate_peers(&self, peer: PeerId) { let _ = self .missing_candidate_sender .send(peer) @@ -118,12 +118,12 @@ impl LiveTransactionTracker { }); } - pub fn add_transaction(&self, peer: PeerKey, tx: Transaction) { + pub fn add_transaction(&self, peer: PeerId, tx: Transaction) { self.tx_per_peer.entry(peer).or_default().push(tx); } pub fn remove_finished_transaction(&self, tx: Transaction) { - let keys_to_remove: Vec = self + let keys_to_remove: Vec = self .tx_per_peer .iter() .filter(|entry| entry.value().iter().any(|otx| otx == &tx)) @@ -138,11 +138,11 @@ impl LiveTransactionTracker { } } - fn prune_transactions_from_peer(&self, peer: &PeerKey) { + fn prune_transactions_from_peer(&self, peer: &PeerId) { self.tx_per_peer.remove(peer); } - fn new() -> (Self, sync::mpsc::Receiver) { + fn new() -> (Self, sync::mpsc::Receiver) { let (missing_peer, rx) = sync::mpsc::channel(10); ( Self { @@ -153,7 +153,7 @@ impl LiveTransactionTracker { ) } - fn has_live_connection(&self, peer: &PeerKey) -> bool { + fn has_live_connection(&self, peer: &PeerId) -> bool { self.tx_per_peer.contains_key(peer) } @@ -171,7 +171,7 @@ impl LiveTransactionTracker { pub(crate) struct Ring { pub rnd_if_htl_above: usize, pub max_hops_to_live: usize, - pub peer_key: PeerKey, + pub peer_key: PeerId, max_connections: usize, min_connections: usize, router: Arc>, @@ -180,7 +180,7 @@ pub(crate) struct Ring { /// Slow is for when there are enough connections so we need to drop a connection in order to replace it. fast_acquisition: AtomicBool, connections_by_location: RwLock>>, - location_for_peer: RwLock>, + location_for_peer: RwLock>, /// contracts in the ring cached by this node cached_contracts: DashSet, own_location: AtomicU64, @@ -225,13 +225,13 @@ impl Ring { const MAX_HOPS_TO_LIVE: usize = 10; pub fn new( - config: &NodeBuilder, + config: &NodeConfig, gateways: &[PeerKeyLocation], event_loop_notifier: EventLoopNotificationsSender, ) -> Result, anyhow::Error> { let (live_tx_tracker, missing_candidate_rx) = LiveTransactionTracker::new(); - let peer_key = config.public_key; + let peer_key = config.peer_id; // for location here consider -1 == None let own_location = AtomicU64::new(u64::from_le_bytes((-1f64).to_le_bytes())); @@ -438,7 +438,7 @@ impl Ring { .record_request(requested_location, request_type); } - pub fn add_connection(&self, loc: Location, peer: PeerKey) { + pub fn add_connection(&self, loc: Location, peer: PeerId) { let mut cbl = self.connections_by_location.write(); cbl.entry(loc).or_default().push(Connection { location: PeerKeyLocation { @@ -464,7 +464,7 @@ impl Ring { pub fn closest_caching( &self, contract_key: &ContractKey, - skip_list: impl Contains, + skip_list: impl Contains, ) -> Option { self.routing(Location::from(contract_key), None, skip_list) } @@ -473,8 +473,8 @@ impl Ring { pub fn routing( &self, target: Location, - requesting: Option<&PeerKey>, - skip_list: impl Contains, + requesting: Option<&PeerId>, + skip_list: impl Contains, ) -> Option { let connections = self.connections_by_location.read(); let peers = connections.values().filter_map(|conns| { @@ -497,7 +497,7 @@ impl Ring { /// Get a random peer from the known ring connections. pub fn random_peer(&self, filter_fn: F) -> Option where - F: Fn(&PeerKey) -> bool, + F: Fn(&PeerId) -> bool, { let peers = &*self.location_for_peer.read(); let amount = peers.len(); @@ -564,7 +564,7 @@ impl Ring { self.connections_by_location.read().len() } - pub fn prune_connection(&self, peer: PeerKey) { + pub fn prune_connection(&self, peer: PeerId) { #[cfg(debug_assertions)] { tracing::info!(%peer, "Removing connection"); @@ -590,7 +590,7 @@ impl Ring { pub fn closest_to_location( &self, location: Location, - skip_list: &[PeerKey], + skip_list: &[PeerId], ) -> Option { self.connections_by_location .read() @@ -617,12 +617,10 @@ impl Ring { self: Arc, notifier: EventLoopNotificationsSender, live_tx_tracker: LiveTransactionTracker, - mut missing_candidates: sync::mpsc::Receiver, + mut missing_candidates: sync::mpsc::Receiver, ) -> Result<(), DynError> { /// Peers whose connection should be acquired. - fn should_swap<'a>( - _connections: impl Iterator, - ) -> Vec { + fn should_swap<'a>(_connections: impl Iterator) -> Vec { // todo: instead we should be using ConnectionEvaluator here vec![] } @@ -786,7 +784,7 @@ impl Ring { async fn acquire_new( &self, ideal_location: Location, - skip_list: &[&PeerKey], + skip_list: &[&PeerId], notifier: &EventLoopNotificationsSender, missing_connections: usize, ) -> Result, DynError> { diff --git a/crates/core/src/runtime/store.rs b/crates/core/src/runtime/store.rs index 09c372249..c44e4793e 100644 --- a/crates/core/src/runtime/store.rs +++ b/crates/core/src/runtime/store.rs @@ -33,9 +33,9 @@ impl SafeWriter { }; let s = Self { file: BufWriter::new(file), - _marker: std::marker::PhantomData, compact, lock_file_path: path.with_extension("lock"), + _marker: std::marker::PhantomData, }; Ok(s) } @@ -304,7 +304,6 @@ fn compact_index_file(key_file_path: &Path) -> std::io::Re } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { // The lock file already exists, so a compaction is in progress - println!("locked"); return Ok(()); } Err(e) => { @@ -593,7 +592,6 @@ mod tests { return Err(err); } barrier.wait(); - println!("next compaction (this should print 4 times"); // compact a last time so we know what data to compare against super::compact_index_file::(&key_file_path).map_err(|err| { eprintln!("Thread encountered an error during compaction: {err}"); diff --git a/crates/core/src/util.rs b/crates/core/src/util.rs index 18fa7f9be..c4ae05e6c 100644 --- a/crates/core/src/util.rs +++ b/crates/core/src/util.rs @@ -8,7 +8,7 @@ use rand::{ SeedableRng, }; -use crate::node::PeerKey; +use crate::node::PeerId; pub fn set_cleanup_on_exit() -> Result<(), ctrlc::Error> { ctrlc::set_handler(move || { @@ -225,14 +225,14 @@ pub(crate) trait Contains { fn has_element(&self, target: &T) -> bool; } -impl<'x> Contains for &'x [PeerKey] { - fn has_element(&self, target: &PeerKey) -> bool { +impl<'x> Contains for &'x [PeerId] { + fn has_element(&self, target: &PeerId) -> bool { self.contains(target) } } -impl<'x> Contains for &'x [&PeerKey] { - fn has_element(&self, target: &PeerKey) -> bool { +impl<'x> Contains for &'x [&PeerId] { + fn has_element(&self, target: &PeerId) -> bool { self.contains(&target) } } diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index 7b155dd26..aeca1ecd8 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -130,7 +130,7 @@ async fn config_sim_network(base_config: &TestConfig) -> Result Result<(), super::Error> { let mut simulated_network = super::config_sim_network(base_config).await?; let peers = simulated_network.build_peers(); - let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerKey::random())); + let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerId::random())); for (label, node) in peers { let mut user_events = MemoryEventsGen::::new_with_seed( receiver_ch.clone(), From 7169d5d730c76ca7fa8f319c82d10425422194da Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 17 Nov 2023 16:57:15 +0100 Subject: [PATCH 09/12] Set child proc from supervisor proc --- crates/core/src/config.rs | 2 +- crates/core/src/lib.rs | 5 +- crates/core/src/node.rs | 2 +- crates/core/src/node/testing_impl.rs | 11 +- .../src/node/testing_impl/inter_process.rs | 46 +++- crates/core/src/util.rs | 2 +- crates/fdev/src/config.rs | 11 + crates/fdev/src/main.rs | 46 ++-- crates/fdev/src/testing.rs | 208 +++++++++++++++--- 9 files changed, 267 insertions(+), 66 deletions(-) diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 1c05a8703..fa3f22345 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -372,7 +372,7 @@ pub fn set_logger() { } #[cfg(feature = "trace")] -pub(super) mod tracer { +mod tracer { use tracing_subscriber::{Layer, Registry}; use crate::DynError; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index dea062df5..f0ae6a72c 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -29,7 +29,10 @@ pub mod dev_tool { pub use crate::config::Config; pub use client_events::{test::MemoryEventsGen, ClientEventsProxy, ClientId, OpenRequest}; pub use contract::{storages::Storage, Executor, OperationMode}; - pub use node::{testing_impl::SimNetwork, InitPeerNode, NodeConfig, PeerCliConfig, PeerId}; + pub use node::{ + testing_impl::{NodeLabel, SimNetwork, SimPeer}, + InitPeerNode, NodeConfig, PeerCliConfig, PeerId, + }; pub use ring::Location; pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index c69ae424b..7e930d1a6 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -93,7 +93,7 @@ impl Node { #[derive(Serialize, Deserialize)] pub struct NodeConfig { /// public identifier for the peer - pub(crate) peer_id: PeerId, + pub peer_id: PeerId, // optional local info, in case this is an initial bootstrap node /// IP to bind to the listener pub(crate) local_ip: Option, diff --git a/crates/core/src/node/testing_impl.rs b/crates/core/src/node/testing_impl.rs index 38dade048..f641ace01 100644 --- a/crates/core/src/node/testing_impl.rs +++ b/crates/core/src/node/testing_impl.rs @@ -29,7 +29,7 @@ use crate::{ mod in_memory; mod inter_process; -use self::inter_process::SimPeer; +pub use self::inter_process::SimPeer; use super::{network_bridge::EventLoopNotifications, ConnectionError, NetworkBridge, PeerId}; @@ -487,17 +487,12 @@ impl SimNetwork { } /// Builds peer nodes and returns the controller to trigger events. - pub fn build_peers(&mut self) -> Vec<(NodeLabel, SimPeer)> { + pub fn build_peers(&mut self) -> Vec<(NodeLabel, NodeConfig)> { let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); let mut peers = vec![]; for (builder, label) in gw.chain(self.nodes.drain(..)).collect::>() { self.labels.push((label.clone(), builder.peer_key)); - peers.push(( - label, - SimPeer { - config: builder.config, - }, - )); + peers.push((label, builder.config)); } self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); peers.sort_by(|(a, _), (b, _)| a.cmp(b)); diff --git a/crates/core/src/node/testing_impl/inter_process.rs b/crates/core/src/node/testing_impl/inter_process.rs index 2cbc3b6c5..1d191bd90 100644 --- a/crates/core/src/node/testing_impl/inter_process.rs +++ b/crates/core/src/node/testing_impl/inter_process.rs @@ -1,31 +1,53 @@ use std::sync::Arc; +use serde::{Deserialize, Serialize}; use tracing::Instrument; use crate::{ config::GlobalExecutor, contract::{self, ContractHandler, MemoryContractHandler}, - dev_tool::{ClientEventsProxy, NodeConfig, PeerId}, + dev_tool::{ClientEventsProxy, NodeConfig}, node::{ network_bridge::{inter_process::InterProcessConnManager, EventLoopNotifications}, - NetEventRegister, OpManager, + EventRegister, NetEventRegister, OpManager, }, }; +#[derive(Serialize, Deserialize)] +#[serde(transparent)] pub struct SimPeer { pub(super) config: NodeConfig, } impl SimPeer { - pub fn peer_key(&self) -> PeerId { - self.config.peer_id + pub async fn start_child(self, event_generator: UsrEv) -> Result<(), anyhow::Error> + where + UsrEv: ClientEventsProxy + Send + 'static, + { + // disable console logging but enable trace collector (if it's enabled) + std::env::set_var("FREENET_DISABLE_LOGS", "1"); + crate::config::set_logger(); + + let event_register = { + #[cfg(feature = "trace-ot")] + { + use crate::node::network_event_log::OTEventRegister; + crate::node::CombinedRegister::new([ + Box::new(EventRegister::new()), + Box::new(OTEventRegister::new()), + ]) + } + #[cfg(not(feature = "trace-ot"))] + { + EventRegister::new() + } + }; + self.run_node(event_generator, event_register).await } -} -impl SimPeer { - pub(crate) async fn run_node( + async fn run_node( self, - user_events: UsrEv, + event_generator: UsrEv, event_register: ER, ) -> Result<(), anyhow::Error> where @@ -69,9 +91,15 @@ impl SimPeer { conn_manager, event_register: Box::new(event_register), is_gateway, - user_events: Some(user_events), + user_events: Some(event_generator), parent_span: None, }; super::run_node(running_node).await } } + +impl From for SimPeer { + fn from(config: NodeConfig) -> Self { + Self { config } + } +} diff --git a/crates/core/src/util.rs b/crates/core/src/util.rs index c4ae05e6c..b7089bb98 100644 --- a/crates/core/src/util.rs +++ b/crates/core/src/util.rs @@ -247,7 +247,7 @@ pub mod tests { let dir = tempfile::Builder::new() .tempdir() .expect("Failed to create a temporary directory"); - eprintln!("Created temp dir: {:?}", dir.path()); + // eprintln!("Created temp dir: {:?}", dir.path()); dir } } diff --git a/crates/fdev/src/config.rs b/crates/fdev/src/config.rs index dcbaea38d..3dfff3e0e 100644 --- a/crates/fdev/src/config.rs +++ b/crates/fdev/src/config.rs @@ -43,6 +43,17 @@ pub enum SubCommand { Test(crate::testing::TestConfig), } +impl SubCommand { + pub fn is_child(&self) -> bool { + if let SubCommand::Test(config) = self { + if let crate::testing::TestMode::MultiProcess(config) = &config.command { + return matches!(config.mode, crate::testing::Process::Child); + } + } + false + } +} + /// Core CLI tool for interacting with the Freenet local node. /// /// This tool allows the execution of commands against the local node diff --git a/crates/fdev/src/main.rs b/crates/fdev/src/main.rs index 114f4456e..fe8b3d3db 100644 --- a/crates/fdev/src/main.rs +++ b/crates/fdev/src/main.rs @@ -32,26 +32,34 @@ enum Error { CommandFailed(&'static str), } -#[tokio::main] -async fn main() -> Result<(), anyhow::Error> { - freenet::config::set_logger(); - let cwd = std::env::current_dir()?; +fn main() -> Result<(), anyhow::Error> { + let tokio_rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; let config = Config::parse(); freenet::config::Config::set_op_mode(config.additional.mode); - let r = match config.sub_command { - SubCommand::WasmRuntime(local_node_config) => run_local_executor(local_node_config).await, - SubCommand::Build(build_tool_config) => build_package(build_tool_config, &cwd), - SubCommand::Inspect(inspect_config) => inspect(inspect_config), - SubCommand::New(new_pckg_config) => create_new_package(new_pckg_config), - SubCommand::Publish(publish_config) => put(publish_config, config.additional).await, - SubCommand::Execute(cmd_config) => match cmd_config.command { - config::NodeCommand::Put(put_config) => put(put_config, config.additional).await, - config::NodeCommand::Update(update_config) => { - update(update_config, config.additional).await + if !config.sub_command.is_child() { + freenet::config::set_logger(); + } + tokio_rt.block_on(async move { + let cwd = std::env::current_dir()?; + let r = match config.sub_command { + SubCommand::WasmRuntime(local_node_config) => { + run_local_executor(local_node_config).await } - }, - SubCommand::Test(test_config) => testing::test_framework(test_config).await, - }; - // todo: make all commands return concrete `thiserror` compatible errors so we can use anyhow - r.map_err(|e| anyhow::format_err!(e)) + SubCommand::Build(build_tool_config) => build_package(build_tool_config, &cwd), + SubCommand::Inspect(inspect_config) => inspect(inspect_config), + SubCommand::New(new_pckg_config) => create_new_package(new_pckg_config), + SubCommand::Publish(publish_config) => put(publish_config, config.additional).await, + SubCommand::Execute(cmd_config) => match cmd_config.command { + config::NodeCommand::Put(put_config) => put(put_config, config.additional).await, + config::NodeCommand::Update(update_config) => { + update(update_config, config.additional).await + } + }, + SubCommand::Test(test_config) => testing::test_framework(test_config).await, + }; + // todo: make all commands return concrete `thiserror` compatible errors so we can use anyhow + r.map_err(|e| anyhow::format_err!(e)) + }) } diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index aeca1ecd8..bdfe3d4f8 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -3,6 +3,8 @@ use std::time::Duration; use anyhow::Error; use freenet::dev_tool::SimNetwork; +pub(crate) use multiple_process::Process; + /// Testing framework for running Freenet network simulations. #[derive(clap::Parser, Clone)] pub struct TestConfig { @@ -84,7 +86,7 @@ pub enum TestMode { /// Runs multiple simulated nodes in a single process. SingleProcess, /// Runs multiple simulated nodes in multiple processes. - MultiProcess, + MultiProcess(multiple_process::MultiProcessConfig), /// Runs multiple simulated nodes in multiple processes and multiple machines. Network, } @@ -94,8 +96,8 @@ pub(crate) async fn test_framework(base_config: TestConfig) -> Result<(), Error> TestMode::SingleProcess => { single_process::run(&base_config).await?; } - TestMode::MultiProcess => { - todo!() + TestMode::MultiProcess(config) => { + multiple_process::run(&base_config, config).await?; } TestMode::Network => { todo!() @@ -130,25 +132,181 @@ async fn config_sim_network(base_config: &TestConfig) -> Result Vec { + let mut args = Vec::new(); + + args.push("test".to_owned()); + + if let Some(name) = &self.name { + args.push("--name".to_owned()); + args.push(name.to_string()); + } + + args.push("--seed".to_owned()); + args.push(seed.to_string()); + + args.push("--gateways".to_owned()); + args.push(self.gateways.to_string()); + args.push("--nodes".to_owned()); + args.push(self.nodes.to_string()); + args.push("--ring_max_htl".to_owned()); + args.push(self.ring_max_htl.to_string()); + args.push("--rnd_if_htl_above".to_owned()); + args.push(self.rnd_if_htl_above.to_string()); + args.push("--max_connections".to_owned()); + args.push(self.max_connections.to_string()); + args.push("--min_connections".to_owned()); + args.push(self.min_connections.to_string()); + + if let Some(max_contract_number) = self.max_contract_number { + args.push("--max_contract_number".to_owned()); + args.push(max_contract_number.to_string()); + } + + args.push("--events".to_owned()); + args.push(self.events.to_string()); - struct MultiProcessEventSender {} + if let Some(wait_duration) = self.wait_duration { + args.push("--wait_duration".to_owned()); + args.push(wait_duration.to_string()); + } + + if let Some(event_wait_time) = self.event_wait_time { + args.push("--event_wait_time".to_owned()); + args.push(event_wait_time.to_string()); + } - struct MultiProcessEventReceiver {} + args.push("multi-process".to_owned()); + args.push("child".to_owned()); - pub(super) async fn run(base_config: &super::TestConfig) -> Result<(), super::Error> { - let mut simulated_network = super::config_sim_network(base_config).await?; + args + } + } + + #[derive(clap::Parser, Clone)] + pub struct MultiProcessConfig { + #[arg(long)] + pub mode: Process, + #[arg(long)] + id: Option, + } + + #[derive(Default, Clone, clap::ValueEnum)] + pub enum Process { + #[default] + Supervisor, + Child, + } + + #[derive(Default)] + struct SubProcessManager { + processes: Vec, + } + + struct SubProcess { + label: NodeLabel, + child: std::process::Child, + } + + impl SubProcess { + fn start(cmd_args: &[String], label: NodeLabel) -> Result { + let cmd = Command::new("fdev") + .args(cmd_args) + .arg("--id") + .arg(label.number().to_string()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn()?; + Ok(Self { label, child: cmd }) + } + + fn config(&mut self, config: NodeConfig) -> Result<(), Error> { + let input = self.get_input().ok_or_else(|| anyhow!("closed"))?; + let serialize = bincode::serialize(&config)?; + input.write_all(&(serialize.len() as u64).to_le_bytes())?; + input.write_all(&serialize)?; + Ok(()) + } + + fn send_event(&mut self, event: usize) -> Result<(), std::io::Error> { + let input = self.get_input().ok_or(std::io::ErrorKind::BrokenPipe)?; + input.write_all(&event.to_le_bytes())?; + Ok(()) + } + + #[inline] + fn get_input(&mut self) -> Option<&mut std::process::ChildStdin> { + self.child.stdin.as_mut() + } + + fn close(mut self) { + let _ = self.child.kill(); + } + } + + pub(super) async fn run( + config: &super::TestConfig, + cmd_config: &MultiProcessConfig, + ) -> Result<(), Error> { + match cmd_config.mode { + Process::Supervisor => supervisor(config).await, + Process::Child => { + child(config, cmd_config.id.expect("id should be set for child")).await + } + } + } + + async fn supervisor(config: &super::TestConfig) -> Result<(), Error> { + let mut simulated_network = super::config_sim_network(config).await?; let peers = simulated_network.build_peers(); - let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerId::random())); + + let seed = config.seed(); + let mut manager = SubProcessManager::default(); + let cmd_args = config.subprocess_command(seed); for (label, node) in peers { - let mut user_events = MemoryEventsGen::::new_with_seed( - receiver_ch.clone(), - node.peer_key(), - base_config.seed(), - ); - // user_events.rng_params(label.number(), total_peer_num, max_contract_num, iterations); + let mut subprocess = SubProcess::start(&cmd_args, label)?; + subprocess.config(node)?; + manager.processes.push(subprocess); } - todo!() + Ok(()) + } + + async fn child(config: &super::TestConfig, id: usize) -> Result<(), Error> { + let mut input = std::io::stdin(); + let mut output = std::io::stdout(); + let mut buf = [0u8; 8]; + input.read_exact(&mut buf)?; + let config_len = u64::from_le_bytes(buf); + let mut config_buf = vec![0u8; config_len as usize]; + input.read_exact(&mut config_buf)?; + let node_config: NodeConfig = bincode::deserialize(&config_buf)?; + + let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerId::random())); + let mut event_generator = MemoryEventsGen::::new_with_seed( + receiver_ch.clone(), + node_config.peer_id, + config.seed.expect("seed should be set for child process"), + ); + event_generator.rng_params( + id, + config.gateways + config.nodes, + config.max_contract_number.unwrap_or(config.nodes * 10), + config.events, + ); + let config = SimPeer::from(node_config); + config.start_child(event_generator).await?; + Ok(()) } } @@ -158,23 +316,21 @@ mod single_process { use futures::StreamExt; use tokio::signal; - pub(super) async fn run(base_config: &super::TestConfig) -> Result<(), super::Error> { - let mut simulated_network = super::config_sim_network(base_config).await?; + pub(super) async fn run(config: &super::TestConfig) -> Result<(), super::Error> { + let mut simulated_network = super::config_sim_network(config).await?; let join_handles = simulated_network .start_with_rand_gen::( - base_config.seed(), - base_config - .max_contract_number - .unwrap_or(base_config.nodes * 10), - base_config.events, + config.seed(), + config.max_contract_number.unwrap_or(config.nodes * 10), + config.events, ) .await; - let events = base_config.events; - let next_event_wait_time = base_config.event_wait_time.map(Duration::from_millis); + let events = config.events; + let next_event_wait_time = config.event_wait_time.map(Duration::from_millis); let (connectivity_timeout, network_connection_percent) = - base_config.get_connection_check_params(); + config.get_connection_check_params(); let events_generated = tokio::task::spawn_blocking(move || { println!( "Waiting for network to be sufficiently connected ({}ms timeout, {}%)", From c505c7b8e467578ae71157747cfd13cfe0104a35 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Mon, 20 Nov 2023 17:43:30 +0100 Subject: [PATCH 10/12] Plug together supervior and IPC to set network --- crates/core/Cargo.toml | 2 +- crates/core/src/contract/executor.rs | 12 +- crates/core/src/contract/storages/rocks_db.rs | 11 +- crates/core/src/contract/storages/sqlite.rs | 49 ++- crates/core/src/lib.rs | 4 +- crates/core/src/node.rs | 1 + .../core/src/node/network_bridge/in_memory.rs | 44 +-- .../src/node/network_bridge/inter_process.rs | 61 ++- crates/core/src/node/testing_impl.rs | 153 +++++--- .../core/src/node/testing_impl/in_memory.rs | 2 +- crates/fdev/Cargo.toml | 2 +- crates/fdev/src/commands.rs | 6 +- crates/fdev/src/config.rs | 3 + crates/fdev/src/testing.rs | 348 ++++++++++++++---- crates/fdev/src/wasm_runtime/state.rs | 5 +- 15 files changed, 502 insertions(+), 201 deletions(-) diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index a91942bb0..6ae5af7c9 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -53,7 +53,7 @@ sqlx = { features = ["runtime-tokio-rustls", "sqlite"], optional = true, version stretto = { features = ["async", "sync"], version = "0.8" } tar = { version = "0.4.38" } thiserror = "1" -tokio = { features = ["fs", "macros", "rt-multi-thread", "sync"], version = "1" } +tokio = { features = ["fs", "macros", "rt-multi-thread", "sync", "process"], version = "1" } tower-http = { features = ["fs", "trace"], version = "0.4" } ulid = { features = ["serde"], version = "0.4" } unsigned-varint = "0.7" diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index d02734f81..e4d707a50 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -391,7 +391,9 @@ impl Executor { const MAX_MEM_CACHE: u32 = 10_000_000; let static_conf = crate::config::Config::conf(); - let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE).unwrap(); + let db_path = crate::config::Config::conf().db_dir().join("freenet.db"); + let state_store = + StateStore::new(Storage::new(Some(&db_path)).await?, MAX_MEM_CACHE).unwrap(); let contract_dir = config .node_data_dir @@ -1204,8 +1206,10 @@ impl Executor { std::fs::create_dir_all(&contracts_data_dir).expect("directory created"); let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; - // uses inmemory SQLite - let state_store = StateStore::new(Storage::new().await?, u16::MAX as u32).unwrap(); + let db_path = tmp_path.join("freenet.db"); + std::fs::create_dir_all(&db_path).expect("directory created"); + let state_store = + StateStore::new(Storage::new(Some(&db_path)).await?, u16::MAX as u32).unwrap(); let executor = Executor::new( state_store, @@ -1338,7 +1342,7 @@ mod test { const MAX_MEM_CACHE: u32 = 10_000_000; let tmp_path = std::env::temp_dir().join("freenet-test"); let contract_store = ContractStore::new(tmp_path.join("executor-test"), MAX_SIZE)?; - let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE).unwrap(); + let state_store = StateStore::new(Storage::new(None).await?, MAX_MEM_CACHE).unwrap(); let mut counter = 0; Executor::new( state_store, diff --git a/crates/core/src/contract/storages/rocks_db.rs b/crates/core/src/contract/storages/rocks_db.rs index 4192e73f1..1e5f94e2d 100644 --- a/crates/core/src/contract/storages/rocks_db.rs +++ b/crates/core/src/contract/storages/rocks_db.rs @@ -1,22 +1,23 @@ +use std::path::Path; + use freenet_stdlib::prelude::*; use rocksdb::{Options, DB}; +use crate::contract::ContractKey; use crate::runtime::StateStorage; -use crate::{config::Config, contract::ContractKey}; pub struct RocksDb(DB); impl RocksDb { #[cfg_attr(feature = "sqlite", allow(unused))] - pub async fn new() -> Result { - let path = Config::conf().db_dir().join("freenet.db"); - tracing::info!("loading contract store from {path:?}"); + pub async fn new(db_path: &Path) -> Result { + tracing::info!("loading contract store from {db_path:?}"); let mut opts = Options::default(); opts.create_if_missing(true); opts.set_log_level(rocksdb::LogLevel::Debug); - let db = DB::open(&opts, path).unwrap(); + let db = DB::open(&opts, db_path).unwrap(); Ok(Self(db)) } diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index 9a02fb335..39172a243 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -1,46 +1,25 @@ -use std::str::FromStr; +use std::{path::Path, str::FromStr}; use freenet_stdlib::prelude::*; -use once_cell::sync::Lazy; use sqlx::{ sqlite::{SqliteConnectOptions, SqliteRow}, ConnectOptions, Row, SqlitePool, }; use crate::{ - config::Config, contract::ContractKey, runtime::{ContractError, StateStorage, StateStoreError}, }; -// Is fine to clone this as it wraps by an Arc. -static POOL: Lazy = Lazy::new(|| { - let opts = if cfg!(test) { - SqliteConnectOptions::from_str("sqlite::memory:").unwrap() - } else { - let conn_str = Config::conf().db_dir().join("freenet.db"); - tracing::info!("loading contract store from {conn_str:?}"); - SqliteConnectOptions::new() - .create_if_missing(true) - .filename(conn_str) - }; - let opts = opts.log_statements(tracing::log::LevelFilter::Debug); - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current() - .block_on(async move { SqlitePool::connect_with(opts).await }) - }) - .unwrap() -}); - -async fn create_contracts_table() -> Result<(), SqlDbError> { +async fn create_contracts_table(pool: &SqlitePool) -> Result<(), SqlDbError> { sqlx::query( "CREATE TABLE IF NOT EXISTS states ( - contract BLOB PRIMARY KEY, + contract BLOB PRIMARY KEY, state BLOB, params BLOB )", ) - .execute(&*POOL) + .execute(pool) .await?; Ok(()) } @@ -49,9 +28,23 @@ async fn create_contracts_table() -> Result<(), SqlDbError> { pub struct Pool(SqlitePool); impl Pool { - pub async fn new() -> Result { - create_contracts_table().await?; - Ok(Self(POOL.clone())) + pub async fn new(db_path: Option<&Path>) -> Result { + let opts = if let Some(db_path) = db_path { + tracing::info!("loading contract store from {db_path:?}"); + SqliteConnectOptions::new() + .create_if_missing(true) + .filename(db_path) + } else { + SqliteConnectOptions::from_str("sqlite::memory:").unwrap() + }; + let opts = opts.log_statements(tracing::log::LevelFilter::Debug); + let pool = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current() + .block_on(async move { SqlitePool::connect_with(opts).await }) + }) + .unwrap(); + create_contracts_table(&pool).await?; + Ok(Self(pool.clone())) } } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index f0ae6a72c..cd837abcc 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -30,8 +30,8 @@ pub mod dev_tool { pub use client_events::{test::MemoryEventsGen, ClientEventsProxy, ClientId, OpenRequest}; pub use contract::{storages::Storage, Executor, OperationMode}; pub use node::{ - testing_impl::{NodeLabel, SimNetwork, SimPeer}, - InitPeerNode, NodeConfig, PeerCliConfig, PeerId, + testing_impl::{EventChain, NodeLabel, SimNetwork, SimPeer}, + InitPeerNode, InterProcessConnManager, NodeConfig, PeerCliConfig, PeerId, }; pub use ring::Location; pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 7e930d1a6..35ffd31a1 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -42,6 +42,7 @@ use crate::{ }; use crate::operations::handle_op_request; +pub use network_bridge::inter_process::InterProcessConnManager; pub(crate) use network_bridge::{ConnectionError, EventLoopNotificationsSender, NetworkBridge}; #[cfg(feature = "trace-ot")] pub(crate) use network_event_log::CombinedRegister; diff --git a/crates/core/src/node/network_bridge/in_memory.rs b/crates/core/src/node/network_bridge/in_memory.rs index 0d1923d86..d4547fcf4 100644 --- a/crates/core/src/node/network_bridge/in_memory.rs +++ b/crates/core/src/node/network_bridge/in_memory.rs @@ -21,11 +21,8 @@ use crate::{ }, }; -static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = - OnceCell::new(); - pub(in crate::node) struct MemoryConnManager { - pub transport: InMemoryTransport, + transport: InMemoryTransport, log_register: Arc>>, op_manager: Arc, msg_queue: Arc>>, @@ -66,23 +63,6 @@ impl MemoryConnManager { } } -impl NetworkBridgeExt for MemoryConnManager { - fn recv(&mut self) -> BoxFuture<'_, Result> { - async { - loop { - let mut queue = self.msg_queue.lock().await; - let Some(msg) = queue.pop() else { - std::mem::drop(queue); - tokio::time::sleep(Duration::from_millis(10)).await; - continue; - }; - return Ok(msg); - } - } - .boxed() - } -} - impl Clone for MemoryConnManager { fn clone(&self) -> Self { let log_register = loop { @@ -124,6 +104,23 @@ impl NetworkBridge for MemoryConnManager { } } +impl NetworkBridgeExt for MemoryConnManager { + fn recv(&mut self) -> BoxFuture<'_, Result> { + async { + loop { + let mut queue = self.msg_queue.lock().await; + let Some(msg) = queue.pop() else { + std::mem::drop(queue); + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + }; + return Ok(msg); + } + } + .boxed() + } +} + #[derive(Clone, Debug)] struct MessageOnTransit { origin: PeerId, @@ -131,8 +128,11 @@ struct MessageOnTransit { data: Vec, } +static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = + OnceCell::new(); + #[derive(Clone, Debug)] -pub struct InMemoryTransport { +struct InMemoryTransport { interface_peer: PeerId, /// received messages per each peer awaiting processing msg_stack_queue: Arc>>, diff --git a/crates/core/src/node/network_bridge/inter_process.rs b/crates/core/src/node/network_bridge/inter_process.rs index ab83fec90..628415055 100644 --- a/crates/core/src/node/network_bridge/inter_process.rs +++ b/crates/core/src/node/network_bridge/inter_process.rs @@ -1,4 +1,13 @@ -use futures::future::BoxFuture; +use std::sync::{Arc, OnceLock}; + +use futures::{future::BoxFuture, FutureExt}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + sync::{ + watch::{Receiver, Sender}, + Mutex, + }, +}; use crate::{ message::NetMessage, @@ -7,25 +16,65 @@ use crate::{ use super::{ConnectionError, NetworkBridge}; +type Data = Vec; + +static INCOMING_DATA: OnceLock> = OnceLock::new(); + #[derive(Clone)] -pub(in crate::node) struct InterProcessConnManager {} +pub struct InterProcessConnManager { + recv: Receiver, + output: Arc>, +} impl InterProcessConnManager { - pub fn new() -> Self { - Self {} + pub(in crate::node) fn new() -> Self { + let (sender, recv) = tokio::sync::watch::channel(vec![]); + INCOMING_DATA.set(sender).expect("shouldn't be set"); + Self { + recv, + output: Arc::new(Mutex::new(tokio::io::stdout())), + } + } + + pub fn push_msg(data: Vec) { + let _ = INCOMING_DATA.get().expect("should be set").send(data); + } + + pub async fn pull_msg( + stdout: &mut tokio::process::ChildStdout, + ) -> std::io::Result<(PeerId, Data)> { + let mut msg_len = [0u8; 4]; + stdout.read_exact(&mut msg_len).await?; + let msg_len = u32::from_le_bytes(msg_len) as usize; + let buf = &mut vec![0u8; msg_len]; + stdout.read_exact(buf).await?; + bincode::deserialize(buf).map_err(|_| std::io::ErrorKind::Other.into()) } } impl NetworkBridgeExt for InterProcessConnManager { fn recv(&mut self) -> BoxFuture> { - todo!() + async { + self.recv + .changed() + .await + .map_err(|_| ConnectionError::Timeout)?; + let data = &*self.recv.borrow(); + let deser = bincode::deserialize(data)?; + Ok(deser) + } + .boxed() } } #[async_trait::async_trait] impl NetworkBridge for InterProcessConnManager { async fn send(&self, target: &PeerId, msg: NetMessage) -> super::ConnResult<()> { - todo!() + let data = bincode::serialize(&(*target, msg)).expect("proper encoding"); + let output = &mut *self.output.lock().await; + output.write_all(&(data.len() as u32).to_le_bytes()).await?; + output.write_all(&data).await?; + Ok(()) } async fn add_connection(&mut self, _peer: PeerId) -> super::ConnResult<()> { diff --git a/crates/core/src/node/testing_impl.rs b/crates/core/src/node/testing_impl.rs index f641ace01..56df0101d 100644 --- a/crates/core/src/node/testing_impl.rs +++ b/crates/core/src/node/testing_impl.rs @@ -52,7 +52,7 @@ pub fn get_dynamic_port() -> u16 { rand::thread_rng().gen_range(FIRST_DYNAMIC_PORT..LAST_DYNAMIC_PORT) } -pub(crate) type EventId = usize; +pub(crate) type EventId = u32; #[derive(PartialEq, Eq, Hash, Clone, PartialOrd, Ord, Debug)] pub struct NodeLabel(Arc); @@ -133,24 +133,40 @@ struct GatewayConfig { } pub struct EventChain { - network: SimNetwork, - total_events: usize, - count: usize, + labels: Vec<(NodeLabel, PeerId)>, + user_ev_controller: Sender<(EventId, PeerId)>, + total_events: u32, + count: u32, rng: rand::rngs::SmallRng, + clean_up_tmp_dirs: bool, +} + +impl EventChain { + pub fn new( + labels: Vec<(NodeLabel, PeerId)>, + user_ev_controller: Sender<(EventId, PeerId)>, + total_events: u32, + clean_up_tmp_dirs: bool, + ) -> Self { + const SEED: u64 = 0xdeadbeef; + EventChain { + labels, + user_ev_controller, + total_events, + count: 0, + rng: rand::rngs::SmallRng::seed_from_u64(SEED), + clean_up_tmp_dirs, + } + } } impl Iterator for EventChain { - type Item = usize; + type Item = EventId; fn next(&mut self) -> Option { (self.count < self.total_events).then(|| { - let (_, id) = self - .network - .labels - .choose(&mut self.rng) - .expect("not empty"); - self.network - .user_ev_controller + let (_, id) = self.labels.choose(&mut self.rng).expect("not empty"); + self.user_ev_controller .send((self.count, *id)) .expect("peer controller should be alive"); self.count += 1; @@ -159,6 +175,14 @@ impl Iterator for EventChain { } } +impl Drop for EventChain { + fn drop(&mut self) { + if self.clean_up_tmp_dirs { + clean_up_tmp_dirs(&self.labels) + } + } +} + #[cfg(feature = "trace-ot")] type DefaultRegistry = CombinedRegister<2>; @@ -168,7 +192,7 @@ type DefaultRegistry = TestEventListener; pub(super) struct Builder { pub(super) peer_key: PeerId, config: NodeConfig, - ch_builder: String, + contract_handler_name: String, add_noise: bool, event_register: ER, contracts: Vec<(ContractContainer, WrappedState)>, @@ -180,14 +204,14 @@ impl Builder { pub fn build( builder: NodeConfig, event_register: ER, - ch_builder: String, + contract_handler_name: String, add_noise: bool, ) -> Builder { let peer_key = builder.peer_id; Builder { peer_key, config: builder, - ch_builder, + contract_handler_name, add_noise, event_register, contracts: Vec::new(), @@ -199,10 +223,10 @@ impl Builder { /// A simulated in-memory network topology. pub struct SimNetwork { name: String, - debug: bool, + clean_up_tmp_dirs: bool, labels: Vec<(NodeLabel, PeerId)>, pub(crate) event_listener: TestEventListener, - user_ev_controller: Sender<(EventId, PeerId)>, + user_ev_controller: Option>, receiver_ch: Receiver<(EventId, PeerId)>, number_of_gateways: usize, gateways: Vec<(Builder, GatewayConfig)>, @@ -216,24 +240,6 @@ pub struct SimNetwork { add_noise: bool, } -#[cfg(any(debug_assertions, test))] -impl std::fmt::Debug for SimNetwork { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SimNetwork") - .field("name", &self.name) - .field("labels", &self.labels) - .field("number_of_gateways", &self.number_of_gateways) - .field("number_of_nodes", &self.number_of_nodes) - .field("ring_max_htl", &self.ring_max_htl) - .field("rnd_if_htl_above", &self.rnd_if_htl_above) - .field("max_connections", &self.max_connections) - .field("min_connections", &self.min_connections) - .field("init_backoff", &self.init_backoff) - .field("add_noise", &self.add_noise) - .finish() - } -} - impl SimNetwork { pub async fn new( name: &str, @@ -248,10 +254,10 @@ impl SimNetwork { let (user_ev_controller, receiver_ch) = channel((0, PeerId::random())); let mut net = Self { name: name.into(), - debug: false, + clean_up_tmp_dirs: true, event_listener: TestEventListener::new(), labels: Vec::with_capacity(nodes + gateways), - user_ev_controller, + user_ev_controller: Some(user_ev_controller), receiver_ch, number_of_gateways: gateways, gateways: Vec::with_capacity(gateways), @@ -264,8 +270,8 @@ impl SimNetwork { init_backoff: Duration::from_millis(1), add_noise: false, }; - net.build_gateways(gateways).await; - net.build_nodes(nodes).await; + net.config_gateways(gateways).await; + net.config_nodes(nodes).await; net } } @@ -283,10 +289,10 @@ impl SimNetwork { #[allow(unused)] pub fn debug(&mut self) { - self.debug = true; + self.clean_up_tmp_dirs = false; } - async fn build_gateways(&mut self, num: usize) { + async fn config_gateways(&mut self, num: usize) { info!("Building {} gateways", num); let mut configs = Vec::with_capacity(num); for node_no in 0..num { @@ -358,7 +364,7 @@ impl SimNetwork { } } - async fn build_nodes(&mut self, num: usize) { + async fn config_nodes(&mut self, num: usize) { info!("Building {} regular nodes", num); let gateways: Vec<_> = self .gateways @@ -618,6 +624,8 @@ impl SimNetwork { .map_err(|_| anyhow::anyhow!("peer not found"))?; let (_, peer) = &self.labels[pos]; self.user_ev_controller + .as_ref() + .expect("should be set") .send((event_id, *peer)) .expect("node listeners disconnected"); if let Some(sleep_time) = await_for { @@ -632,19 +640,18 @@ impl SimNetwork { /// nodes built through the [`build_peers`](`Self::build_peers`) method. pub fn event_chain( mut self, - total_events: usize, + total_events: u32, controller: Option>, ) -> EventChain { - const SEED: u64 = 0xdeadbeef; - if let Some(controller) = controller { - self.user_ev_controller = controller; - } - EventChain { - network: self, - total_events, - count: 0, - rng: rand::rngs::SmallRng::seed_from_u64(SEED), - } + let user_ev_controller = controller.unwrap_or_else(|| { + self.user_ev_controller + .take() + .expect("controller should be ser") + }); + let labels = std::mem::take(&mut self.labels); + let debug_val = self.clean_up_tmp_dirs; + self.clean_up_tmp_dirs = false; // set to false to avoid cleaning up the tmp dirs + EventChain::new(labels, user_ev_controller, total_events, debug_val) } /// Checks that all peers in the network have acquired at least one connection to any @@ -768,18 +775,43 @@ impl SimNetwork { } } +#[cfg(any(debug_assertions, test))] +impl std::fmt::Debug for SimNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SimNetwork") + .field("name", &self.name) + .field("labels", &self.labels) + .field("number_of_gateways", &self.number_of_gateways) + .field("number_of_nodes", &self.number_of_nodes) + .field("ring_max_htl", &self.ring_max_htl) + .field("rnd_if_htl_above", &self.rnd_if_htl_above) + .field("max_connections", &self.max_connections) + .field("min_connections", &self.min_connections) + .field("init_backoff", &self.init_backoff) + .field("add_noise", &self.add_noise) + .finish() + } +} + impl Drop for SimNetwork { fn drop(&mut self) { - if !self.debug { - for (label, _) in &self.labels { - let p = std::env::temp_dir() - .join(format!("freenet-executor-{sim}-{label}", sim = self.name)); - let _ = std::fs::remove_dir_all(p); - } + if self.clean_up_tmp_dirs { + clean_up_tmp_dirs(&self.labels); } } } +fn clean_up_tmp_dirs(labels: &[(NodeLabel, PeerId)]) { + for (label, _) in labels { + let p = std::env::temp_dir().join(format!( + "freenet-executor-{sim}-{label}", + sim = "sim", + label = label + )); + let _ = std::fs::remove_dir_all(p); + } +} + fn group_locations_in_buckets( locs: impl IntoIterator, scale: i32, @@ -1027,7 +1059,6 @@ where }; let op_storage = op_storage.clone(); - let conn_manager = conn_manager.clone(); let event_listener = event_register.trait_clone(); let span = { @@ -1053,7 +1084,7 @@ where let msg = super::process_message( msg, op_storage, - conn_manager, + conn_manager.clone(), event_listener, None, None, diff --git a/crates/core/src/node/testing_impl/in_memory.rs b/crates/core/src/node/testing_impl/in_memory.rs index 170120571..65d29f98b 100644 --- a/crates/core/src/node/testing_impl/in_memory.rs +++ b/crates/core/src/node/testing_impl/in_memory.rs @@ -42,7 +42,7 @@ impl Builder { )?); let (_executor_listener, executor_sender) = executor_channel(op_storage.clone()); let contract_handler = - MemoryContractHandler::build(ch_channel, executor_sender, self.ch_builder) + MemoryContractHandler::build(ch_channel, executor_sender, self.contract_handler_name) .await .map_err(|e| anyhow::anyhow!(e))?; diff --git a/crates/fdev/Cargo.toml b/crates/fdev/Cargo.toml index 315824af9..004e29289 100644 --- a/crates/fdev/Cargo.toml +++ b/crates/fdev/Cargo.toml @@ -25,7 +25,7 @@ serde_with = { workspace = true } semver = { workspace = true } tar = "0.4" thiserror = "1" -tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "signal", "parking_lot"] } +tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "signal", "parking_lot", "process"] } toml = { version = "0.8", features = ["default", "preserve_order"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } diff --git a/crates/fdev/src/commands.rs b/crates/fdev/src/commands.rs index eb86fefc1..ac150d9a4 100644 --- a/crates/fdev/src/commands.rs +++ b/crates/fdev/src/commands.rs @@ -159,11 +159,13 @@ async fn execute_command( let secrets_data_path = other .secret_data_dir .unwrap_or_else(|| Config::conf().secrets_dir()); - + let database_path = other + .database_dir + .unwrap_or_else(|| Config::conf().db_dir()); let contract_store = ContractStore::new(contracts_data_path, DEFAULT_MAX_CONTRACT_SIZE)?; let delegate_store = DelegateStore::new(delegates_data_path, DEFAULT_MAX_DELEGATE_SIZE)?; let secret_store = SecretsStore::new(secrets_data_path)?; - let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE)?; + let state_store = StateStore::new(Storage::new(Some(&database_path)).await?, MAX_MEM_CACHE)?; let rt = freenet::dev_tool::Runtime::build(contract_store, delegate_store, secret_store, false)?; let mut executor = Executor::new(state_store, || Ok(()), OperationMode::Local, rt) diff --git a/crates/fdev/src/config.rs b/crates/fdev/src/config.rs index 3dfff3e0e..98fa68417 100644 --- a/crates/fdev/src/config.rs +++ b/crates/fdev/src/config.rs @@ -27,6 +27,9 @@ pub struct BaseConfig { /// Overrides the default data directory where Freenet secret files are stored. #[arg(long)] pub(crate) secret_data_dir: Option, + /// Overrides the default data directory where Freenet database files are stored. + #[arg(long)] + pub(crate) database_dir: Option, /// Node operation mode. #[arg(value_enum, default_value_t=OperationMode::Local)] pub mode: OperationMode, diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index bdfe3d4f8..ca6d7bda3 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -37,8 +37,8 @@ pub struct TestConfig { max_contract_number: Option, /// Number of events that will be executed in this simulation. /// Events are simulated get, puts and other operations. - #[arg(long, default_value_t = usize::MAX)] - events: usize, + #[arg(long, default_value_t = u32::MAX)] + events: u32, /// Time in milliseconds to wait for the network to be sufficiently connected to start requesting events. /// (20% of the expected connections to be processed per gateway) #[arg(long)] @@ -91,7 +91,7 @@ pub enum TestMode { Network, } -pub(crate) async fn test_framework(base_config: TestConfig) -> Result<(), Error> { +pub(crate) async fn test_framework(base_config: TestConfig) -> anyhow::Result<(), Error> { match &base_config.command { TestMode::SingleProcess => { single_process::run(&base_config).await?; @@ -106,7 +106,7 @@ pub(crate) async fn test_framework(base_config: TestConfig) -> Result<(), Error> Ok(()) } -async fn config_sim_network(base_config: &TestConfig) -> Result { +async fn config_sim_network(base_config: &TestConfig) -> anyhow::Result { if base_config.gateways == 0 { anyhow::bail!("Gateways should be higher than 0"); } @@ -133,12 +133,21 @@ async fn config_sim_network(base_config: &TestConfig) -> Result, + pub(super) async fn run( + config: &super::TestConfig, + cmd_config: &MultiProcessConfig, + ) -> anyhow::Result<(), Error> { + match cmd_config.mode { + Process::Supervisor => supervisor(config).await, + Process::Child => { + child(config, cmd_config.id.expect("id should be set for child")).await + } + } + } + + async fn supervisor(config: &super::TestConfig) -> anyhow::Result<(), Error> { + let mut simulated_network = super::config_sim_network(config).await?; + simulated_network.debug(); // set to avoid deleting temp dirs created + let peers = simulated_network.build_peers(); + + let (user_ev_controller, event_rx) = tokio::sync::watch::channel((0, PeerId::random())); + + let seed = config.seed(); + let mut supervisor = Supervisor { + processes: HashMap::new(), + event_rx, + waiting_response: FuturesUnordered::new(), + queued: HashMap::new(), + }; + let cmd_args = config.subprocess_command(seed); + for (label, node) in &peers { + let mut subprocess = SubProcess::start(&cmd_args, label, node.peer_id)?; + subprocess.config(node).await?; + supervisor.processes.insert(node.peer_id, subprocess); + } + + let peers = peers + .into_iter() + .map(|(label, config)| (label, config.peer_id)) + .collect(); + let supervisor_task = tokio::task::spawn(supervisor.start_simulation()); + + let events = EventChain::new(peers, user_ev_controller, config.events, true); + let next_event_wait_time = config.event_wait_time.map(Duration::from_millis); + for _event_id in events { + if let Some(t) = next_event_wait_time { + std::thread::sleep(t); + } + } + supervisor_task.await?; + + Ok(()) + } + + type ChildResponses = Vec<(PeerId, Vec)>; + + /// Event driver for the supervisor process. + struct Supervisor { + processes: HashMap, + event_rx: tokio::sync::watch::Receiver<(u32, PeerId)>, + waiting_response: + FuturesUnordered>>, + queued: HashMap>, + } + + impl Supervisor { + async fn start_simulation(mut self) { + loop { + tokio::select! { + biased; + res = self.waiting_response.next() => { + let (subprocess, responses) = res.unwrap().unwrap(); + if let Err(err) = self.process_responses(subprocess, responses).await { + eprintln!("Error processing responses: {err}"); + break; + } + + } + msg = self.event_rx.changed() => { + if msg.is_err() { + break; + } + let (event, peer) = *self.event_rx.borrow(); + let Some(mut subprocess) = self.processes.remove(&peer) else { + self.queued.entry(peer).or_default().push_back(IPCMessage::FiredEvent(event)); + continue; + }; + let mut pending = self.queued.remove(&peer).unwrap_or_default(); + pending.push_back(IPCMessage::FiredEvent(event)); + let task = async move { + for msg in pending { + msg.send(subprocess.child.stdin.as_mut().expect("not taken")) + .await?; + } + let mut returned = vec![]; + while let Ok((target, msg)) = InterProcessConnManager::pull_msg( + subprocess.child.stdout.as_mut().expect("stdout not taken"), + ) + .await + { + returned.push((target, msg)); + } + Ok((subprocess, returned)) + }.boxed(); + self.waiting_response.push(task); + } + } + } + for (_, subprocess) in self.processes.drain() { + subprocess.close().await; + } + println!("Simulation finished"); + } + + async fn process_responses( + &mut self, + mut subprocess: SubProcess, + responses: ChildResponses, + ) -> anyhow::Result<()> { + for (target, data) in responses { + if let Some(target) = self.processes.remove(&target) { + let task = async move { target.send_msg(IPCMessage::Data(data)).await }.boxed(); + self.waiting_response.push(task); + } else { + self.queued + .entry(target) + .or_default() + .push_back(IPCMessage::Data(data)); + } + } + if let Some(mut queued) = self.queued.remove(&subprocess.id) { + for msg in queued.drain(..) { + msg.send(subprocess.child.stdin.as_mut().expect("not taken")) + .await?; + while let Ok((target, data)) = InterProcessConnManager::pull_msg( + subprocess.child.stdout.as_mut().expect("stdout not taken"), + ) + .await + { + self.queued + .entry(target) + .or_default() + .push_back(IPCMessage::Data(data)); + } + } + } + self.processes.insert(subprocess.id, subprocess); + Ok(()) + } } struct SubProcess { - label: NodeLabel, - child: std::process::Child, + child: tokio::process::Child, + id: PeerId, } impl SubProcess { - fn start(cmd_args: &[String], label: NodeLabel) -> Result { + fn start( + cmd_args: &[String], + label: &NodeLabel, + id: PeerId, + ) -> anyhow::Result { let cmd = Command::new("fdev") .args(cmd_args) .arg("--id") @@ -228,71 +384,53 @@ mod multiple_process { .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn()?; - Ok(Self { label, child: cmd }) + Ok(Self { child: cmd, id }) } - fn config(&mut self, config: NodeConfig) -> Result<(), Error> { + async fn config(&mut self, config: &NodeConfig) -> anyhow::Result<(), Error> { let input = self.get_input().ok_or_else(|| anyhow!("closed"))?; let serialize = bincode::serialize(&config)?; - input.write_all(&(serialize.len() as u64).to_le_bytes())?; - input.write_all(&serialize)?; - Ok(()) - } - - fn send_event(&mut self, event: usize) -> Result<(), std::io::Error> { - let input = self.get_input().ok_or(std::io::ErrorKind::BrokenPipe)?; - input.write_all(&event.to_le_bytes())?; + input + .write_all(&(serialize.len() as u64).to_le_bytes()) + .await?; + input.write_all(&serialize).await?; Ok(()) } - #[inline] - fn get_input(&mut self) -> Option<&mut std::process::ChildStdin> { - self.child.stdin.as_mut() - } - - fn close(mut self) { - let _ = self.child.kill(); - } - } - - pub(super) async fn run( - config: &super::TestConfig, - cmd_config: &MultiProcessConfig, - ) -> Result<(), Error> { - match cmd_config.mode { - Process::Supervisor => supervisor(config).await, - Process::Child => { - child(config, cmd_config.id.expect("id should be set for child")).await + async fn send_msg(mut self, msg: IPCMessage) -> anyhow::Result<(Self, ChildResponses)> { + msg.send(self.child.stdin.as_mut().expect("not taken")) + .await?; + let mut returned = vec![]; + while let Ok((target, msg)) = InterProcessConnManager::pull_msg( + self.child.stdout.as_mut().expect("stdout not taken"), + ) + .await + { + returned.push((target, msg)); } + Ok((self, returned)) } - } - async fn supervisor(config: &super::TestConfig) -> Result<(), Error> { - let mut simulated_network = super::config_sim_network(config).await?; - let peers = simulated_network.build_peers(); + async fn close(mut self) { + let _ = self.child.kill().await; + } - let seed = config.seed(); - let mut manager = SubProcessManager::default(); - let cmd_args = config.subprocess_command(seed); - for (label, node) in peers { - let mut subprocess = SubProcess::start(&cmd_args, label)?; - subprocess.config(node)?; - manager.processes.push(subprocess); + #[inline] + fn get_input(&mut self) -> Option<&mut tokio::process::ChildStdin> { + self.child.stdin.as_mut() } - Ok(()) } - async fn child(config: &super::TestConfig, id: usize) -> Result<(), Error> { - let mut input = std::io::stdin(); - let mut output = std::io::stdout(); - let mut buf = [0u8; 8]; - input.read_exact(&mut buf)?; - let config_len = u64::from_le_bytes(buf); - let mut config_buf = vec![0u8; config_len as usize]; - input.read_exact(&mut config_buf)?; - let node_config: NodeConfig = bincode::deserialize(&config_buf)?; - + async fn child(config: &super::TestConfig, id: usize) -> anyhow::Result<()> { let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerId::random())); + let mut input = tokio::io::stdin(); + let node_config = Child::get_config(&mut input).await?; + let this_child = Child { + input, + user_ev_controller, + peer_id: node_config.peer_id, + }; + let mut event_generator = MemoryEventsGen::::new_with_seed( receiver_ch.clone(), node_config.peer_id, @@ -302,12 +440,88 @@ mod multiple_process { id, config.gateways + config.nodes, config.max_contract_number.unwrap_or(config.nodes * 10), - config.events, + config.events as usize, ); let config = SimPeer::from(node_config); + tokio::task::spawn(this_child.event_loop()); config.start_child(event_generator).await?; Ok(()) } + + /// Controller for a child process. + struct Child { + input: tokio::io::Stdin, + user_ev_controller: tokio::sync::watch::Sender<(u32, PeerId)>, + peer_id: PeerId, + } + + impl Child { + async fn get_config(input: &mut tokio::io::Stdin) -> anyhow::Result { + let mut buf = [0u8; 4]; + input.read_exact(&mut buf).await?; + let config_len = u32::from_le_bytes(buf); + let mut config_buf = vec![0u8; config_len as usize]; + input.read_exact(&mut config_buf).await?; + let node_config: NodeConfig = bincode::deserialize(&config_buf)?; + Ok(node_config) + } + + async fn event_loop(mut self) -> anyhow::Result<()> { + loop { + match IPCMessage::recv(&mut self.input).await? { + IPCMessage::FiredEvent(id) => { + self.user_ev_controller.send((id, self.peer_id))?; + } + IPCMessage::Data(data) => { + InterProcessConnManager::push_msg(data); + } + } + } + } + } + + enum IPCMessage { + FiredEvent(u32), + Data(Vec), + } + + impl IPCMessage { + async fn send(self, out: &mut tokio::process::ChildStdin) -> anyhow::Result<()> { + match self { + Self::FiredEvent(id) => { + out.write_u8(0).await?; + out.write_all(&id.to_le_bytes()).await?; + } + Self::Data(data) => { + out.write_u8(1).await?; + out.write_all(&(data.len() as u32).to_le_bytes()).await?; + out.write_all(&data).await?; + } + } + Ok(()) + } + + async fn recv(input: &mut tokio::io::Stdin) -> anyhow::Result { + let marker = input.read_u8().await?; + match marker { + 0 => { + let mut buf = [0u8; 4]; + input.read_exact(&mut buf).await?; + let event_id = u32::from_le_bytes(buf); + Ok(Self::FiredEvent(event_id)) + } + 1 => { + let mut buf = [0u8; 4]; + input.read_exact(&mut buf).await?; + let data_len = u32::from_le_bytes(buf); + let mut data = vec![0u8; data_len as usize]; + input.read_exact(&mut data).await?; + Ok(Self::Data(data)) + } + _ => unimplemented!(), + } + } + } } mod single_process { @@ -316,14 +530,14 @@ mod single_process { use futures::StreamExt; use tokio::signal; - pub(super) async fn run(config: &super::TestConfig) -> Result<(), super::Error> { + pub(super) async fn run(config: &super::TestConfig) -> anyhow::Result<(), super::Error> { let mut simulated_network = super::config_sim_network(config).await?; let join_handles = simulated_network .start_with_rand_gen::( config.seed(), config.max_contract_number.unwrap_or(config.nodes * 10), - config.events, + config.events as usize, ) .await; @@ -370,7 +584,7 @@ mod single_process { match res? { Ok(()) => { println!("Test events generated successfully"); - *events_generated = tokio::task::spawn(futures::future::pending::>()); + *events_generated = tokio::task::spawn(futures::future::pending::>()); continue; } Err(err) => { diff --git a/crates/fdev/src/wasm_runtime/state.rs b/crates/fdev/src/wasm_runtime/state.rs index d1391949f..22d45bf17 100644 --- a/crates/fdev/src/wasm_runtime/state.rs +++ b/crates/fdev/src/wasm_runtime/state.rs @@ -30,7 +30,10 @@ impl AppState { Self::DEFAULT_MAX_DELEGATE_SIZE, )?; let secrets_store = SecretsStore::new(Config::conf().secrets_dir())?; - let state_store = StateStore::new(Storage::new().await?, Self::MAX_MEM_CACHE)?; + let state_store = StateStore::new( + Storage::new(Some(&Config::conf().db_dir())).await?, + Self::MAX_MEM_CACHE, + )?; let rt = freenet::dev_tool::Runtime::build( contract_store, delegate_store, From 1e88a7087d0487fe814658daf292e8667ac395c6 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Tue, 21 Nov 2023 15:07:52 +0100 Subject: [PATCH 11/12] Fixes to msg passign between processes --- crates/core/src/client_events.rs | 2 +- crates/core/src/config.rs | 35 ++- crates/core/src/contract/executor.rs | 17 +- crates/core/src/contract/handler.rs | 4 +- crates/core/src/contract/in_memory.rs | 8 +- crates/core/src/contract/storages/sqlite.rs | 9 +- .../src/node/network_bridge/inter_process.rs | 9 +- crates/core/src/node/network_event_log.rs | 1 + crates/core/src/node/testing_impl.rs | 124 ++++++++-- .../src/node/testing_impl/inter_process.rs | 4 - crates/fdev/src/testing.rs | 222 ++++++++++++------ 11 files changed, 315 insertions(+), 120 deletions(-) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index 92ec15d09..d1fc30eb9 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -260,7 +260,7 @@ pub(crate) mod test { async { loop { if self.signal.changed().await.is_ok() { - let (ev_id, pk) = *self.signal.borrow(); + let (ev_id, pk) = *self.signal.borrow_and_update(); if self.rng.is_some() && pk == self.id { let res = OpenRequest { client_id: ClientId::FIRST, diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index fa3f22345..763e49697 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -14,6 +14,7 @@ use std::{ use directories::ProjectDirs; use libp2p::{identity, PeerId}; use once_cell::sync::Lazy; +use parking_lot::Mutex; use tokio::runtime::Runtime; use crate::local_node::OperationMode; @@ -102,7 +103,7 @@ pub struct ConfigPaths { delegates_dir: PathBuf, secrets_dir: PathBuf, db_dir: PathBuf, - event_log: PathBuf, + event_log: Mutex, } impl ConfigPaths { @@ -157,7 +158,7 @@ impl ConfigPaths { delegates_dir, secrets_dir, db_dir, - event_log, + event_log: Mutex::new(event_log), }) } } @@ -170,6 +171,7 @@ impl Config { .store(local_mode, std::sync::atomic::Ordering::SeqCst); } + #[cfg(feature = "trace-ot")] fn node_mode() -> OperationMode { if Self::conf() .local_mode @@ -215,14 +217,24 @@ impl Config { pub fn event_log(&self) -> PathBuf { if self.local_mode.load(std::sync::atomic::Ordering::SeqCst) { - let mut local_file = self.config_paths.event_log.clone(); + let mut local_file = self.config_paths.event_log.lock().clone(); local_file.set_file_name("_EVENT_LOG_LOCAL"); local_file } else { - self.config_paths.event_log.to_owned() + self.config_paths.event_log.lock().to_owned() } } + pub fn set_event_log(path: PathBuf) { + tracing::debug!("setting event log file to: {:?}", &path); + fs::OpenOptions::new() + .write(true) + .create(true) + .open(&path) + .expect("couln't create event log file"); + *Self::conf().config_paths.event_log.lock() = path; + } + pub fn conf() -> &'static Config { CONFIG.get_or_init(|| match Config::load_conf() { Ok(config) => config, @@ -377,16 +389,14 @@ mod tracer { use crate::DynError; - use super::*; - pub fn init_tracer() -> Result<(), DynError> { - let filter = if cfg!(any(test, debug_assertions)) { + let default_filter = if cfg!(any(test, debug_assertions)) { tracing_subscriber::filter::LevelFilter::DEBUG } else { tracing_subscriber::filter::LevelFilter::INFO }; let filter_layer = tracing_subscriber::EnvFilter::builder() - .with_default_directive(filter.into()) + .with_default_directive(default_filter.into()) .from_env_lossy() .add_directive("stretto=off".parse().expect("infallible")) .add_directive("sqlx=error".parse().expect("infallible")); @@ -395,6 +405,7 @@ mod tracer { use tracing_subscriber::layer::SubscriberExt; let disabled_logs = std::env::var("FREENET_DISABLE_LOGS").is_ok(); + let to_stderr = std::env::var("FREENET_LOG_TO_STDERR").is_ok(); let layers = { let fmt_layer = tracing_subscriber::fmt::layer().with_level(true); let fmt_layer = if cfg!(any(test, debug_assertions)) { @@ -402,9 +413,17 @@ mod tracer { } else { fmt_layer }; + let fmt_layer = if to_stderr { + fmt_layer.with_writer(std::io::stderr).boxed() + } else { + fmt_layer.boxed() + }; + #[cfg(feature = "trace-ot")] { + use super::*; let disabled_ot_traces = std::env::var("FREENET_DISABLE_TRACES").is_ok(); + // FIXME let identifier = if matches!(Config::node_mode(), OperationMode::Local) { "freenet-core".to_string() } else { diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index e4d707a50..c2e6be3f1 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -391,7 +391,7 @@ impl Executor { const MAX_MEM_CACHE: u32 = 10_000_000; let static_conf = crate::config::Config::conf(); - let db_path = crate::config::Config::conf().db_dir().join("freenet.db"); + let db_path = crate::config::Config::conf().db_dir(); let state_store = StateStore::new(Storage::new(Some(&db_path)).await?, MAX_MEM_CACHE).unwrap(); @@ -1199,15 +1199,18 @@ impl Executor { } impl Executor { - pub async fn new_mock(data_dir: &str) -> Result { - let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{data_dir}")); + pub async fn new_mock(identifier: &str) -> Result { + let data_dir = std::env::temp_dir().join(format!("freenet-executor-{identifier}")); - let contracts_data_dir = tmp_path.join("contracts"); + let contracts_data_dir = data_dir.join("contracts"); std::fs::create_dir_all(&contracts_data_dir).expect("directory created"); let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; - let db_path = tmp_path.join("freenet.db"); + let db_path = data_dir.join("db"); std::fs::create_dir_all(&db_path).expect("directory created"); + let log_file = data_dir.join("_EVENT_LOG_LOCAL"); + eprintln!("{log_file:?}"); + crate::config::Config::set_event_log(log_file); let state_store = StateStore::new(Storage::new(Some(&db_path)).await?, u16::MAX as u32).unwrap(); @@ -1340,8 +1343,8 @@ mod test { async fn local_node_handle() -> Result<(), Box> { const MAX_SIZE: i64 = 10 * 1024 * 1024; const MAX_MEM_CACHE: u32 = 10_000_000; - let tmp_path = std::env::temp_dir().join("freenet-test"); - let contract_store = ContractStore::new(tmp_path.join("executor-test"), MAX_SIZE)?; + let tmp_dir = tempfile::tempdir()?; + let contract_store = ContractStore::new(tmp_dir.path().join("executor-test"), MAX_SIZE)?; let state_store = StateStore::new(Storage::new(None).await?, MAX_MEM_CACHE).unwrap(); let mut counter = 0; Executor::new( diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 3d237156b..8716e213e 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -138,13 +138,13 @@ impl ContractHandler for NetworkContractHandler { fn build( channel: ContractHandlerChannel, _executor_request_sender: ExecutorToEventLoopChannel, - data_dir: Self::Builder, + identifier: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { async move { - let executor = Executor::new_mock(&data_dir).await?; + let executor = Executor::new_mock(&identifier).await?; Ok(Self { executor, channel }) } .boxed() diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs index 83b080186..f832b5ccd 100644 --- a/crates/core/src/contract/in_memory.rs +++ b/crates/core/src/contract/in_memory.rs @@ -22,11 +22,11 @@ pub(crate) struct MemoryContractHandler { impl MemoryContractHandler { pub async fn new( channel: ContractHandlerChannel, - data_dir: &str, + identifier: &str, ) -> Self { MemoryContractHandler { channel, - runtime: Executor::new_mock(data_dir).await.unwrap(), + runtime: Executor::new_mock(identifier).await.unwrap(), } } } @@ -38,12 +38,12 @@ impl ContractHandler for MemoryContractHandler { fn build( channel: ContractHandlerChannel, _executor_request_sender: ExecutorToEventLoopChannel, - config: Self::Builder, + identifier: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { - async move { Ok(MemoryContractHandler::new(channel, &config).await) }.boxed() + async move { Ok(MemoryContractHandler::new(channel, &identifier).await) }.boxed() } fn channel(&mut self) -> &mut ContractHandlerChannel { diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index 39172a243..2e05f2820 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -28,12 +28,13 @@ async fn create_contracts_table(pool: &SqlitePool) -> Result<(), SqlDbError> { pub struct Pool(SqlitePool); impl Pool { - pub async fn new(db_path: Option<&Path>) -> Result { - let opts = if let Some(db_path) = db_path { - tracing::info!("loading contract store from {db_path:?}"); + pub async fn new(db_dir: Option<&Path>) -> Result { + let opts = if let Some(db_dir) = db_dir { + let file = db_dir.join("freenet.db"); + tracing::info!("loading contract store from {file:?}"); SqliteConnectOptions::new() .create_if_missing(true) - .filename(db_path) + .filename(file) } else { SqliteConnectOptions::from_str("sqlite::memory:").unwrap() }; diff --git a/crates/core/src/node/network_bridge/inter_process.rs b/crates/core/src/node/network_bridge/inter_process.rs index 628415055..4b43420e6 100644 --- a/crates/core/src/node/network_bridge/inter_process.rs +++ b/crates/core/src/node/network_bridge/inter_process.rs @@ -48,7 +48,10 @@ impl InterProcessConnManager { let msg_len = u32::from_le_bytes(msg_len) as usize; let buf = &mut vec![0u8; msg_len]; stdout.read_exact(buf).await?; - bincode::deserialize(buf).map_err(|_| std::io::ErrorKind::Other.into()) + let (target, data) = bincode::deserialize(buf) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other))?; + tracing::debug!(%target, "network message received"); + Ok((target, data)) } } @@ -59,7 +62,7 @@ impl NetworkBridgeExt for InterProcessConnManager { .changed() .await .map_err(|_| ConnectionError::Timeout)?; - let data = &*self.recv.borrow(); + let data = &*self.recv.borrow_and_update(); let deser = bincode::deserialize(data)?; Ok(deser) } @@ -70,7 +73,7 @@ impl NetworkBridgeExt for InterProcessConnManager { #[async_trait::async_trait] impl NetworkBridge for InterProcessConnManager { async fn send(&self, target: &PeerId, msg: NetMessage) -> super::ConnResult<()> { - let data = bincode::serialize(&(*target, msg)).expect("proper encoding"); + let data = bincode::serialize(&(*target, msg))?; let output = &mut *self.output.lock().await; output.write_all(&(data.len() as u32).to_le_bytes()).await?; output.write_all(&data).await?; diff --git a/crates/core/src/node/network_event_log.rs b/crates/core/src/node/network_event_log.rs index 8a41d5a8d..9a40f6cc9 100644 --- a/crates/core/src/node/network_event_log.rs +++ b/crates/core/src/node/network_event_log.rs @@ -490,6 +490,7 @@ impl EventRegister { Ok(()) } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; // wait for the node to start let event_log_path = crate::config::Config::conf().event_log(); tracing::info!(?event_log_path); let mut event_log = match OpenOptions::new() diff --git a/crates/core/src/node/testing_impl.rs b/crates/core/src/node/testing_impl.rs index 56df0101d..44e63fbc9 100644 --- a/crates/core/src/node/testing_impl.rs +++ b/crates/core/src/node/testing_impl.rs @@ -2,17 +2,18 @@ use std::{ collections::{HashMap, HashSet}, fmt::Write, net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener}, + pin::Pin, sync::Arc, time::{Duration, Instant}, }; use either::Either; use freenet_stdlib::prelude::*; -use futures::future::BoxFuture; +use futures::{future::BoxFuture, Future}; use itertools::Itertools; use libp2p::{identity, PeerId as Libp2pPeerId}; use rand::{seq::SliceRandom, Rng}; -use tokio::sync::watch::{channel, Receiver, Sender}; +use tokio::sync::{mpsc, watch}; use tracing::{info, Instrument}; #[cfg(feature = "trace-ot")] @@ -132,19 +133,21 @@ struct GatewayConfig { location: Location, } -pub struct EventChain { +pub struct EventChain> { labels: Vec<(NodeLabel, PeerId)>, - user_ev_controller: Sender<(EventId, PeerId)>, + // user_ev_controller: Sender<(EventId, PeerId)>, + user_ev_controller: S, total_events: u32, count: u32, rng: rand::rngs::SmallRng, clean_up_tmp_dirs: bool, + choice: Option, } -impl EventChain { +impl EventChain { pub fn new( labels: Vec<(NodeLabel, PeerId)>, - user_ev_controller: Sender<(EventId, PeerId)>, + user_ev_controller: S, total_events: u32, clean_up_tmp_dirs: bool, ) -> Self { @@ -156,26 +159,104 @@ impl EventChain { count: 0, rng: rand::rngs::SmallRng::seed_from_u64(SEED), clean_up_tmp_dirs, + choice: None, } } + + fn increment_count(self: Pin<&mut Self>) { + unsafe { + // This is safe because we're not moving the EventChain, just modifying a field + let this = self.get_unchecked_mut(); + this.count += 1; + } + } + + fn choose_peer(self: Pin<&mut Self>) -> PeerId { + let this = unsafe { + // This is safe because we're not moving the EventChain, just copying one inner valur + self.get_unchecked_mut() + }; + if let Some(id) = this.choice.take() { + return id; + } + let rng = &mut this.rng; + let labels = &mut this.labels; + let (_, id) = labels.choose(rng).expect("not empty"); + *id + } + + fn set_choice(self: Pin<&mut Self>, id: PeerId) { + let this = unsafe { + // This is safe because we're not moving the EventChain, just copying one inner valur + self.get_unchecked_mut() + }; + this.choice = Some(id); + } } -impl Iterator for EventChain { +trait EventSender { + fn send( + &self, + cx: &mut std::task::Context<'_>, + value: (EventId, PeerId), + ) -> std::task::Poll>; +} + +impl EventSender for mpsc::Sender<(EventId, PeerId)> { + fn send( + &self, + cx: &mut std::task::Context<'_>, + value: (EventId, PeerId), + ) -> std::task::Poll> { + let f = self.send(value); + futures::pin_mut!(f); + f.poll(cx).map(|r| r.map_err(|_| ())) + } +} + +impl EventSender for watch::Sender<(EventId, PeerId)> { + fn send( + &self, + _cx: &mut std::task::Context<'_>, + value: (EventId, PeerId), + ) -> std::task::Poll> { + match self.send(value) { + Ok(_) => std::task::Poll::Ready(Ok(())), + Err(_) => std::task::Poll::Ready(Err(())), + } + } +} + +impl futures::stream::Stream for EventChain { type Item = EventId; - fn next(&mut self) -> Option { - (self.count < self.total_events).then(|| { - let (_, id) = self.labels.choose(&mut self.rng).expect("not empty"); - self.user_ev_controller - .send((self.count, *id)) - .expect("peer controller should be alive"); - self.count += 1; - self.count - }) + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + if self.count < self.total_events { + let id = self.as_mut().choose_peer(); + match self + .user_ev_controller + .send(cx, (self.count, id)) + .map_err(|_| { + tracing::error!("peer controller should be alive, finishing event chain") + }) { + std::task::Poll::Ready(_) => {} + std::task::Poll::Pending => { + self.as_mut().set_choice(id); + return std::task::Poll::Pending; + } + } + self.as_mut().increment_count(); + std::task::Poll::Ready(Some(self.count)) + } else { + std::task::Poll::Ready(None) + } } } -impl Drop for EventChain { +impl Drop for EventChain { fn drop(&mut self) { if self.clean_up_tmp_dirs { clean_up_tmp_dirs(&self.labels) @@ -226,8 +307,8 @@ pub struct SimNetwork { clean_up_tmp_dirs: bool, labels: Vec<(NodeLabel, PeerId)>, pub(crate) event_listener: TestEventListener, - user_ev_controller: Option>, - receiver_ch: Receiver<(EventId, PeerId)>, + user_ev_controller: Option>, + receiver_ch: watch::Receiver<(EventId, PeerId)>, number_of_gateways: usize, gateways: Vec<(Builder, GatewayConfig)>, number_of_nodes: usize, @@ -251,7 +332,8 @@ impl SimNetwork { min_connections: usize, ) -> Self { assert!(gateways > 0 && nodes > 0); - let (user_ev_controller, receiver_ch) = channel((0, PeerId::random())); + let (user_ev_controller, mut receiver_ch) = watch::channel((0, PeerId::random())); + receiver_ch.borrow_and_update(); let mut net = Self { name: name.into(), clean_up_tmp_dirs: true, @@ -641,7 +723,7 @@ impl SimNetwork { pub fn event_chain( mut self, total_events: u32, - controller: Option>, + controller: Option>, ) -> EventChain { let user_ev_controller = controller.unwrap_or_else(|| { self.user_ev_controller diff --git a/crates/core/src/node/testing_impl/inter_process.rs b/crates/core/src/node/testing_impl/inter_process.rs index 1d191bd90..0035fa4f9 100644 --- a/crates/core/src/node/testing_impl/inter_process.rs +++ b/crates/core/src/node/testing_impl/inter_process.rs @@ -24,10 +24,6 @@ impl SimPeer { where UsrEv: ClientEventsProxy + Send + 'static, { - // disable console logging but enable trace collector (if it's enabled) - std::env::set_var("FREENET_DISABLE_LOGS", "1"); - crate::config::set_logger(); - let event_register = { #[cfg(feature = "trace-ot")] { diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index ca6d7bda3..a6b608fd2 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -39,7 +39,7 @@ pub struct TestConfig { /// Events are simulated get, puts and other operations. #[arg(long, default_value_t = u32::MAX)] events: u32, - /// Time in milliseconds to wait for the network to be sufficiently connected to start requesting events. + /// Time in milliseconds to wait for the network to be sufficiently connected to start sending events. /// (20% of the expected connections to be processed per gateway) #[arg(long)] wait_duration: Option, @@ -134,6 +134,7 @@ async fn config_sim_network(base_config: &TestConfig) -> anyhow::Result, @@ -218,6 +207,15 @@ mod multiple_process { Child, } + impl Display for Process { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Supervisor => write!(f, "supervisor"), + Self::Child => write!(f, "child"), + } + } + } + pub(super) async fn run( config: &super::TestConfig, cmd_config: &MultiProcessConfig, @@ -235,12 +233,13 @@ mod multiple_process { simulated_network.debug(); // set to avoid deleting temp dirs created let peers = simulated_network.build_peers(); - let (user_ev_controller, event_rx) = tokio::sync::watch::channel((0, PeerId::random())); + // let (user_ev_controller, event_rx) = tokio::sync::watch::channel((0, PeerId::random())); + let (user_ev_controller, event_rx) = tokio::sync::mpsc::channel(1); let seed = config.seed(); let mut supervisor = Supervisor { processes: HashMap::new(), - event_rx, + event_rx: Some(event_rx), waiting_response: FuturesUnordered::new(), queued: HashMap::new(), }; @@ -255,16 +254,67 @@ mod multiple_process { .into_iter() .map(|(label, config)| (label, config.peer_id)) .collect(); + let mut events = EventChain::new(peers, user_ev_controller, config.events, true); + let next_event_wait_time = config + .event_wait_time + .map(Duration::from_millis) + .unwrap_or(Duration::from_millis(200)); + let (connectivity_timeout, network_connection_percent) = + config.get_connection_check_params(); + let events_generated = tokio::task::spawn(async move { + tracing::info!( + "Waiting for network to be sufficiently connected ({}ms timeout, {}%)", + connectivity_timeout.as_millis(), + network_connection_percent * 100.0 + ); + simulated_network + .check_partial_connectivity(connectivity_timeout, network_connection_percent)?; + tracing::info!("Network is sufficiently connected, start sending events"); + while events.next().await.is_some() { + tokio::time::sleep(next_event_wait_time).await; + } + Ok::<_, super::Error>(()) + }); + + let ctrl_c = tokio::signal::ctrl_c(); + let supervisor_task = tokio::task::spawn(supervisor.start_simulation()); - let events = EventChain::new(peers, user_ev_controller, config.events, true); - let next_event_wait_time = config.event_wait_time.map(Duration::from_millis); - for _event_id in events { - if let Some(t) = next_event_wait_time { - std::thread::sleep(t); + tokio::pin!(events_generated); + tokio::pin!(supervisor_task); + tokio::pin!(ctrl_c); + + loop { + tokio::select! { + _ = &mut ctrl_c /* SIGINT handling */ => { + break; + } + res = &mut events_generated => { + match res? { + Ok(()) => { + tracing::info!("Test events generated successfully"); + *events_generated = tokio::task::spawn(futures::future::pending::>()); + continue; + } + Err(err) => { + return Err(err); + } + } + } + finalized = &mut supervisor_task => { + match finalized? { + Ok(_) => { + tracing::info!("Test finalized successfully"); + break; + } + Err(e) => { + tracing::error!("Test finalized with error: {}", e); + return Err(e); + } + } + } } } - supervisor_task.await?; Ok(()) } @@ -274,30 +324,49 @@ mod multiple_process { /// Event driver for the supervisor process. struct Supervisor { processes: HashMap, - event_rx: tokio::sync::watch::Receiver<(u32, PeerId)>, + event_rx: Option>, waiting_response: FuturesUnordered>>, queued: HashMap>, } impl Supervisor { - async fn start_simulation(mut self) { + async fn start_simulation(mut self) -> anyhow::Result<()> { + let ctrl_c = tokio::signal::ctrl_c(); + tokio::pin!(ctrl_c); + + let mut event_rx = self.event_rx.take().expect("should be set"); + let mut finished_events = false; + loop { tokio::select! { - biased; + _ = &mut ctrl_c /* SIGINT handling */ => { + break; + } res = self.waiting_response.next() => { - let (subprocess, responses) = res.unwrap().unwrap(); + let (subprocess, responses) = match res { + Some(Ok(res)) => res, + Some(Err(err)) => { + tracing::error!("Error processing responses: {err}"); + return Err(err); + } + None => { + continue; + } + }; if let Err(err) = self.process_responses(subprocess, responses).await { - eprintln!("Error processing responses: {err}"); - break; + tracing::error!("Error processing responses: {err}"); + return Err(err); } } - msg = self.event_rx.changed() => { - if msg.is_err() { - break; - } - let (event, peer) = *self.event_rx.borrow(); + event = event_rx.recv(), if !finished_events => { + let Some((event, peer)) = event else { + tracing::info!("No more events"); + finished_events = true; + continue; + }; + tracing::debug!("Event {event} fired for {peer}"); let Some(mut subprocess) = self.processes.remove(&peer) else { self.queued.entry(peer).or_default().push_back(IPCMessage::FiredEvent(event)); continue; @@ -305,9 +374,9 @@ mod multiple_process { let mut pending = self.queued.remove(&peer).unwrap_or_default(); pending.push_back(IPCMessage::FiredEvent(event)); let task = async move { - for msg in pending { - msg.send(subprocess.child.stdin.as_mut().expect("not taken")) - .await?; + for msg in pending { + msg.send(subprocess.child.stdin.as_mut().expect("not taken")) + .await?; } let mut returned = vec![]; while let Ok((target, msg)) = InterProcessConnManager::pull_msg( @@ -326,7 +395,8 @@ mod multiple_process { for (_, subprocess) in self.processes.drain() { subprocess.close().await; } - println!("Simulation finished"); + tracing::info!("Simulation finished"); + Ok(()) } async fn process_responses( @@ -336,6 +406,7 @@ mod multiple_process { ) -> anyhow::Result<()> { for (target, data) in responses { if let Some(target) = self.processes.remove(&target) { + tracing::debug!(to = %subprocess.id, "sending message"); let task = async move { target.send_msg(IPCMessage::Data(data)).await }.boxed(); self.waiting_response.push(task); } else { @@ -377,21 +448,23 @@ mod multiple_process { label: &NodeLabel, id: PeerId, ) -> anyhow::Result { - let cmd = Command::new("fdev") + let child = Command::new("fdev") + .kill_on_drop(true) .args(cmd_args) .arg("--id") .arg(label.number().to_string()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) .spawn()?; - Ok(Self { child: cmd, id }) + Ok(Self { child, id }) } async fn config(&mut self, config: &NodeConfig) -> anyhow::Result<(), Error> { let input = self.get_input().ok_or_else(|| anyhow!("closed"))?; let serialize = bincode::serialize(&config)?; input - .write_all(&(serialize.len() as u64).to_le_bytes()) + .write_all(&(serialize.len() as u32).to_le_bytes()) .await?; input.write_all(&serialize).await?; Ok(()) @@ -422,15 +495,20 @@ mod multiple_process { } async fn child(config: &super::TestConfig, id: usize) -> anyhow::Result<()> { - let (user_ev_controller, receiver_ch) = tokio::sync::watch::channel((0, PeerId::random())); - let mut input = tokio::io::stdin(); + // write logs to stderr so stdout and stdin are free of unexpected data + std::env::set_var("FREENET_LOG_TO_STDERR", "1"); + freenet::config::set_logger(); + + let (user_ev_controller, mut receiver_ch) = + tokio::sync::watch::channel((0, PeerId::random())); + receiver_ch.borrow_and_update(); + let mut input = BufReader::new(tokio::io::stdin()); let node_config = Child::get_config(&mut input).await?; let this_child = Child { input, user_ev_controller, peer_id: node_config.peer_id, }; - let mut event_generator = MemoryEventsGen::::new_with_seed( receiver_ch.clone(), node_config.peer_id, @@ -450,13 +528,13 @@ mod multiple_process { /// Controller for a child process. struct Child { - input: tokio::io::Stdin, + input: BufReader, user_ev_controller: tokio::sync::watch::Sender<(u32, PeerId)>, peer_id: PeerId, } impl Child { - async fn get_config(input: &mut tokio::io::Stdin) -> anyhow::Result { + async fn get_config(input: &mut BufReader) -> anyhow::Result { let mut buf = [0u8; 4]; input.read_exact(&mut buf).await?; let config_len = u32::from_le_bytes(buf); @@ -468,11 +546,19 @@ mod multiple_process { async fn event_loop(mut self) -> anyhow::Result<()> { loop { - match IPCMessage::recv(&mut self.input).await? { - IPCMessage::FiredEvent(id) => { + match IPCMessage::recv(&mut self.input).await { + Err(err) => { + tracing::error!( + "Error at peer {} while receiving message: {err}", + self.peer_id + ); + break Err(err); + } + Ok(IPCMessage::FiredEvent(id)) => { + tracing::debug!("Event {id} received"); self.user_ev_controller.send((id, self.peer_id))?; } - IPCMessage::Data(data) => { + Ok(IPCMessage::Data(data)) => { InterProcessConnManager::push_msg(data); } } @@ -493,15 +579,17 @@ mod multiple_process { out.write_all(&id.to_le_bytes()).await?; } Self::Data(data) => { + tracing::info!("Sending data"); out.write_u8(1).await?; out.write_all(&(data.len() as u32).to_le_bytes()).await?; out.write_all(&data).await?; + tracing::info!("Sent data"); } } Ok(()) } - async fn recv(input: &mut tokio::io::Stdin) -> anyhow::Result { + async fn recv(input: &mut BufReader) -> anyhow::Result { let marker = input.read_u8().await?; match marker { 0 => { @@ -542,21 +630,23 @@ mod single_process { .await; let events = config.events; - let next_event_wait_time = config.event_wait_time.map(Duration::from_millis); + let next_event_wait_time = config + .event_wait_time + .map(Duration::from_millis) + .unwrap_or(Duration::from_millis(200)); let (connectivity_timeout, network_connection_percent) = config.get_connection_check_params(); - let events_generated = tokio::task::spawn_blocking(move || { - println!( + let events_generated = tokio::task::spawn(async move { + tracing::info!( "Waiting for network to be sufficiently connected ({}ms timeout, {}%)", connectivity_timeout.as_millis(), network_connection_percent * 100.0 ); simulated_network .check_partial_connectivity(connectivity_timeout, network_connection_percent)?; - for _ in simulated_network.event_chain(events, None) { - if let Some(t) = next_event_wait_time { - std::thread::sleep(t); - } + let mut stream = simulated_network.event_chain(events, None); + while stream.next().await.is_some() { + tokio::time::sleep(next_event_wait_time).await; } Ok::<_, super::Error>(()) }); @@ -583,8 +673,8 @@ mod single_process { res = &mut events_generated => { match res? { Ok(()) => { - println!("Test events generated successfully"); - *events_generated = tokio::task::spawn(futures::future::pending::>()); + tracing::info!("Test events generated successfully"); + *events_generated = tokio::task::spawn(futures::future::pending::>()); continue; } Err(err) => { @@ -595,11 +685,11 @@ mod single_process { finalized = &mut join_peer_tasks => { match finalized { Ok(_) => { - println!("Test finalized successfully"); + tracing::info!("Test finalized successfully"); break; } Err(e) => { - println!("Test finalized with error: {}", e); + tracing::error!("Test finalized with error: {}", e); return Err(e); } } From 4ba69542292a0f5a9e7f7b59d9314e68dcc45b53 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Tue, 21 Nov 2023 19:29:32 +0100 Subject: [PATCH 12/12] Avoid potential deadlocks make child stdout being always processed --- crates/core/src/client_events.rs | 2 +- crates/core/src/config.rs | 23 +-- crates/core/src/contract/executor.rs | 1 - crates/core/src/node.rs | 21 +-- .../src/node/network_bridge/inter_process.rs | 26 ++- crates/core/src/node/network_event_log.rs | 3 +- crates/core/src/node/op_state_manager.rs | 9 +- .../core/src/node/testing_impl/in_memory.rs | 2 + .../src/node/testing_impl/inter_process.rs | 2 +- crates/core/src/operations/put.rs | 2 +- crates/core/src/ring.rs | 11 +- crates/fdev/src/testing.rs | 148 ++++++++++-------- 12 files changed, 140 insertions(+), 110 deletions(-) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index d1fc30eb9..92ec15d09 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -260,7 +260,7 @@ pub(crate) mod test { async { loop { if self.signal.changed().await.is_ok() { - let (ev_id, pk) = *self.signal.borrow_and_update(); + let (ev_id, pk) = *self.signal.borrow(); if self.rng.is_some() && pk == self.id { let res = OpenRequest { client_id: ClientId::FIRST, diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 763e49697..1a089383a 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -171,18 +171,6 @@ impl Config { .store(local_mode, std::sync::atomic::Ordering::SeqCst); } - #[cfg(feature = "trace-ot")] - fn node_mode() -> OperationMode { - if Self::conf() - .local_mode - .load(std::sync::atomic::Ordering::SeqCst) - { - OperationMode::Local - } else { - OperationMode::Network - } - } - pub fn db_dir(&self) -> PathBuf { if self.local_mode.load(std::sync::atomic::Ordering::SeqCst) { self.config_paths.db_dir.join("local") @@ -421,16 +409,11 @@ mod tracer { #[cfg(feature = "trace-ot")] { - use super::*; let disabled_ot_traces = std::env::var("FREENET_DISABLE_TRACES").is_ok(); - // FIXME - let identifier = if matches!(Config::node_mode(), OperationMode::Local) { - "freenet-core".to_string() + let identifier = if let Ok(peer) = std::env::var("FREENET_PEER_ID") { + format!("freenet-core-{peer}") } else { - format!( - "freenet-core-{peer}", - peer = Config::conf().local_peer_keypair.public().to_peer_id() - ) + "freenet-core".to_string() }; let tracing_ot_layer = { // Connect the Jaeger OT tracer with the tracing middleware diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index c2e6be3f1..00ee03ad0 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -1209,7 +1209,6 @@ impl Executor { let db_path = data_dir.join("db"); std::fs::create_dir_all(&db_path).expect("directory created"); let log_file = data_dir.join("_EVENT_LOG_LOCAL"); - eprintln!("{log_file:?}"); crate::config::Config::set_event_log(log_file); let state_store = StateStore::new(Storage::new(Some(&db_path)).await?, u16::MAX as u32).unwrap(); diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 35ffd31a1..8bdbfdb0c 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -406,7 +406,11 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { - todo!() + // FIXME: DO THIS + tracing::debug!( + this_peer = %op_storage.ring.peer_key, + "Received update from user event", + ); } ContractRequest::Get { key, @@ -414,8 +418,8 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { // Initialize a get op. tracing::debug!( - "Received get from user event @ {}", - &op_storage.ring.peer_key + this_peer = %op_storage.ring.peer_key, + "Received get from user event", ); let op = get::start_op(key, contract); if let Err(err) = get::request_get(&op_storage, op, Some(client_id)).await { @@ -479,10 +483,9 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { tracing::debug!( tx = %$id, - concat!("Handling ", $op, " request @ {}"), - $op_storage.ring.peer_key, + this_peer = %$op_storage.ring.peer_key, + concat!("Handling ", $op, " request"), ); }; } diff --git a/crates/core/src/node/network_bridge/inter_process.rs b/crates/core/src/node/network_bridge/inter_process.rs index 4b43420e6..dafdfdc8c 100644 --- a/crates/core/src/node/network_bridge/inter_process.rs +++ b/crates/core/src/node/network_bridge/inter_process.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, OnceLock}; use futures::{future::BoxFuture, FutureExt}; use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, + io::{AsyncReadExt, AsyncWriteExt, BufWriter, Stdout}, sync::{ watch::{Receiver, Sender}, Mutex, @@ -23,7 +23,7 @@ static INCOMING_DATA: OnceLock> = OnceLock::new(); #[derive(Clone)] pub struct InterProcessConnManager { recv: Receiver, - output: Arc>, + output: Arc>>, } impl InterProcessConnManager { @@ -32,7 +32,7 @@ impl InterProcessConnManager { INCOMING_DATA.set(sender).expect("shouldn't be set"); Self { recv, - output: Arc::new(Mutex::new(tokio::io::stdout())), + output: Arc::new(Mutex::new(BufWriter::new(tokio::io::stdout()))), } } @@ -42,16 +42,23 @@ impl InterProcessConnManager { pub async fn pull_msg( stdout: &mut tokio::process::ChildStdout, - ) -> std::io::Result<(PeerId, Data)> { + ) -> std::io::Result> { let mut msg_len = [0u8; 4]; - stdout.read_exact(&mut msg_len).await?; + let Ok(read_res) = tokio::time::timeout( + std::time::Duration::from_millis(100), + stdout.read_exact(&mut msg_len), + ) + .await + else { + return Ok(None); + }; + read_res?; let msg_len = u32::from_le_bytes(msg_len) as usize; let buf = &mut vec![0u8; msg_len]; stdout.read_exact(buf).await?; let (target, data) = bincode::deserialize(buf) .map_err(|_| std::io::Error::from(std::io::ErrorKind::Other))?; - tracing::debug!(%target, "network message received"); - Ok((target, data)) + Ok(Some((target, data))) } } @@ -62,7 +69,7 @@ impl NetworkBridgeExt for InterProcessConnManager { .changed() .await .map_err(|_| ConnectionError::Timeout)?; - let data = &*self.recv.borrow_and_update(); + let data = &*self.recv.borrow(); let deser = bincode::deserialize(data)?; Ok(deser) } @@ -73,10 +80,13 @@ impl NetworkBridgeExt for InterProcessConnManager { #[async_trait::async_trait] impl NetworkBridge for InterProcessConnManager { async fn send(&self, target: &PeerId, msg: NetMessage) -> super::ConnResult<()> { + tracing::debug!(%target, ?msg, "sending network message out"); let data = bincode::serialize(&(*target, msg))?; let output = &mut *self.output.lock().await; output.write_all(&(data.len() as u32).to_le_bytes()).await?; output.write_all(&data).await?; + output.flush().await?; + tracing::debug!(%target, bytes = data.len(), "sent network message out"); Ok(()) } diff --git a/crates/core/src/node/network_event_log.rs b/crates/core/src/node/network_event_log.rs index 9a40f6cc9..5e89b643f 100644 --- a/crates/core/src/node/network_event_log.rs +++ b/crates/core/src/node/network_event_log.rs @@ -490,9 +490,8 @@ impl EventRegister { Ok(()) } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; // wait for the node to start + tokio::time::sleep(std::time::Duration::from_millis(200)).await; // wait for the node to start let event_log_path = crate::config::Config::conf().event_log(); - tracing::info!(?event_log_path); let mut event_log = match OpenOptions::new() .write(true) .read(true) diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index d92c5ea22..71d2a677e 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -70,7 +70,12 @@ impl OpManager { let ops = Arc::new(Ops::default()); let (new_transactions, rx) = tokio::sync::mpsc::channel(100); - let parent_span = tracing::Span::current(); + let current_span = tracing::Span::current(); + let garbage_span = if current_span.is_none() { + tracing::info_span!("garbage_cleanup_task") + } else { + tracing::info_span!(parent: current_span, "garbage_cleanup_task") + }; GlobalExecutor::spawn( garbage_cleanup_task( rx, @@ -78,7 +83,7 @@ impl OpManager { ring.live_tx_tracker.clone(), event_register, ) - .instrument(tracing::info_span!(parent: parent_span, "garbage_cleanup_task")), + .instrument(garbage_span), ); Ok(Self { diff --git a/crates/core/src/node/testing_impl/in_memory.rs b/crates/core/src/node/testing_impl/in_memory.rs index 65d29f98b..1bb41b849 100644 --- a/crates/core/src/node/testing_impl/in_memory.rs +++ b/crates/core/src/node/testing_impl/in_memory.rs @@ -33,6 +33,7 @@ impl Builder { let (notification_channel, notification_tx) = EventLoopNotifications::channel(); let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); + let _guard = parent_span.enter(); let op_storage = Arc::new(OpManager::new( notification_tx, ops_ch_channel, @@ -40,6 +41,7 @@ impl Builder { &gateways, self.event_register.clone(), )?); + std::mem::drop(_guard); let (_executor_listener, executor_sender) = executor_channel(op_storage.clone()); let contract_handler = MemoryContractHandler::build(ch_channel, executor_sender, self.contract_handler_name) diff --git a/crates/core/src/node/testing_impl/inter_process.rs b/crates/core/src/node/testing_impl/inter_process.rs index 0035fa4f9..48824a63e 100644 --- a/crates/core/src/node/testing_impl/inter_process.rs +++ b/crates/core/src/node/testing_impl/inter_process.rs @@ -76,7 +76,7 @@ impl SimPeer { GlobalExecutor::spawn( contract::contract_handling(contract_handler) - .instrument(tracing::info_span!("contract_handling")), + .instrument(tracing::info_span!("contract_handling", peer = %self.config.peer_id)), ); let running_node = super::RunnerConfig { diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 6a7a7b3b5..d09ac6831 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -189,7 +189,7 @@ impl Operation for PutOp { let key = contract.key(); tracing::debug!( - "Rquesting put for contract {} from {} to {}", + "Requesting put for contract {} from {} to {}", key, sender.peer, target.peer diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index f284d0da5..36edef23f 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -297,11 +297,16 @@ impl Ring { } let ring = Arc::new(ring); - let parent_span = tracing::Span::current(); + let current_span = tracing::Span::current(); + let span = if current_span.is_none() { + tracing::info_span!("connection_maintenance") + } else { + tracing::info_span!(parent: current_span, "connection_maintenance") + }; GlobalExecutor::spawn( ring.clone() .connection_maintenance(event_loop_notifier, live_tx_tracker, missing_candidate_rx) - .instrument(tracing::info_span!(parent: parent_span, "connection_maintenance")), + .instrument(span), ); Ok(ring) } @@ -696,7 +701,7 @@ impl Ring { match loc { Ok(loc) => loc, Err(_) => { - tracing::debug!(peer = %self.own_location(), "Insufficient data gathered by the topology manager"); + tracing::trace!(peer = %self.own_location(), "Insufficient data gathered by the topology manager"); acquire_max_connections.tick().await; continue; } diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index a6b608fd2..0fd124ad5 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -145,9 +145,10 @@ mod multiple_process { SimPeer, }; use futures::{future::BoxFuture, stream::FuturesUnordered, FutureExt, StreamExt}; + use rand::Rng; use tokio::{ io::{AsyncReadExt, AsyncWriteExt, BufReader, Stdin}, - process::Command, + process::{ChildStdout, Command}, }; use super::Error; @@ -239,9 +240,10 @@ mod multiple_process { let seed = config.seed(); let mut supervisor = Supervisor { processes: HashMap::new(), - event_rx: Some(event_rx), - waiting_response: FuturesUnordered::new(), queued: HashMap::new(), + sending: FuturesUnordered::new(), + responses: FuturesUnordered::new(), + event_rx: Some(event_rx), }; let cmd_args = config.subprocess_command(seed); for (label, node) in &peers { @@ -324,10 +326,11 @@ mod multiple_process { /// Event driver for the supervisor process. struct Supervisor { processes: HashMap, - event_rx: Option>, - waiting_response: - FuturesUnordered>>, queued: HashMap>, + sending: FuturesUnordered>>, + responses: + FuturesUnordered>>, + event_rx: Option>, } impl Supervisor { @@ -338,13 +341,22 @@ mod multiple_process { let mut event_rx = self.event_rx.take().expect("should be set"); let mut finished_events = false; + for child_stdout in self + .processes + .values_mut() + .map(|sp| sp.child.stdout.take().expect("should be set")) + { + self.responses + .push(SubProcess::get_child_responses(child_stdout).boxed()); + } + loop { tokio::select! { _ = &mut ctrl_c /* SIGINT handling */ => { break; } - res = self.waiting_response.next() => { - let (subprocess, responses) = match res { + res = self.responses.next(), if !self.responses.is_empty() => { + let (child_stdout, responses) = match res { Some(Ok(res)) => res, Some(Err(err)) => { tracing::error!("Error processing responses: {err}"); @@ -354,11 +366,36 @@ mod multiple_process { continue; } }; - if let Err(err) = self.process_responses(subprocess, responses).await { - tracing::error!("Error processing responses: {err}"); - return Err(err); + self.responses.push(SubProcess::get_child_responses(child_stdout).boxed()); + self.process_responses(responses); + } + completed_send = self.sending.next(), if !self.sending.is_empty() => { + let mut subprocess = match completed_send { + Some(Ok(res)) => res, + Some(Err(err)) => { + tracing::error!("Error sending message: {err}"); + return Err(err); + } + None => { + continue; + } + }; + let peer_queue = &mut *self.queued.entry(subprocess.id).or_default(); + if !peer_queue.is_empty() { + let n = rand::thread_rng().gen_range(0..=peer_queue.len()); + let messages = peer_queue.drain(..n).collect::>(); + let task = async move { + let stdin = subprocess.child.stdin.as_mut().expect("not taken"); + tracing::debug!(peer = %subprocess.id, "Draining {} messages from queue", n + 1); + for pending in messages { + pending.send(stdin).await?; + } + Ok(subprocess) + }.boxed(); + self.sending.push(task); + } else { + self.processes.insert(subprocess.id, subprocess); } - } event = event_rx.recv(), if !finished_events => { let Some((event, peer)) = event else { @@ -366,7 +403,6 @@ mod multiple_process { finished_events = true; continue; }; - tracing::debug!("Event {event} fired for {peer}"); let Some(mut subprocess) = self.processes.remove(&peer) else { self.queued.entry(peer).or_default().push_back(IPCMessage::FiredEvent(event)); continue; @@ -378,17 +414,9 @@ mod multiple_process { msg.send(subprocess.child.stdin.as_mut().expect("not taken")) .await?; } - let mut returned = vec![]; - while let Ok((target, msg)) = InterProcessConnManager::pull_msg( - subprocess.child.stdout.as_mut().expect("stdout not taken"), - ) - .await - { - returned.push((target, msg)); - } - Ok((subprocess, returned)) + Ok(subprocess) }.boxed(); - self.waiting_response.push(task); + self.sending.push(task); } } } @@ -399,41 +427,24 @@ mod multiple_process { Ok(()) } - async fn process_responses( - &mut self, - mut subprocess: SubProcess, - responses: ChildResponses, - ) -> anyhow::Result<()> { + fn process_responses(&mut self, responses: ChildResponses) { for (target, data) in responses { - if let Some(target) = self.processes.remove(&target) { - tracing::debug!(to = %subprocess.id, "sending message"); - let task = async move { target.send_msg(IPCMessage::Data(data)).await }.boxed(); - self.waiting_response.push(task); + if let Some(mut target) = self.processes.remove(&target) { + tracing::debug!(to = %target.id, "Sending message"); + let task = async move { + target.send_msg(IPCMessage::Data(data)).await?; + Ok(target) + } + .boxed(); + self.sending.push(task); } else { + tracing::debug!(%target, "Queuing message"); self.queued .entry(target) .or_default() .push_back(IPCMessage::Data(data)); } } - if let Some(mut queued) = self.queued.remove(&subprocess.id) { - for msg in queued.drain(..) { - msg.send(subprocess.child.stdin.as_mut().expect("not taken")) - .await?; - while let Ok((target, data)) = InterProcessConnManager::pull_msg( - subprocess.child.stdout.as_mut().expect("stdout not taken"), - ) - .await - { - self.queued - .entry(target) - .or_default() - .push_back(IPCMessage::Data(data)); - } - } - } - self.processes.insert(subprocess.id, subprocess); - Ok(()) } } @@ -470,18 +481,33 @@ mod multiple_process { Ok(()) } - async fn send_msg(mut self, msg: IPCMessage) -> anyhow::Result<(Self, ChildResponses)> { + #[inline] + async fn send_msg(&mut self, msg: IPCMessage) -> anyhow::Result<()> { msg.send(self.child.stdin.as_mut().expect("not taken")) .await?; + Ok(()) + } + + async fn get_child_responses( + mut stdout: ChildStdout, + ) -> anyhow::Result<(ChildStdout, Vec<(PeerId, Vec)>)> { let mut returned = vec![]; - while let Ok((target, msg)) = InterProcessConnManager::pull_msg( - self.child.stdout.as_mut().expect("stdout not taken"), - ) - .await - { - returned.push((target, msg)); + loop { + match InterProcessConnManager::pull_msg(&mut stdout).await { + Ok(Some((target, data))) => { + returned.push((target, data)); + } + Ok(None) if !returned.is_empty() => { + break; + } + Ok(None) => {} + Err(err) => { + return Err(err.into()); + } + } } - Ok((self, returned)) + tracing::debug!(len = %returned.len(), "Returning messages"); + Ok((stdout, returned)) } async fn close(mut self) { @@ -497,7 +523,6 @@ mod multiple_process { async fn child(config: &super::TestConfig, id: usize) -> anyhow::Result<()> { // write logs to stderr so stdout and stdin are free of unexpected data std::env::set_var("FREENET_LOG_TO_STDERR", "1"); - freenet::config::set_logger(); let (user_ev_controller, mut receiver_ch) = tokio::sync::watch::channel((0, PeerId::random())); @@ -509,6 +534,8 @@ mod multiple_process { user_ev_controller, peer_id: node_config.peer_id, }; + std::env::set_var("FREENET_PEER_ID", node_config.peer_id.to_string()); + freenet::config::set_logger(); let mut event_generator = MemoryEventsGen::::new_with_seed( receiver_ch.clone(), node_config.peer_id, @@ -555,7 +582,6 @@ mod multiple_process { break Err(err); } Ok(IPCMessage::FiredEvent(id)) => { - tracing::debug!("Event {id} received"); self.user_ev_controller.send((id, self.peer_id))?; } Ok(IPCMessage::Data(data)) => { @@ -579,11 +605,9 @@ mod multiple_process { out.write_all(&id.to_le_bytes()).await?; } Self::Data(data) => { - tracing::info!("Sending data"); out.write_u8(1).await?; out.write_all(&(data.len() as u32).to_le_bytes()).await?; out.write_all(&data).await?; - tracing::info!("Sent data"); } } Ok(())