From 997c0e4300387339a18ca634809043cf78f5ac17 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 13 Oct 2023 16:32:38 +0200 Subject: [PATCH 01/14] Make sim network compile --- crates/core/src/node/event_log.rs | 6 +- crates/core/src/node/tests.rs | 158 ++++++++++++++---------- crates/core/src/operations/get.rs | 26 ++-- crates/core/src/operations/join_ring.rs | 2 +- crates/core/src/operations/put.rs | 14 +-- crates/core/src/operations/subscribe.rs | 4 +- 6 files changed, 119 insertions(+), 91 deletions(-) diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index e7d6bcee7..f27e3d8bc 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -411,13 +411,13 @@ pub(super) mod test_utils { use parking_lot::RwLock; use super::*; - use crate::{message::TxType, ring::Distance}; + use crate::{message::TxType, node::tests::NodeLabel, ring::Distance}; static LOG_ID: AtomicUsize = AtomicUsize::new(0); #[derive(Clone)] pub(crate) struct TestEventListener { - node_labels: Arc>, + node_labels: Arc>, tx_log: Arc>>, logs: Arc>>, } @@ -431,7 +431,7 @@ pub(super) mod test_utils { } } - pub fn add_node(&mut self, label: String, peer: PeerKey) { + pub fn add_node(&mut self, label: NodeLabel, peer: PeerKey) { self.node_labels.insert(label, peer); } diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index 88b9d1d68..86919eb44 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fmt::Write, net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener}, + sync::Arc, time::{Duration, Instant}, }; @@ -44,12 +45,12 @@ pub fn get_dynamic_port() -> u16 { /// A simulated in-memory network topology. pub(crate) struct SimNetwork { - pub labels: HashMap, + pub labels: HashMap, pub event_listener: TestEventListener, - usr_ev_controller: Sender<(EventId, PeerKey)>, + user_ev_controller: Sender<(EventId, PeerKey)>, receiver_ch: Receiver<(EventId, PeerKey)>, gateways: Vec<(NodeInMemory, GatewayConfig)>, - nodes: Vec<(NodeInMemory, String)>, + nodes: Vec<(NodeInMemory, NodeLabel)>, ring_max_htl: usize, rnd_if_htl_above: usize, max_connections: usize, @@ -58,6 +59,43 @@ pub(crate) struct SimNetwork { pub(crate) type EventId = usize; +#[derive(PartialEq, Eq, Hash, Clone)] +pub(crate) struct NodeLabel(Arc); + +impl NodeLabel { + fn gateway(id: usize) -> Self { + Self(format!("gateway-{id}").into()) + } + + fn node(id: usize) -> Self { + Self(format!("node-{id}").into()) + } + + fn is_gateway(&self) -> bool { + self.0.starts_with("gateway") + } +} + +impl std::fmt::Display for NodeLabel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::ops::Deref for NodeLabel { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a> From<&'a str> for NodeLabel { + fn from(value: &'a str) -> Self { + Self(value.to_string().into()) + } +} + #[derive(Clone)] pub(crate) struct NodeSpecification { /// Pair of contract and the initial value @@ -69,7 +107,7 @@ pub(crate) struct NodeSpecification { #[derive(Clone)] struct GatewayConfig { - label: String, + label: NodeLabel, port: u16, id: PeerId, location: Location, @@ -85,12 +123,12 @@ impl SimNetwork { min_connections: usize, ) -> Self { assert!(gateways > 0 && nodes > 0); - let (usr_ev_controller, _rcv_copy) = channel((0, PeerKey::random())); + let (user_ev_controller, receiver_ch) = channel((0, PeerKey::random())); let mut net = Self { event_listener: TestEventListener::new(), labels: HashMap::new(), - usr_ev_controller, - receiver_ch: _rcv_copy, + user_ev_controller, + receiver_ch, gateways: Vec::with_capacity(gateways), nodes: Vec::with_capacity(nodes), ring_max_htl, @@ -98,17 +136,17 @@ impl SimNetwork { max_connections, min_connections, }; - net.build_gateways(gateways); + net.build_gateways(gateways).await; net.build_nodes(nodes).await; net } #[instrument(skip(self))] - fn build_gateways(&mut self, num: usize) { + async fn build_gateways(&mut self, num: usize) { info!("Building {} gateways", num); let mut configs = Vec::with_capacity(num); for node_no in 0..num { - let label = format!("gateway-{}", node_no); + let label = NodeLabel::gateway(node_no); let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); let port = get_free_port().unwrap(); @@ -141,31 +179,29 @@ impl SimNetwork { )); } - // FIXME: - // for (mut this_node, this_config) in configs { - // for GatewayConfig { - // port, id, location, .. - // } in configs.iter().filter_map(|(_, config)| { - // if this_config.label != config.label { - // Some(config) - // } else { - // None - // } - // }) { - // this_node.add_gateway( - // InitPeerNode::new(*id, *location) - // .listening_ip(Ipv6Addr::LOCALHOST) - // .listening_port(*port), - // ); - // } - - // let gateway = NodeInMemory::::build::( - // this_node, - // Some(Box::new(self.event_listener.clone())), - // ) - // .unwrap(); - // self.gateways.push((gateway, this_config)); - // } + let gateways: Vec<_> = configs.iter().map(|(_, gw)| gw.clone()).collect(); + for (mut this_node, this_config) in configs { + for GatewayConfig { + port, id, location, .. + } in gateways + .iter() + .filter(|config| this_config.label != config.label) + { + this_node.add_gateway( + InitPeerNode::new(*id, *location) + .listening_ip(Ipv6Addr::LOCALHOST) + .listening_port(*port), + ); + } + let gateway = NodeInMemory::build::( + this_node, + self.event_listener.clone(), + (), + ) + .await + .unwrap(); + self.gateways.push((gateway, this_config)); + } } #[instrument(skip(self))] @@ -178,7 +214,7 @@ impl SimNetwork { .collect(); for node_no in 0..num { - let label = format!("node-{}", node_no); + let label = NodeLabel::node(node_no); let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); @@ -205,7 +241,7 @@ impl SimNetwork { self.event_listener .add_node(label.clone(), PeerKey::from(id)); - let node = NodeInMemory::build::( + let node = NodeInMemory::build::( config, self.event_listener.clone(), (), @@ -220,7 +256,7 @@ impl SimNetwork { self.build_with_specs(HashMap::new()).await } - pub async fn build_with_specs(&mut self, mut specs: HashMap) { + pub async fn build_with_specs(&mut self, mut specs: HashMap) { let mut gw_not_init = self.gateways.len(); let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { @@ -237,7 +273,7 @@ impl SimNetwork { fn initialize_peer( &mut self, mut peer: NodeInMemory, - label: String, + label: NodeLabel, node_specs: Option, ) { let mut user_events = MemoryEventsGen::new(self.receiver_ch.clone(), peer.peer_key); @@ -257,23 +293,20 @@ impl SimNetwork { }); } - pub fn get_locations_by_node(&self) -> HashMap { - let mut locations_by_node: HashMap = HashMap::new(); + pub fn get_locations_by_node(&self) -> HashMap { + let mut locations_by_node: HashMap = HashMap::new(); // Get node and gateways location by label for (node, label) in &self.nodes { - locations_by_node.insert(label.to_string(), node.op_storage.ring.own_location()); + locations_by_node.insert(label.clone(), node.op_storage.ring.own_location()); } for (node, config) in &self.gateways { - locations_by_node.insert( - config.label.to_string(), - node.op_storage.ring.own_location(), - ); + locations_by_node.insert(config.label.clone(), node.op_storage.ring.own_location()); } locations_by_node } - pub fn connected(&self, peer: &str) -> bool { + pub fn connected(&self, peer: &NodeLabel) -> bool { if let Some(key) = self.labels.get(peer) { self.event_listener.is_connected(key) } else { @@ -281,7 +314,12 @@ impl SimNetwork { } } - pub fn has_put_contract(&self, peer: &str, key: &ContractKey, value: &WrappedState) -> bool { + pub fn has_put_contract( + &self, + peer: &NodeLabel, + key: &ContractKey, + value: &WrappedState, + ) -> bool { if let Some(pk) = self.labels.get(peer) { self.event_listener.has_put_contract(pk, key, value) } else { @@ -289,7 +327,7 @@ impl SimNetwork { } } - pub fn has_got_contract(&self, peer: &str, key: &ContractKey) -> bool { + pub fn has_got_contract(&self, peer: &NodeLabel, key: &ContractKey) -> bool { if let Some(pk) = self.labels.get(peer) { self.event_listener.has_got_contract(pk, key) } else { @@ -311,7 +349,7 @@ impl SimNetwork { /// Returns the connectivity in the network per peer (that is all the connections /// this peers has registered). - pub fn node_connectivity(&self) -> HashMap> { + pub fn node_connectivity(&self) -> HashMap> { let mut peers_connections = HashMap::with_capacity(self.labels.len()); let key_to_label: HashMap<_, _> = self.labels.iter().map(|(k, v)| (v, k)).collect(); for (label, key) in &self.labels { @@ -328,7 +366,7 @@ impl SimNetwork { pub async fn trigger_event( &self, - label: &str, + label: &NodeLabel, event_id: EventId, await_for: Option, ) -> Result<(), anyhow::Error> { @@ -336,7 +374,7 @@ impl SimNetwork { .labels .get(label) .ok_or_else(|| anyhow::anyhow!("node not found"))?; - self.usr_ev_controller + self.user_ev_controller .send((event_id, *peer)) .expect("node listeners disconnected"); if let Some(sleep_time) = await_for { @@ -375,7 +413,7 @@ pub(crate) async fn check_connectivity( let elapsed = Instant::now(); while elapsed.elapsed() < time_out && connected.len() < num_nodes { for node in 0..num_nodes { - if !connected.contains(&node) && sim_nodes.connected(&format!("node-{}", node)) { + if !connected.contains(&node) && sim_nodes.connected(&NodeLabel::node(node)) { connected.insert(node); } } @@ -409,13 +447,7 @@ pub(crate) async fn check_connectivity( let mut connections_per_peer: Vec<_> = node_connectivity .iter() .map(|(k, v)| (k, v.len())) - .filter_map(|(k, v)| { - if !k.starts_with("gateway") { - Some(v) - } else { - None - } - }) + .filter_map(|(k, v)| if !k.is_gateway() { Some(v) } else { None }) .collect(); // ensure at least some normal nodes have more than one connection @@ -433,12 +465,12 @@ pub(crate) async fn check_connectivity( Ok(()) } -fn pretty_print_connections(conns: &HashMap>) -> String { +fn pretty_print_connections(conns: &HashMap>) -> String { let mut connections = String::from("Node connections:\n"); let mut conns = conns.iter().collect::>(); - conns.sort_by(|a, b| a.0.cmp(b.0)); + conns.sort_by(|(a, _), (b, _)| a.cmp(b)); for (peer, conns) in conns { - if peer.starts_with("gateway") { + if peer.is_gateway() { continue; } writeln!(&mut connections, "{peer}:").unwrap(); diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 56d03bc11..33f8af4a4 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -807,7 +807,6 @@ mod test { use super::*; use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_get_op_between_nodes() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 1usize; @@ -840,10 +839,7 @@ mod test { contract_subscribers: HashMap::new(), }; - let get_specs = HashMap::from_iter([ - ("node-0".to_string(), node_0), - ("gateway-0".to_string(), gw_0), - ]); + let get_specs = HashMap::from_iter([("node-0".into(), node_0), ("gateway-0".into(), gw_0)]); // establish network let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; @@ -852,10 +848,10 @@ mod test { // trigger get @ node-0, which does not own the contract sim_nodes - .trigger_event("node-0", 1, Some(Duration::from_millis(100))) + .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(100))) .await?; tokio::time::sleep(Duration::from_millis(100)).await; - assert!(sim_nodes.has_got_contract("node-0", &key)); + assert!(sim_nodes.has_got_contract(&"node-0".into(), &key)); Ok(()) } @@ -882,7 +878,7 @@ mod test { contract_subscribers: HashMap::new(), }; - let get_specs = HashMap::from_iter([("node-1".to_string(), node_1)]); + let get_specs = HashMap::from_iter([("node-1".into(), node_1)]); // establish network let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; @@ -891,9 +887,9 @@ mod test { // trigger get @ node-1, which does not own the contract sim_nodes - .trigger_event("node-1", 1, Some(Duration::from_millis(100))) + .trigger_event(&"node-1".into(), 1, Some(Duration::from_millis(100))) .await?; - assert!(!sim_nodes.has_got_contract("node-1", &key)); + assert!(!sim_nodes.has_got_contract(&"node-1".into(), &key)); Ok(()) } @@ -940,9 +936,9 @@ mod test { }; let get_specs = HashMap::from_iter([ - ("node-0".to_string(), node_0), - ("node-1".to_string(), node_1), - ("gateway-0".to_string(), gw_0), + ("node-0".into(), node_0), + ("node-1".into(), node_1), + ("gateway-0".into(), gw_0), ]); // establish network @@ -951,9 +947,9 @@ mod test { check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; sim_nodes - .trigger_event("node-0", 1, Some(Duration::from_millis(500))) + .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(500))) .await?; - assert!(sim_nodes.has_got_contract("node-0", &key)); + assert!(sim_nodes.has_got_contract(&"node-0".into(), &key)); Ok(()) } } diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index 32415cade..936751a87 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -1006,7 +1006,7 @@ mod test { let mut sim_nodes = SimNetwork::new(1, 1, 1, 1, 2, 2).await; sim_nodes.build().await; tokio::time::sleep(Duration::from_secs(3)).await; - assert!(sim_nodes.connected("node-0")); + assert!(sim_nodes.connected(&"node-0".into())); } /// Once a gateway is left without remaining open slots, ensure forwarding connects diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 181a1ffc6..4a157ac2d 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -918,8 +918,8 @@ mod test { let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; let mut locations = sim_nodes.get_locations_by_node(); - let node0_loc = locations.remove("node-0").unwrap(); - let node1_loc = locations.remove("node-1").unwrap(); + let node0_loc = locations.remove(&"node-0".into()).unwrap(); + let node1_loc = locations.remove(&"node-1".into()).unwrap(); // both own the contract, and one triggers an update let node_0 = NodeSpecification { @@ -961,9 +961,9 @@ mod test { // establish network let put_specs = HashMap::from_iter([ - ("node-0".to_string(), node_0), - ("node-1".to_string(), node_1), - ("gateway-0".to_string(), gw_0), + ("node-0".into(), node_0), + ("node-1".into(), node_1), + ("gateway-0".into(), gw_0), ]); sim_nodes.build_with_specs(put_specs).await; @@ -972,9 +972,9 @@ mod test { // trigger the put op @ gw-0, this sim_nodes - .trigger_event("gateway-0", 1, Some(Duration::from_secs(3))) + .trigger_event(&"gateway-0".into(), 1, Some(Duration::from_secs(3))) .await?; - assert!(sim_nodes.has_put_contract("gateway-0", &key, &new_value)); + assert!(sim_nodes.has_put_contract(&"gateway-0".into(), &key, &new_value)); assert!(sim_nodes.event_listener.contract_broadcasted(&key)); Ok(()) } diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 84b03563d..5b02883cb 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -506,8 +506,8 @@ mod test { }; let subscribe_specs = HashMap::from_iter([ - ("node-0".to_string(), first_node), - ("node-1".to_string(), second_node), + ("node-0".into(), first_node), + ("node-1".into(), second_node), ]); let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; sim_nodes.build_with_specs(subscribe_specs).await; From f3395da16470332df019a8780755054ed118e173 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sun, 15 Oct 2023 20:53:16 +0200 Subject: [PATCH 02/14] Add mock executor impl --- crates/core/src/client_events.rs | 66 +++++---- crates/core/src/contract.rs | 41 +++--- crates/core/src/contract/executor.rs | 139 ++++++++++++------ crates/core/src/contract/handler.rs | 8 +- crates/core/src/contract/in_memory.rs | 31 ++-- crates/core/src/contract/storages/rocks_db.rs | 7 +- crates/core/src/contract/storages/sqlite.rs | 7 +- crates/core/src/lib.rs | 2 +- crates/core/src/node.rs | 75 +++++----- crates/core/src/node/p2p_impl.rs | 4 +- crates/core/src/node/tests.rs | 7 +- crates/core/src/operations.rs | 3 + crates/core/src/operations/get.rs | 36 ++++- crates/core/src/operations/join_ring.rs | 24 ++- crates/core/src/operations/put.rs | 11 +- crates/core/src/operations/subscribe.rs | 11 +- crates/core/src/ring.rs | 3 +- crates/core/src/runtime/mod.rs | 2 +- crates/fdev/src/commands.rs | 14 +- crates/fdev/src/local_node/state.rs | 11 +- 20 files changed, 309 insertions(+), 193 deletions(-) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index b8574ed32..a6aca2218 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -145,8 +145,8 @@ pub(crate) mod test { // FIXME: remove unused #![allow(unused)] - use std::collections::HashMap; use std::sync::Arc; + use std::{collections::HashMap, time::Duration}; use freenet_stdlib::client_api::ContractRequest; use freenet_stdlib::prelude::*; @@ -264,37 +264,39 @@ pub(crate) mod test { impl ClientEventsProxy for MemoryEventsGen { fn recv(&mut self) -> BoxFuture<'_, Result, ClientError>> { - // async move { - // loop { - // if self.signal.changed().await.is_ok() { - // let (ev_id, pk) = *self.signal.borrow(); - // if pk == self.id && !self.random { - // let res = OpenRequest { - // id: ClientId(1), - // request: self - // .generate_deterministic_event(&ev_id) - // .expect("event not found"), - // notification_channel: None, - // }; - // return Ok(res); - // } else if pk == self.id { - // let res = OpenRequest { - // id: ClientId(1), - // request: self.generate_rand_event(), - // notification_channel: None, - // }; - // return Ok(res); - // } - // } else { - // log::debug!("sender half of user event gen dropped"); - // // probably the process finished, wait for a bit and then kill the thread - // tokio::time::sleep(Duration::from_secs(1)).await; - // panic!("finished orphan background thread"); - // } - // } - // } - // .boxed() - todo!("fixme") + async { + loop { + if self.signal.changed().await.is_ok() { + let (ev_id, pk) = *self.signal.borrow(); + if pk == self.id && !self.random { + let res = OpenRequest { + client_id: ClientId::FIRST, + request: self + .generate_deterministic_event(&ev_id) + .expect("event not found") + .into(), + notification_channel: None, + token: None, + }; + return Ok(res.into_owned()); + } else if pk == self.id { + let res = OpenRequest { + client_id: ClientId::FIRST, + request: self.generate_rand_event().into(), + notification_channel: None, + token: None, + }; + return Ok(res.into_owned()); + } + } else { + tracing::debug!("sender half of user event gen dropped"); + // probably the process finished, wait for a bit and then kill the thread + tokio::time::sleep(Duration::from_secs(1)).await; + panic!("finished orphan background thread"); + } + } + } + .boxed() } fn send( diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index ef3d77fdd..419282147 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -1,4 +1,5 @@ use crate::runtime::ContractError as ContractRtError; +use either::Either; use freenet_stdlib::prelude::*; mod executor; @@ -23,6 +24,7 @@ pub use executor::{Executor, ExecutorError, OperationMode}; use executor::ContractExecutor; pub(crate) async fn contract_handling<'a, CH>(mut contract_handler: CH) -> Result<(), ContractError> +// todo: remove result where CH: ContractHandler + Send + 'static, { @@ -85,30 +87,21 @@ where } } } - ContractHandlerEvent::PutQuery { - key: _key, - state: _state, - } => { - // let _put_result = contract_handler - // .handle_request(ClientRequest::Put { - // contract: todo!(), - // state: _state, - // }.into()) - // .await - // .map(|r| { - // let _r = r.unwrap_put(); - // unimplemented!(); - // }); - // contract_handler - // .channel() - // .send_to_listener( - // _id, - // ContractHandlerEvent::PushResponse { - // new_value: put_result, - // }, - // ) - // .await?; - todo!("perform put request"); + ContractHandlerEvent::PutQuery { key, state } => { + let put_result = contract_handler + .executor() + .upsert_contract_state(key, Either::Left(state)) + .await + .map_err(Into::into); + contract_handler + .channel() + .send_to_event_loop( + id, + ContractHandlerEvent::PutResponse { + new_value: put_result, + }, + ) + .await?; } _ => unreachable!(), } diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index a708ca866..e201cc444 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -237,6 +237,11 @@ pub(crate) trait ContractExecutor: Send + Sync + 'static { &mut self, contract: ContractContainer, ) -> Result<(), crate::runtime::ContractError>; + async fn upsert_contract_state( + &mut self, + key: ContractKey, + state: Either>, + ) -> Result; } /// A WASM executor which will run any contracts, delegates, etc. registered. @@ -338,8 +343,46 @@ impl Executor { } } -impl Executor { - pub async fn from_config(config: NodeConfig) -> Result { +impl Executor { + pub async fn new( + state_store: StateStore, + ctrl_handler: impl FnOnce(), + mode: OperationMode, + runtime: R, + ) -> Result { + ctrl_handler(); + + Ok(Self { + #[cfg(any( + all(feature = "local-mode", feature = "network-mode"), + all(not(feature = "local-mode"), not(feature = "network-mode")), + ))] + mode, + runtime, + state_store, + update_notifications: HashMap::default(), + subscriber_summaries: HashMap::default(), + delegate_attested_ids: HashMap::default(), + #[cfg(any( + not(feature = "local-mode"), + feature = "network-mode", + all(not(feature = "local-mode"), not(feature = "network-mode")) + ))] + event_loop_channel: None, + }) + } + + async fn get_stores( + config: &NodeConfig, + ) -> Result< + ( + ContractStore, + DelegateStore, + SecretsStore, + StateStore, + ), + DynError, + > { const MAX_SIZE: i64 = 10 * 1024 * 1024; const MAX_MEM_CACHE: u32 = 10_000_000; let static_conf = crate::config::Config::conf(); @@ -367,50 +410,26 @@ impl Executor { .unwrap_or_else(|| static_conf.secrets_dir()); let secret_store = SecretsStore::new(secrets_dir)?; + Ok((contract_store, delegate_store, secret_store, state_store)) + } +} + +impl Executor { + pub async fn from_config(config: NodeConfig) -> Result { + let (contract_store, delegate_store, secret_store, state_store) = + Self::get_stores(&config).await?; + let rt = Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(); Executor::new( - contract_store, - delegate_store, - secret_store, state_store, || { crate::util::set_cleanup_on_exit().unwrap(); }, OperationMode::Local, + rt, ) .await } - #[allow(unused_variables)] - pub async fn new( - contract_store: ContractStore, - delegate_store: DelegateStore, - secret_store: SecretsStore, - contract_state: StateStore, - ctrl_handler: impl FnOnce(), - mode: OperationMode, - ) -> Result { - ctrl_handler(); - - Ok(Self { - #[cfg(any( - all(feature = "local-mode", feature = "network-mode"), - all(not(feature = "local-mode"), not(feature = "network-mode")), - ))] - mode, - runtime: Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(), - state_store: contract_state, - update_notifications: HashMap::default(), - subscriber_summaries: HashMap::default(), - delegate_attested_ids: HashMap::default(), - #[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - event_loop_channel: None, - }) - } - pub fn register_contract_notifier( &mut self, key: ContractKey, @@ -1174,8 +1193,22 @@ impl Executor { #[cfg(test)] impl Executor { - pub async fn new_mock() -> Result { - todo!() + pub async fn new_mock(test: &str) -> Result { + let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{test}")); + + let contracts_data_dir = tmp_path.join("contracts"); + let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; + + let state_store = StateStore::new(Storage::new().await?, u16::MAX as u32).unwrap(); + + let executor = Executor::new( + state_store, + || {}, + OperationMode::Local, + super::MockRuntime { contract_store }, + ) + .await?; + Ok(executor) } pub async fn handle_request<'a>( @@ -1215,6 +1248,14 @@ impl ContractExecutor for Executor { ) -> Result<(), crate::runtime::ContractError> { self.runtime.contract_store.store_contract(contract) } + + async fn upsert_contract_state( + &mut self, + _key: ContractKey, + _state: Either>, + ) -> Result { + todo!() + } } #[cfg(test)] @@ -1254,12 +1295,28 @@ impl ContractExecutor for Executor { self.runtime.contract_store.store_contract(contract)?; Ok(()) } + + async fn upsert_contract_state( + &mut self, + _key: ContractKey, + state: Either>, + ) -> Result { + // todo: instead allow to perform mutations per contract based on incoming value so we can track + // state values over the network + match state { + Either::Left(state) => Ok(state), + Either::Right(delta) => Ok(WrappedState::from(delta.as_ref())), + } + } } #[cfg(test)] mod test { use super::*; - use crate::runtime::{ContractStore, StateStore}; + use crate::{ + contract::MockRuntime, + runtime::{ContractStore, StateStore}, + }; #[tokio::test(flavor = "multi_thread")] async fn local_node_handle() -> Result<(), Box> { @@ -1270,14 +1327,12 @@ mod test { let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE).unwrap(); let mut counter = 0; Executor::new( - contract_store, - DelegateStore::default(), - SecretsStore::default(), state_store, || { counter += 1; }, OperationMode::Local, + MockRuntime { contract_store }, ) .await .expect("local node with handle"); diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 311916723..eca64c1a0 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -141,19 +141,19 @@ impl ContractHandler for NetworkContractHandler { #[cfg(test)] impl ContractHandler for NetworkContractHandler { - type Builder = (); + type Builder = String; type ContractExecutor = Executor; fn build( channel: ContractHandlerToEventLoopChannel, _executor_request_sender: ExecutorToEventLoopChannel, - _builder: Self::Builder, + test: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { - async { - let executor = Executor::new_mock().await?; + async move { + let executor = Executor::new_mock(&test).await?; Ok(Self { executor, channel }) } .boxed() diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs index f390c7625..5374e65f2 100644 --- a/crates/core/src/contract/in_memory.rs +++ b/crates/core/src/contract/in_memory.rs @@ -15,7 +15,7 @@ use super::{ storages::in_memory::MemKVStore, Executor, }; -use crate::{config::Config, DynError}; +use crate::DynError; pub(crate) struct MockRuntime { pub contract_store: ContractStore, @@ -27,7 +27,7 @@ where { channel: ContractHandlerToEventLoopChannel, _kv_store: StateStore, - _runtime: MockRuntime, + runtime: Executor, } impl MemoryContractHandler @@ -35,40 +35,39 @@ where KVStore: StateStorage + Send + Sync + 'static, ::Error: Into>, { - const MAX_MEM_CACHE: i64 = 10_000_000; - - pub fn new( + pub async fn new( channel: ContractHandlerToEventLoopChannel, kv_store: KVStore, + data_dir: &str, ) -> Self { + // let rt = MockRuntime { + // contract_store: ContractStore::new( + // Config::conf().contracts_dir(), + // Self::MAX_MEM_CACHE, + // ) + // .unwrap(); MemoryContractHandler { channel, _kv_store: StateStore::new(kv_store, 10_000_000).unwrap(), - _runtime: MockRuntime { - contract_store: ContractStore::new( - Config::conf().contracts_dir(), - Self::MAX_MEM_CACHE, - ) - .unwrap(), - }, + runtime: Executor::new_mock(data_dir).await.unwrap(), } } } impl ContractHandler for MemoryContractHandler { - type Builder = (); + type Builder = String; type ContractExecutor = Executor; fn build( channel: ContractHandlerToEventLoopChannel, _executor_request_sender: ExecutorToEventLoopChannel, - _config: Self::Builder, + config: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { let store = MemKVStore::new(); - async move { Ok(MemoryContractHandler::new(channel, store)) }.boxed() + async move { Ok(MemoryContractHandler::new(channel, store, &config).await) }.boxed() } fn channel(&mut self) -> &mut ContractHandlerToEventLoopChannel { @@ -85,7 +84,7 @@ impl ContractHandler for MemoryContractHandler { } fn executor(&mut self) -> &mut Self::ContractExecutor { - todo!() + &mut self.runtime } } diff --git a/crates/core/src/contract/storages/rocks_db.rs b/crates/core/src/contract/storages/rocks_db.rs index 90850728f..69d5be411 100644 --- a/crates/core/src/contract/storages/rocks_db.rs +++ b/crates/core/src/contract/storages/rocks_db.rs @@ -109,10 +109,11 @@ mod test { }; // Prepare and get handler for rocksdb - async fn get_handler() -> Result, DynError> { + async fn get_handler(test: &str) -> Result, DynError> { let (_, ch_handler) = contract_handler_channel(); let (_, executor_sender) = executor_channel_test(); - let handler = NetworkContractHandler::build(ch_handler, executor_sender, ()).await?; + let handler = + NetworkContractHandler::build(ch_handler, executor_sender, test.to_string()).await?; Ok(handler) } @@ -120,7 +121,7 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn contract_handler() -> Result<(), DynError> { // Create a rocksdb handler and initialize the database - let mut handler = get_handler().await?; + let mut handler = get_handler("contract_handler").await?; // Generate a contract let contract_bytes = b"Test contract value".to_vec(); diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index 253181b5f..a51249ef8 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -152,10 +152,11 @@ mod test { }; // Prepare and get handler for an in-memory sqlite db - async fn get_handler() -> Result, DynError> { + async fn get_handler(test: &str) -> Result, DynError> { let (_, ch_handler) = contract_handler_channel(); let (_, executor_sender) = executor_channel_test(); - let handler = NetworkContractHandler::build(ch_handler, executor_sender, ()).await?; + let handler = + NetworkContractHandler::build(ch_handler, executor_sender, test.to_owned()).await?; Ok(handler) } @@ -163,7 +164,7 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn contract_handler() -> Result<(), DynError> { // Create a sqlite handler and initialize the database - let mut handler = get_handler().await?; + let mut handler = get_handler("contract_handler").await?; // Generate a contract let contract_bytes = b"test contract value".to_vec(); diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 02ddb6b48..5a1f0641a 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -37,5 +37,5 @@ pub mod dev_tool { pub use crate::config::Config; pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; pub use contract::{storages::Storage, Executor, OperationMode}; - pub use runtime::{ContractStore, DelegateStore, SecretsStore, StateStore}; + pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index aa71a80ec..909279c5f 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -629,51 +629,48 @@ where CM: ConnectionBridge + Send + Sync, { tracing::warn!("Failed tx `{}`, potentially attempting a retry", tx); - match tx.tx_type() { - TransactionType::JoinRing => { - const MSG: &str = "Fatal error: unable to connect to the network"; - // the attempt to join the network failed, this could be a fatal error since the node - // is useless without connecting to the network, we will retry with exponential backoff - match op_storage.pop(&tx) { - Some(OpEnum::JoinRing(op)) if op.has_backoff() => { - if let JoinRingOp { - backoff: Some(backoff), - gateway, - .. - } = *op - { - if cfg!(test) { - join_ring_request(None, peer_key, &gateway, op_storage, conn_manager) - .await?; - } else { - join_ring_request( - Some(backoff), - peer_key, - &gateway, - op_storage, - conn_manager, - ) + if let TransactionType::JoinRing = tx.tx_type() { + const MSG: &str = "Fatal error: unable to connect to the network"; + // the attempt to join the network failed, this could be a fatal error since the node + // is useless without connecting to the network, we will retry with exponential backoff + match op_storage.pop(&tx) { + Some(OpEnum::JoinRing(op)) if op.has_backoff() => { + if let JoinRingOp { + backoff: Some(backoff), + gateway, + .. + } = *op + { + if cfg!(test) { + join_ring_request(None, peer_key, &gateway, op_storage, conn_manager) .await?; - } - } - } - None | Some(OpEnum::JoinRing(_)) => { - let rand_gw = gateways - .shuffle() - .take(1) - .next() - .expect("at least one gateway"); - if !cfg!(test) { - tracing::error!("{}", MSG); } else { - tracing::debug!("{}", MSG); + join_ring_request( + Some(backoff), + peer_key, + &gateway, + op_storage, + conn_manager, + ) + .await?; } - join_ring_request(None, peer_key, rand_gw, op_storage, conn_manager).await?; } - _ => {} } + None | Some(OpEnum::JoinRing(_)) => { + let rand_gw = gateways + .shuffle() + .take(1) + .next() + .expect("at least one gateway"); + if !cfg!(test) { + tracing::error!("{}", MSG); + } else { + tracing::debug!("{}", MSG); + } + join_ring_request(None, peer_key, rand_gw, op_storage, conn_manager).await?; + } + _ => {} } - _ => unreachable!(), } Ok(()) } diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index a1fc3684d..256a8c913 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -211,7 +211,7 @@ mod test { NodeP2P::build::( config, event_log::TestEventListener::new(), - (), + "ping-listener".into(), ) .await?, ); @@ -228,7 +228,7 @@ mod test { let mut peer2 = NodeP2P::build::( config, event_log::TestEventListener::new(), - (), + "ping-dialer".into(), ) .await .unwrap(); diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index 86919eb44..d01e01fef 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -45,6 +45,7 @@ pub fn get_dynamic_port() -> u16 { /// A simulated in-memory network topology. pub(crate) struct SimNetwork { + name: String, pub labels: HashMap, pub event_listener: TestEventListener, user_ev_controller: Sender<(EventId, PeerKey)>, @@ -115,6 +116,7 @@ struct GatewayConfig { impl SimNetwork { pub async fn new( + name: &str, gateways: usize, nodes: usize, ring_max_htl: usize, @@ -125,6 +127,7 @@ impl SimNetwork { assert!(gateways > 0 && nodes > 0); let (user_ev_controller, receiver_ch) = channel((0, PeerKey::random())); let mut net = Self { + name: name.into(), event_listener: TestEventListener::new(), labels: HashMap::new(), user_ev_controller, @@ -196,7 +199,7 @@ impl SimNetwork { let gateway = NodeInMemory::build::( this_node, self.event_listener.clone(), - (), + format!("{}-{label}", self.name, label = this_config.label), ) .await .unwrap(); @@ -244,7 +247,7 @@ impl SimNetwork { let node = NodeInMemory::build::( config, self.event_listener.clone(), - (), + format!("{}-{label}", self.name), ) .await .unwrap(); diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index 53e953442..aefe865e3 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -9,6 +9,7 @@ use crate::{ message::{InnerMessage, Message, Transaction, TransactionType}, node::{ConnectionBridge, ConnectionError, OpManager, PeerKey}, ring::{Location, PeerKeyLocation, RingError}, + DynError, }; pub(crate) mod get; @@ -183,6 +184,8 @@ pub(crate) enum OpError { RingError(#[from] RingError), #[error(transparent)] ContractError(#[from] ContractError), + #[error(transparent)] + ExecutorError(DynError), #[error("unexpected operation state")] UnexpectedOpState, diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 33f8af4a4..fb553f746 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -293,8 +293,8 @@ impl Operation for GetOp { return_msg = None; new_state = None; } else if let ContractHandlerEvent::GetResponse { - response: value, key: returned_key, + response: value, } = op_storage .notify_contract_handler( ContractHandlerEvent::GetQuery { @@ -330,10 +330,17 @@ impl Operation for GetOp { Some(GetState::ReceivedRequest) => { tracing::debug!("Returning contract {} to {}", key, sender.peer); new_state = None; + let value = match value { + Ok(res) => res, + Err(err) => { + tracing::error!("error: {err}"); + return Err(OpError::ExecutorError(err)); + } + }; return_msg = Some(GetMsg::ReturnGet { id, key, - value: value.unwrap(), + value, sender: target, target: sender, }); @@ -842,7 +849,16 @@ mod test { let get_specs = HashMap::from_iter([("node-0".into(), node_0), ("gateway-0".into(), gw_0)]); // establish network - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; + let mut sim_nodes = SimNetwork::new( + "successful_get_op_between_nodes", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 2, + ) + .await; sim_nodes.build_with_specs(get_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; @@ -881,7 +897,8 @@ mod test { let get_specs = HashMap::from_iter([("node-1".into(), node_1)]); // establish network - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; + let mut sim_nodes = + SimNetwork::new("get_contract_not_found", NUM_GW, NUM_NODES, 3, 2, 4, 2).await; sim_nodes.build_with_specs(get_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; @@ -942,7 +959,16 @@ mod test { ]); // establish network - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 3).await; + let mut sim_nodes = SimNetwork::new( + "get_contract_found_after_retry", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 3, + ) + .await; sim_nodes.build_with_specs(get_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index 936751a87..261597409 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -1003,7 +1003,7 @@ mod test { #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn one_node_connects_to_gw() { - let mut sim_nodes = SimNetwork::new(1, 1, 1, 1, 2, 2).await; + let mut sim_nodes = SimNetwork::new("join_one_node_connects_to_gw", 1, 1, 1, 1, 2, 2).await; sim_nodes.build().await; tokio::time::sleep(Duration::from_secs(3)).await; assert!(sim_nodes.connected(&"node-0".into())); @@ -1015,7 +1015,16 @@ mod test { async fn forward_connection_to_node() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; + let mut sim_nodes = SimNetwork::new( + "join_forward_connection_to_node", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 2, + ) + .await; sim_nodes.build().await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await } @@ -1026,7 +1035,16 @@ mod test { async fn all_nodes_should_connect() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 1000, 2).await; + let mut sim_nodes = SimNetwork::new( + "join_all_nodes_should_connect", + NUM_GW, + NUM_NODES, + 3, + 2, + 1000, + 2, + ) + .await; sim_nodes.build().await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(10)).await } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 4a157ac2d..cf712d67b 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -916,7 +916,16 @@ mod test { let contract_val: WrappedState = gen.arbitrary()?; let new_value = WrappedState::new(Vec::from_iter(gen.arbitrary::<[u8; 20]>().unwrap())); - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; + let mut sim_nodes = SimNetwork::new( + "successful_put_op_between_nodes", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 2, + ) + .await; let mut locations = sim_nodes.get_locations_by_node(); let node0_loc = locations.remove(&"node-0".into()).unwrap(); let node1_loc = locations.remove(&"node-1".into()).unwrap(); diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 5b02883cb..14de3c74e 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -509,7 +509,16 @@ mod test { ("node-0".into(), first_node), ("node-1".into(), second_node), ]); - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; + let mut sim_nodes = SimNetwork::new( + "successful_subscribe_op_between_nodes", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 2, + ) + .await; sim_nodes.build_with_specs(subscribe_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index e33c2551c..702bfccb2 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -474,7 +474,8 @@ impl Eq for Location {} impl Ord for Location { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.partial_cmp(other) + self.0 + .partial_cmp(&other.0) .expect("always should return a cmp value") } } diff --git a/crates/core/src/runtime/mod.rs b/crates/core/src/runtime/mod.rs index 72a6a7a6e..590417d5d 100644 --- a/crates/core/src/runtime/mod.rs +++ b/crates/core/src/runtime/mod.rs @@ -21,4 +21,4 @@ pub(crate) use error::RuntimeResult; pub use secrets_store::SecretsStore; pub use state_store::StateStore; pub(crate) use state_store::{StateStorage, StateStoreError}; -pub(crate) use wasm_runtime::{ContractExecError, Runtime}; +pub use wasm_runtime::{ContractExecError, Runtime}; diff --git a/crates/fdev/src/commands.rs b/crates/fdev/src/commands.rs index a61a0c508..b323044b2 100644 --- a/crates/fdev/src/commands.rs +++ b/crates/fdev/src/commands.rs @@ -167,16 +167,10 @@ async fn execute_command( let contract_store = ContractStore::new(contracts_data_path, DEFAULT_MAX_CONTRACT_SIZE)?; let delegate_store = DelegateStore::new(delegates_data_path, DEFAULT_MAX_DELEGATE_SIZE)?; let secret_store = SecretsStore::new(secrets_data_path)?; - let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE).unwrap(); - let mut executor = Executor::new( - contract_store, - delegate_store, - secret_store, - state_store, - || {}, - OperationMode::Local, - ) - .await?; + let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE)?; + let rt = + freenet::dev_tool::Runtime::build(contract_store, delegate_store, secret_store, false)?; + let mut executor = Executor::new(state_store, || {}, OperationMode::Local, rt).await?; executor .handle_request(ClientId::FIRST, request, None) diff --git a/crates/fdev/src/local_node/state.rs b/crates/fdev/src/local_node/state.rs index 0ded9ce22..9a25ff836 100644 --- a/crates/fdev/src/local_node/state.rs +++ b/crates/fdev/src/local_node/state.rs @@ -24,17 +24,22 @@ impl AppState { let contract_dir = Config::conf().contracts_dir(); let contract_store = ContractStore::new(contract_dir, config.max_contract_size)?; let state_store = StateStore::new(Storage::new().await?, Self::MAX_MEM_CACHE).unwrap(); + let rt = freenet::dev_tool::Runtime::build( + contract_store, + DelegateStore::default(), + SecretsStore::default(), + false, + ) + .unwrap(); Ok(AppState { local_node: Arc::new(RwLock::new( Executor::new( - contract_store, - DelegateStore::default(), - SecretsStore::default(), state_store, || { freenet::util::set_cleanup_on_exit().unwrap(); }, OperationMode::Local, + rt, ) .await?, )), From fce539f1ba19cfc6c581b9ef6779f87b088082c8 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Mon, 16 Oct 2023 15:04:51 +0200 Subject: [PATCH 03/14] Get the `get` sim network tests to pass --- crates/core/src/bin/freenet.rs | 13 +--- crates/core/src/client_events.rs | 11 ++- crates/core/src/config.rs | 14 ++++ crates/core/src/contract.rs | 10 ++- crates/core/src/contract/executor.rs | 20 +++-- crates/core/src/contract/handler.rs | 51 ++++++++++++- crates/core/src/node.rs | 18 +++-- .../core/src/node/conn_manager/p2p_protoc.rs | 1 + crates/core/src/node/in_memory_impl.rs | 23 +++--- crates/core/src/node/tests.rs | 65 ++++++++++++----- crates/core/src/operations/get.rs | 12 +-- crates/core/src/operations/join_ring.rs | 73 ++++++++++--------- crates/core/src/operations/put.rs | 44 ++++++++--- crates/core/src/ring.rs | 1 - crates/core/src/runtime/delegate.rs | 2 +- crates/core/src/runtime/error.rs | 4 +- crates/core/src/runtime/state_store.rs | 11 +++ crates/core/src/runtime/tests/mod.rs | 2 +- crates/fdev/src/main.rs | 5 +- stdlib | 2 +- 20 files changed, 267 insertions(+), 115 deletions(-) diff --git a/crates/core/src/bin/freenet.rs b/crates/core/src/bin/freenet.rs index aef4f7f60..da78c34eb 100644 --- a/crates/core/src/bin/freenet.rs +++ b/crates/core/src/bin/freenet.rs @@ -1,8 +1,6 @@ use clap::Parser; use freenet::local_node::{Executor, NodeConfig, OperationMode}; use std::net::SocketAddr; -use tracing::metadata::LevelFilter; -use tracing_subscriber::EnvFilter; type DynError = Box; @@ -23,16 +21,7 @@ async fn run_local(config: NodeConfig) -> Result<(), DynError> { } fn main() -> Result<(), DynError> { - tracing_subscriber::fmt() - .with_level(true) - .with_file(true) - .with_line_number(true) - .with_env_filter( - EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(), - ) - .init(); + freenet::config::set_logger(); let config = NodeConfig::parse(); let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(4) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index a6aca2218..22137b264 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -99,6 +99,16 @@ pub struct OpenRequest<'a> { pub token: Option, } +impl Display for OpenRequest<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "client request {{ client: {}, req: {} }}", + &self.client_id, &*self.request + ) + } +} + impl<'a> OpenRequest<'a> { pub fn into_owned(self) -> OpenRequest<'static> { OpenRequest { @@ -289,7 +299,6 @@ pub(crate) mod test { return Ok(res.into_owned()); } } else { - tracing::debug!("sender half of user event gen dropped"); // probably the process finished, wait for a bit and then kill the thread tokio::time::sleep(Duration::from_secs(1)).await; panic!("finished orphan background thread"); diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 8a36a3b1e..f5cd1167a 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -323,6 +323,20 @@ impl libp2p::swarm::Executor for GlobalExecutor { } } +pub fn set_logger() { + tracing_subscriber::fmt() + .with_level(true) + .with_file(true) + .with_line_number(true) + .with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing_subscriber::filter::LevelFilter::DEBUG.into()) + .from_env_lossy() + .add_directive("stretto=off".parse().unwrap()), + ) + .init(); +} + pub(super) mod tracer { use super::*; diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index 419282147..f4198e5a7 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -23,6 +23,7 @@ pub use executor::{Executor, ExecutorError, OperationMode}; use executor::ContractExecutor; +#[tracing::instrument(skip_all)] pub(crate) async fn contract_handling<'a, CH>(mut contract_handler: CH) -> Result<(), ContractError> // todo: remove result where @@ -30,6 +31,7 @@ where { loop { let (id, event) = contract_handler.channel().recv_from_event_loop().await?; + tracing::debug!(%event, "got contract handling event"); match event { ContractHandlerEvent::GetQuery { key, @@ -87,10 +89,14 @@ where } } } - ContractHandlerEvent::PutQuery { key, state } => { + ContractHandlerEvent::PutQuery { + key, + state, + parameters, + } => { let put_result = contract_handler .executor() - .upsert_contract_state(key, Either::Left(state)) + .upsert_contract_state(key, Either::Left(state), parameters) .await .map_err(Into::into); contract_handler diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index e201cc444..e5fc49686 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -241,6 +241,7 @@ pub(crate) trait ContractExecutor: Send + Sync + 'static { &mut self, key: ContractKey, state: Either>, + params: Option>, ) -> Result; } @@ -1193,8 +1194,8 @@ impl Executor { #[cfg(test)] impl Executor { - pub async fn new_mock(test: &str) -> Result { - let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{test}")); + pub async fn new_mock(data_dir: &str) -> Result { + let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{data_dir}")); let contracts_data_dir = tmp_path.join("contracts"); let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; @@ -1253,6 +1254,7 @@ impl ContractExecutor for Executor { &mut self, _key: ContractKey, _state: Either>, + _params: Option>, ) -> Result { todo!() } @@ -1298,14 +1300,20 @@ impl ContractExecutor for Executor { async fn upsert_contract_state( &mut self, - _key: ContractKey, + key: ContractKey, state: Either>, + params: Option>, ) -> Result { // todo: instead allow to perform mutations per contract based on incoming value so we can track // state values over the network - match state { - Either::Left(state) => Ok(state), - Either::Right(delta) => Ok(WrappedState::from(delta.as_ref())), + match (state, params) { + (Either::Left(state), Some(params)) => { + self.state_store + .store(key, state.clone(), params.into_owned()) + .await?; + return Ok(state); + } + _ => todo!(), } } } diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index eca64c1a0..e4b94593d 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -147,13 +147,13 @@ impl ContractHandler for NetworkContractHandler { fn build( channel: ContractHandlerToEventLoopChannel, _executor_request_sender: ExecutorToEventLoopChannel, - test: Self::Builder, + data_dir: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { async move { - let executor = Executor::new_mock(&test).await?; + let executor = Executor::new_mock(&data_dir).await?; Ok(Self { executor, channel }) } .boxed() @@ -344,6 +344,7 @@ pub(crate) enum ContractHandlerEvent { PutQuery { key: ContractKey, state: WrappedState, + parameters: Option>, }, /// The response to a push query. PutResponse { @@ -365,6 +366,52 @@ pub(crate) enum ContractHandlerEvent { CacheResult(Result<(), ContractError>), } +impl std::fmt::Display for ContractHandlerEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ContractHandlerEvent::PutQuery { + key, + state, + parameters, + } => { + if let Some(params) = parameters { + write!(f, "put query {{ {key}, params: {:?} }}", params.as_ref()) + } else { + write!(f, "put query {{ {key} }}") + } + } + ContractHandlerEvent::PutResponse { new_value } => match new_value { + Ok(v) => { + write!(f, "put query response {{ {v} }}",) + } + Err(e) => { + write!(f, "put query failed {{ {e} }}",) + } + }, + ContractHandlerEvent::GetQuery { + key, + fetch_contract, + } => { + write!(f, "get query {{ {key}, fetch contract: {fetch_contract} }}",) + } + ContractHandlerEvent::GetResponse { key, response } => match response { + Ok(v) => { + write!(f, "get query response {{ {key} }}",) + } + Err(e) => { + write!(f, "get query failed {{ {key} }}",) + } + }, + ContractHandlerEvent::Cache(container) => { + write!(f, "caching {{ {} }}", container.key()) + } + ContractHandlerEvent::CacheResult(r) => { + write!(f, "caching result {{ {} }}", r.is_ok()) + } + } + } +} + impl ContractHandlerEvent { pub async fn into_network_op(self, op_manager: &OpManager) -> Transaction { todo!() diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 909279c5f..28c6b74c3 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -307,6 +307,7 @@ where } /// Process client events. +#[tracing::instrument(skip_all)] async fn client_event_handling( op_storage: Arc, mut client_events: ClientEv, @@ -318,7 +319,10 @@ async fn client_event_handling( tokio::select! { client_request = client_events.recv() => { let req = match client_request { - Ok(req) => req, + Ok(req) => { + tracing::debug!(%req, "got client request event"); + req + } Err(err) => { tracing::debug!(error = %err, "client error"); continue; @@ -332,6 +336,9 @@ async fn client_event_handling( } res = client_responses.recv() => { if let Some((cli_id, res)) = res { + if let Ok(res) = &res { + tracing::debug!(%res, "sending client response"); + } if let Err(err) = client_events.send(cli_id, res).await { tracing::error!("channel closed: {err}"); break; @@ -438,9 +445,9 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { tracing::debug!( - concat!("Handling ", $op, " get request @ {} (tx: {})"), + tx = %$id, + concat!("Handling ", $op, " get request @ {}"), $op_storage.ring.peer_key, - $id ); }; } @@ -504,11 +511,12 @@ async fn report_result( } Ok(None) => {} Err(err) => { - tracing::debug!("Finished tx w/ error: {}", err) + tracing::debug!("Finished transaction with error: {}", err) } } } +#[tracing::instrument(name = "process_network_message", skip_all)] async fn process_message( msg: Result, op_storage: Arc, @@ -628,7 +636,7 @@ async fn handle_cancelled_op( where CM: ConnectionBridge + Send + Sync, { - tracing::warn!("Failed tx `{}`, potentially attempting a retry", tx); + tracing::warn!(%tx, "Failed transaction, potentially attempting a retry"); if let TransactionType::JoinRing = tx.tx_type() { const MSG: &str = "Fatal error: unable to connect to the network"; // the attempt to join the network failed, this could be a fatal error since the node diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index bc43a77b6..cef6b86ee 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -232,6 +232,7 @@ impl P2pConnManager { Ok(()) } + #[tracing::instrument(name = "network_event_listener", skip_all)] pub async fn run_event_listener( mut self, op_manager: Arc, diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index 13a12491a..704ab10ea 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -15,7 +15,8 @@ use crate::{ config::GlobalExecutor, contract::{ self, executor_channel, ClientResponsesSender, ContractError, ContractHandler, - ContractHandlerEvent, ExecutorToEventLoopChannel, NetworkEventListenerHalve, + ContractHandlerEvent, ExecutorToEventLoopChannel, MemoryContractHandler, + NetworkEventListenerHalve, }, message::{Message, NodeEvent, TransactionType}, node::NodeBuilder, @@ -37,14 +38,11 @@ pub(super) struct NodeInMemory { impl NodeInMemory { /// Buils an in-memory node. Does nothing upon construction, - pub async fn build( + pub async fn build( builder: NodeBuilder<1>, event_listener: EL, - ch_builder: CH::Builder, - ) -> Result - where - CH: ContractHandler + Send + Sync + 'static, - { + ch_builder: String, + ) -> Result { let peer_key = PeerKey::from(builder.local_key.public()); let conn_manager = MemoryConnManager::new(peer_key); let gateways = builder.get_gateways()?; @@ -55,9 +53,10 @@ impl NodeInMemory { let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); let op_storage = Arc::new(OpManager::new(ring, notification_tx, ops_ch_channel)); let (_executor_listener, executor_sender) = executor_channel(op_storage.clone()); - let contract_handler = CH::build(ch_channel, executor_sender, ch_builder) - .await - .map_err(|e| anyhow::anyhow!(e))?; + let contract_handler = + MemoryContractHandler::build(ch_channel, executor_sender, ch_builder) + .await + .map_err(|e| anyhow::anyhow!(e))?; GlobalExecutor::spawn(contract::contract_handling(contract_handler)); @@ -106,7 +105,8 @@ impl NodeInMemory { contract_subscribers: HashMap>, ) -> Result<(), ContractError> { for (contract, state) in contracts { - let key = contract.key(); + let key: ContractKey = contract.key(); + let parameters = contract.params(); self.op_storage .notify_contract_handler(ContractHandlerEvent::Cache(contract.clone()), None) .await?; @@ -115,6 +115,7 @@ impl NodeInMemory { ContractHandlerEvent::PutQuery { key: key.clone(), state, + parameters: Some(parameters), }, None, ) diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index d01e01fef..c4ca5e028 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -17,7 +17,6 @@ use tracing::{info, instrument}; use crate::{ client_events::test::MemoryEventsGen, config::GlobalExecutor, - contract::MemoryContractHandler, node::{event_log::TestEventListener, InitPeerNode, NodeBuilder, NodeInMemory}, ring::{Distance, Location, PeerKeyLocation}, }; @@ -43,21 +42,6 @@ pub fn get_dynamic_port() -> u16 { rand::thread_rng().gen_range(FIRST_DYNAMIC_PORT..LAST_DYNAMIC_PORT) } -/// A simulated in-memory network topology. -pub(crate) struct SimNetwork { - name: String, - pub labels: HashMap, - pub event_listener: TestEventListener, - user_ev_controller: Sender<(EventId, PeerKey)>, - receiver_ch: Receiver<(EventId, PeerKey)>, - gateways: Vec<(NodeInMemory, GatewayConfig)>, - nodes: Vec<(NodeInMemory, NodeLabel)>, - ring_max_htl: usize, - rnd_if_htl_above: usize, - max_connections: usize, - min_connections: usize, -} - pub(crate) type EventId = usize; #[derive(PartialEq, Eq, Hash, Clone)] @@ -93,6 +77,16 @@ impl std::ops::Deref for NodeLabel { impl<'a> From<&'a str> for NodeLabel { fn from(value: &'a str) -> Self { + assert!(value.starts_with("gateway-") || value.starts_with("node-")); + let mut parts = value.split('-'); + assert!(parts.next().is_some()); + assert!(parts + .next() + .map(|s| s.parse::()) + .transpose() + .expect("should be an u16") + .is_some()); + assert!(parts.next().is_none()); Self(value.to_string().into()) } } @@ -114,6 +108,22 @@ struct GatewayConfig { location: Location, } +/// A simulated in-memory network topology. +pub(crate) struct SimNetwork { + name: String, + debug: bool, + pub labels: HashMap, + pub event_listener: TestEventListener, + user_ev_controller: Sender<(EventId, PeerKey)>, + receiver_ch: Receiver<(EventId, PeerKey)>, + gateways: Vec<(NodeInMemory, GatewayConfig)>, + nodes: Vec<(NodeInMemory, NodeLabel)>, + ring_max_htl: usize, + rnd_if_htl_above: usize, + max_connections: usize, + min_connections: usize, +} + impl SimNetwork { pub async fn new( name: &str, @@ -128,6 +138,7 @@ impl SimNetwork { let (user_ev_controller, receiver_ch) = channel((0, PeerKey::random())); let mut net = Self { name: name.into(), + debug: false, event_listener: TestEventListener::new(), labels: HashMap::new(), user_ev_controller, @@ -144,6 +155,11 @@ impl SimNetwork { net } + #[allow(unused)] + pub fn debug(&mut self) { + self.debug = true; + } + #[instrument(skip(self))] async fn build_gateways(&mut self, num: usize) { info!("Building {} gateways", num); @@ -196,7 +212,7 @@ impl SimNetwork { .listening_port(*port), ); } - let gateway = NodeInMemory::build::( + let gateway = NodeInMemory::build( this_node, self.event_listener.clone(), format!("{}-{label}", self.name, label = this_config.label), @@ -209,6 +225,7 @@ impl SimNetwork { #[instrument(skip(self))] async fn build_nodes(&mut self, num: usize) { + info!("Building {} regular nodes", num); let gateways: Vec<_> = self .gateways .iter() @@ -244,7 +261,7 @@ impl SimNetwork { self.event_listener .add_node(label.clone(), PeerKey::from(id)); - let node = NodeInMemory::build::( + let node = NodeInMemory::build( config, self.event_listener.clone(), format!("{}-{label}", self.name), @@ -387,6 +404,18 @@ impl SimNetwork { } } +impl Drop for SimNetwork { + fn drop(&mut self) { + if !self.debug { + for label in self.labels.keys() { + let p = std::env::temp_dir() + .join(format!("freenet-executor-{sim}-{label}", sim = self.name)); + let _ = std::fs::remove_dir_all(p); + } + } + } +} + fn group_locations_in_buckets( locs: impl IntoIterator, scale: i32, diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index fb553f746..30e0d8fc7 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -203,7 +203,7 @@ impl Operation for GetOp { self.state, Some(GetState::AwaitingResponse { .. }) )); - tracing::debug!("Seek contract {} @ {} (tx: {})", key, target.peer, id); + tracing::debug!(tx = %id, "Seek contract {} @ {}", key, target.peer); new_state = self.state; stats = Some(GetStats { contract_location: Location::from(&key), @@ -495,11 +495,13 @@ impl Operation for GetOp { } } + let parameters = contract.as_ref().map(|c| c.params()); op_storage .notify_contract_handler( ContractHandlerEvent::PutQuery { key: key.clone(), state: value.clone(), + parameters, }, client_id, ) @@ -683,9 +685,9 @@ pub(crate) async fn request_get( return Err(OpError::UnexpectedOpState); }; tracing::debug!( - "Preparing get contract request to {} (tx: {})", + tx = %id, + "Preparing get contract request to {}", target.peer, - id ); match get_op.state { @@ -866,12 +868,11 @@ mod test { sim_nodes .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(100))) .await?; - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(200)).await; assert!(sim_nodes.has_got_contract(&"node-0".into(), &key)); Ok(()) } - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn contract_not_found() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 2usize; @@ -910,7 +911,6 @@ mod test { Ok(()) } - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn contract_found_after_retry() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 2usize; diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index 261597409..03c5ba263 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -119,6 +119,7 @@ impl Operation for JoinRingOp { } => { // likely a gateway which accepts connections tracing::debug!( + tx = %id, "Initial join request received from {} with HTL {} @ {}", req_peer, hops_to_live, @@ -128,10 +129,10 @@ impl Operation for JoinRingOp { let new_location = Location::random(); // FIXME: don't try to forward to peers which have already been tried (add a rejected_by list) let accepted_by = if op_storage.ring.should_accept(&new_location) { - tracing::debug!("Accepting connection from {}", req_peer,); + tracing::debug!(tx = %id, "Accepting connection from {}", req_peer,); HashSet::from_iter([this_node_loc]) } else { - tracing::debug!("Rejecting connection from peer {}", req_peer); + tracing::debug!(tx = %id, "Rejecting connection from peer {}", req_peer); HashSet::new() }; @@ -151,9 +152,9 @@ impl Operation for JoinRingOp { .await? { tracing::debug!( - "Awaiting proxy response from @ {} (tx: {})", + tx = %id, + "Awaiting proxy response from @ {}", this_node_loc.peer, - id ); updated_state.add_new_proxy(accepted_by)?; // awaiting responses from proxies @@ -162,6 +163,7 @@ impl Operation for JoinRingOp { } else { if !accepted_by.is_empty() { tracing::debug!( + tx = %id, "OC received at gateway {} from requesting peer {}", this_node_loc.peer, req_peer @@ -196,6 +198,7 @@ impl Operation for JoinRingOp { } => { let own_loc = op_storage.ring.own_location(); tracing::debug!( + tx = %id, "Proxy join request received from {} to join new peer {} with HTL {} @ {}", sender.peer, joiner.peer, @@ -206,10 +209,11 @@ impl Operation for JoinRingOp { .ring .should_accept(&joiner.location.ok_or(ConnectionError::LocationUnknown)?) { - tracing::debug!("Accepting proxy connection from {}", joiner.peer); + tracing::debug!(tx = %id, "Accepting proxy connection from {}", joiner.peer); HashSet::from_iter([own_loc]) } else { tracing::debug!( + tx = %id, "Not accepting new proxy connection for sender {}", joiner.peer ); @@ -261,6 +265,7 @@ impl Operation for JoinRingOp { if match_target { new_state = Some(JRState::OCReceived); tracing::debug!( + tx = %id, "Sending response to join request with all the peers that accepted \ connection from gateway {} to peer {}", sender.peer, @@ -284,6 +289,7 @@ impl Operation for JoinRingOp { // connections alive and prune any dead connections new_state = Some(JRState::Connected); tracing::debug!( + tx = %id, "Sending response to join request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", sender.peer, @@ -319,7 +325,7 @@ impl Operation for JoinRingOp { }, .. } => { - tracing::debug!("Join response received from {}", sender.peer); + tracing::debug!(tx = %id, "Join response received from {}", sender.peer); // Set the given location let pk_loc = PeerKeyLocation { @@ -331,6 +337,7 @@ impl Operation for JoinRingOp { Some(JRState::Connecting(ConnectionInfo { gateway, .. })) => { if !accepted_by.clone().is_empty() { tracing::debug!( + tx = %id, "OC received and acknowledged at requesting peer {} from gateway {}", your_peer_id, gateway.peer @@ -372,17 +379,17 @@ impl Operation for JoinRingOp { target, msg: JoinResponse::Proxy { mut accepted_by }, } => { - tracing::debug!("Received proxy join at @ {}", target.peer); + tracing::debug!(tx = %id, "Received proxy join at @ {}", target.peer); match self.state { Some(JRState::Initializing) => { // the sender of the response is the target of the request and // is only a completed tx if it accepted the connection if accepted_by.contains(&sender) { tracing::debug!( - "Return to {}, connected at proxy {} (tx: {})", + tx = %id, + "Return to {}, connected at proxy {}", target.peer, sender.peer, - id ); new_state = Some(JRState::Connected); } else { @@ -423,6 +430,7 @@ impl Operation for JoinRingOp { if is_target_peer { tracing::debug!( + tx = %id, "Sending response to join request with all the peers that accepted \ connection from gateway {} to peer {}", target.peer, @@ -440,6 +448,7 @@ impl Operation for JoinRingOp { }); } else { tracing::debug!( + tx = %id, "Sending response to join request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", target.peer, @@ -472,7 +481,7 @@ impl Operation for JoinRingOp { } => { match self.state { Some(JRState::OCReceived) => { - tracing::debug!("Acknowledge connected at gateway"); + tracing::debug!(tx = %id, "Acknowledge connected at gateway"); new_state = Some(JRState::Connected); return_msg = Some(JoinRingMsg::Connected { id, @@ -491,7 +500,7 @@ impl Operation for JoinRingOp { sender.location.ok_or(ConnectionError::LocationUnknown)?, sender.peer, ); - tracing::debug!("Opened connection with peer {}", by_peer.peer); + tracing::debug!(tx = %id, "Opened connection with peer {}", by_peer.peer); new_state = None; } }; @@ -499,7 +508,7 @@ impl Operation for JoinRingOp { JoinRingMsg::Connected { target, sender, id } => { match self.state { Some(JRState::OCReceived) => { - tracing::debug!("Acknowledge connected at peer"); + tracing::debug!(tx = %id, "Acknowledge connected at peer {}", target.peer); new_state = Some(JRState::Connected); return_msg = None; } @@ -510,6 +519,7 @@ impl Operation for JoinRingOp { return Err(OpError::InvalidStateTransition(id)); } else { tracing::info!( + tx = %id, "Successfully completed connection @ {}, new location = {:?}", target.peer, op_storage.ring.own_location().location @@ -567,14 +577,14 @@ fn try_proxy_connection( ) -> (Option, Option) { let new_state = if accepted_by.contains(own_loc) { tracing::debug!( - "Return to {}, connected at proxy {} (tx: {})", + tx = %id, + "Return to {}, connected at proxy {}", sender.peer, own_loc.peer, - id ); Some(JRState::Connected) } else { - tracing::debug!("Failed to connect at proxy {}", sender.peer); + tracing::debug!(tx = %id, "Failed to connect at proxy {}", sender.peer); None }; let return_msg = Some(JoinRingMsg::Response { @@ -593,12 +603,13 @@ async fn propagate_oc_to_accepted_peers( other_peer: &PeerKeyLocation, msg: JoinRingMsg, ) -> Result<(), OpError> { + let id = msg.id(); if op_storage.ring.should_accept( &other_peer .location .ok_or(ConnectionError::LocationUnknown)?, ) { - tracing::info!("Established connection to {}", other_peer.peer); + tracing::info!(tx = %id, "Establishing connection to {}", other_peer.peer); conn_manager.add_connection(other_peer.peer).await?; op_storage.ring.add_connection( other_peer @@ -612,7 +623,7 @@ async fn propagate_oc_to_accepted_peers( let _ = conn_manager.send(&other_peer.peer, msg.into()).await; } } else { - tracing::debug!("Not accepting connection to {}", other_peer.peer); + tracing::debug!(tx = %id, "Not accepting connection to {}", other_peer.peer); } Ok(()) @@ -690,7 +701,7 @@ pub(crate) fn initial_request( max_hops_to_live: usize, id: Transaction, ) -> JoinRingOp { - tracing::debug!("Connecting to gw {} from {}", gateway.peer, this_peer); + tracing::debug!(tx = %id, "Connecting to gw {} from {}", gateway.peer, this_peer); let state = JRState::Connecting(ConnectionInfo { gateway, this_peer, @@ -733,10 +744,10 @@ where } = state.expect("infallible").try_unwrap_connecting()?; tracing::info!( - "Joining ring via {} (at {}) (tx: {})", + tx = %id, + "Joining ring via {} (at {})", gateway.peer, gateway.location.ok_or(ConnectionError::LocationUnknown)?, - tx ); conn_manager.add_connection(gateway.peer).await?; @@ -786,12 +797,14 @@ where let forward_to = if left_htl >= ring.rnd_if_htl_above { tracing::debug!( + tx = %id, "Randomly selecting peer to forward JoinRequest (requester: {})", req_peer.peer ); ring.random_peer(|p| p.peer != req_peer.peer) } else { tracing::debug!( + tx = %id, "Selecting close peer to forward request (requester: {})", req_peer.peer ); @@ -809,6 +822,7 @@ where }, }); tracing::debug!( + tx = %id, "Forwarding JoinRequest from sender {} to {}", req_peer.peer, forward_to.peer @@ -825,11 +839,11 @@ where } else { if num_accepted != 0 { tracing::warn!( - "Unable to forward, will only be connected to one peer (tx: {})", - id + tx = %id, + "Unable to forward, will only be connected to one peer", ); } else { - tracing::warn!("Unable to forward or accept any connections (tx: {})", id); + tracing::warn!(tx = %id, "Unable to forward or accept any connections"); } Ok(None) } @@ -946,15 +960,6 @@ mod messages { } } - /* - - Peer A ---> Peer B (forward) ----> Peer C - |----- (forward) ---------> Peer D - - - Peer A ---> Peer B (forward) ----> Peer C ----> Peer D - - */ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub(crate) enum JoinRequest { StartReq { @@ -1000,7 +1005,6 @@ mod test { use crate::node::tests::{check_connectivity, SimNetwork}; /// Given a network of one node and one gateway test that both are connected. - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn one_node_connects_to_gw() { let mut sim_nodes = SimNetwork::new("join_one_node_connects_to_gw", 1, 1, 1, 1, 2, 2).await; @@ -1013,6 +1017,7 @@ mod test { #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn forward_connection_to_node() -> Result<(), anyhow::Error> { + // crate::config::set_logger(); const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; let mut sim_nodes = SimNetwork::new( @@ -1030,8 +1035,8 @@ mod test { } /// Given a network of N peers all nodes should have connections. - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn all_nodes_should_connect() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index cf712d67b..14836a177 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -245,7 +245,9 @@ impl Operation for PutOp { // after the contract has been cached, push the update query tracing::debug!("Attempting contract value update"); - let new_value = put_contract(op_storage, key.clone(), value, client_id).await?; + let parameters = contract.params(); + let new_value = + put_contract(op_storage, key.clone(), value, parameters, client_id).await?; tracing::debug!("Contract successfully updated"); // if the change was successful, communicate this back to the requestor and broadcast the change conn_manager @@ -291,7 +293,7 @@ impl Operation for PutOp { self.state, broadcast_to, key.clone(), - new_value, + (contract.params(), new_value), self._ttl, ) .await @@ -307,14 +309,21 @@ impl Operation for PutOp { id, key, new_value, + parameters, sender, sender_subscribers, } => { let target = op_storage.ring.own_location(); tracing::debug!("Attempting contract value update"); - let new_value = - put_contract(op_storage, key.clone(), new_value, client_id).await?; + let new_value = put_contract( + op_storage, + key.clone(), + new_value, + parameters.clone(), + client_id, + ) + .await?; tracing::debug!("Contract successfully updated"); let broadcast_to = op_storage @@ -342,7 +351,7 @@ impl Operation for PutOp { self.state, broadcast_to, key, - new_value, + (parameters, new_value), self._ttl, ) .await @@ -360,6 +369,7 @@ impl Operation for PutOp { mut broadcasted_to, key, new_value, + parameters, } => { let sender = op_storage.ring.own_location(); let msg = PutMsg::BroadcastTo { @@ -368,6 +378,7 @@ impl Operation for PutOp { new_value: new_value.clone(), sender, sender_subscribers: broadcast_to.clone(), + parameters, }; let mut broadcasting = Vec::with_capacity(broadcast_to.len()); @@ -457,7 +468,9 @@ impl Operation for PutOp { }); } // after the contract has been cached, push the update query - let new_value = put_contract(op_storage, key, new_value, client_id).await?; + let new_value = + put_contract(op_storage, key, new_value, contract.params(), client_id) + .await?; //update skip list skip_list.push(peer_loc.peer); @@ -533,7 +546,7 @@ async fn try_to_broadcast( state: Option, broadcast_to: Vec, key: ContractKey, - new_value: WrappedState, + (parameters, new_value): (Parameters<'static>, WrappedState), ttl: Duration, ) -> Result<(Option, Option), OpError> { let new_state; @@ -556,6 +569,7 @@ async fn try_to_broadcast( return_msg = Some(PutMsg::Broadcasting { id, new_value, + parameters, broadcasted_to: 0, broadcast_to, key, @@ -699,11 +713,19 @@ async fn put_contract( op_storage: &OpManager, key: ContractKey, state: WrappedState, + parameters: Parameters<'static>, client_id: Option, ) -> Result { // after the contract has been cached, push the update query match op_storage - .notify_contract_handler(ContractHandlerEvent::PutQuery { key, state }, client_id) + .notify_contract_handler( + ContractHandlerEvent::PutQuery { + key, + state, + parameters: Some(parameters), + }, + client_id, + ) .await { Ok(ContractHandlerEvent::PutResponse { @@ -771,7 +793,7 @@ mod messages { use crate::message::InnerMessage; use serde::{Deserialize, Serialize}; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) enum PutMsg { /// Initialize the put operation by routing the value RouteValue { @@ -824,6 +846,8 @@ mod messages { broadcast_to: Vec, key: ContractKey, new_value: WrappedState, + #[serde(deserialize_with = "Parameters::deser_params")] + parameters: Parameters<'static>, }, /// Broadcasting a change to a peer, which then will relay the changes to other peers. BroadcastTo { @@ -831,6 +855,8 @@ mod messages { sender: PeerKeyLocation, key: ContractKey, new_value: WrappedState, + #[serde(deserialize_with = "Parameters::deser_params")] + parameters: Parameters<'static>, sender_subscribers: Vec, }, } diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index 702bfccb2..abbf146be 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -232,7 +232,6 @@ impl Ring { /// Returns this node location in the ring, if any (must have join the ring already). pub fn own_location(&self) -> PeerKeyLocation { - tracing::debug!("Getting loc for peer {}", self.peer_key); let location = f64::from_le_bytes(self.own_location.load(SeqCst).to_le_bytes()); let location = if (location - -1f64).abs() < f64::EPSILON { None diff --git a/crates/core/src/runtime/delegate.rs b/crates/core/src/runtime/delegate.rs index 368823ccd..6c22407a2 100644 --- a/crates/core/src/runtime/delegate.rs +++ b/crates/core/src/runtime/delegate.rs @@ -417,7 +417,7 @@ mod test { name: &str, ) -> Result<(DelegateContainer, Runtime), Box> { const TEST_PREFIX: &str = "delegate-api"; - let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + // let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); let contracts_dir = super::super::tests::test_dir(TEST_PREFIX); let delegates_dir = super::super::tests::test_dir(TEST_PREFIX); let secrets_dir = super::super::tests::test_dir(TEST_PREFIX); diff --git a/crates/core/src/runtime/error.rs b/crates/core/src/runtime/error.rs index d45cd86e2..a18b84064 100644 --- a/crates/core/src/runtime/error.rs +++ b/crates/core/src/runtime/error.rs @@ -2,6 +2,8 @@ use std::fmt::Display; use freenet_stdlib::prelude::{ContractKey, DelegateKey, SecretsId}; +use crate::DynError; + use super::{delegate, secrets_store, wasm_runtime, DelegateExecError}; pub type RuntimeResult = std::result::Result; @@ -97,7 +99,7 @@ impl_err!(wasmer::RuntimeError); #[derive(thiserror::Error, Debug)] pub(crate) enum RuntimeInnerError { #[error(transparent)] - Any(#[from] Box), + Any(#[from] DynError), #[error(transparent)] BufferError(#[from] freenet_stdlib::memory::buf::Error), diff --git a/crates/core/src/runtime/state_store.rs b/crates/core/src/runtime/state_store.rs index ea507f249..334deab2f 100644 --- a/crates/core/src/runtime/state_store.rs +++ b/crates/core/src/runtime/state_store.rs @@ -11,6 +11,17 @@ pub enum StateStoreError { MissingContract(ContractKey), } +impl From for crate::runtime::ContractError { + fn from(value: StateStoreError) -> Self { + match value { + StateStoreError::Any(err) => crate::runtime::ContractError::from(err), + err @ StateStoreError::MissingContract(_) => { + crate::runtime::ContractError::from(Into::::into(format!("{err}"))) + } + } + } +} + #[async_trait::async_trait] #[allow(clippy::type_complexity)] pub trait StateStorage { diff --git a/crates/core/src/runtime/tests/mod.rs b/crates/core/src/runtime/tests/mod.rs index 0cb8bfacf..32d0bf913 100644 --- a/crates/core/src/runtime/tests/mod.rs +++ b/crates/core/src/runtime/tests/mod.rs @@ -62,7 +62,7 @@ pub(crate) fn get_test_module(name: &str) -> Result, Box Result<(ContractStore, ContractKey), Box> { - let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + // let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); let mut store = ContractStore::new(test_dir("contract"), 10_000)?; let contract_bytes = WrappedContract::new( Arc::new(ContractCode::from(get_test_module(name)?)), diff --git a/crates/fdev/src/main.rs b/crates/fdev/src/main.rs index 9f290c0f7..7c9eeb6b6 100644 --- a/crates/fdev/src/main.rs +++ b/crates/fdev/src/main.rs @@ -2,7 +2,6 @@ use std::borrow::Cow; use clap::Parser; use freenet_stdlib::client_api::ClientRequest; -use tracing_subscriber::EnvFilter; mod build; mod commands; @@ -35,9 +34,7 @@ enum Error { #[tokio::main] async fn main() -> Result<(), anyhow::Error> { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .init(); + freenet::config::set_logger(); let cwd = std::env::current_dir()?; let config = Config::parse(); freenet::config::Config::set_op_mode(config.additional.mode); diff --git a/stdlib b/stdlib index 0cd6ba8c9..3d21697c2 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit 0cd6ba8c9a51ae03d24deef8ce35a88241de57f4 +Subproject commit 3d21697c2393f7b563cf2242237c6fbb6ec08f44 From 39ece9b1d8ad570235be8a51263d0ef97aa64ba2 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Tue, 17 Oct 2023 12:32:09 +0200 Subject: [PATCH 04/14] Multiple fixes around join ring op retries --- Cargo.lock | 257 ++++++++++++++---- crates/core/Cargo.toml | 3 +- crates/core/src/contract.rs | 1 - crates/core/src/contract/executor.rs | 4 +- crates/core/src/contract/handler.rs | 58 ++-- crates/core/src/contract/in_memory.rs | 1 - crates/core/src/contract/storages/rocks_db.rs | 92 ------- crates/core/src/contract/storages/sqlite.rs | 93 ------- crates/core/src/lib.rs | 1 - crates/core/src/message.rs | 41 ++- crates/core/src/node.rs | 75 ++--- .../core/src/node/conn_manager/p2p_protoc.rs | 1 - crates/core/src/node/event_log.rs | 5 +- crates/core/src/node/in_memory_impl.rs | 23 +- crates/core/src/node/op_state.rs | 3 +- crates/core/src/node/p2p_impl.rs | 1 - crates/core/src/node/tests.rs | 38 +-- crates/core/src/operations.rs | 4 +- crates/core/src/operations/get.rs | 14 +- crates/core/src/operations/join_ring.rs | 147 ++++++---- crates/core/src/operations/put.rs | 11 +- crates/core/src/operations/subscribe.rs | 10 +- crates/core/src/ring.rs | 91 ++----- crates/core/src/runtime/contract_store.rs | 2 +- crates/core/src/runtime/wasm_runtime.rs | 1 - crates/core/src/server/mod.rs | 2 +- crates/fdev/src/build.rs | 1 + crates/fdev/src/new_package.rs | 2 +- stdlib | 2 +- 29 files changed, 453 insertions(+), 531 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69f0e71db..15bb518e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,7 +250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ "async-lock", - "autocfg", + "autocfg 1.1.0", "cfg-if", "concurrent-queue", "futures-lite", @@ -305,12 +305,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" - [[package]] name = "atomic" version = "0.6.0" @@ -337,6 +331,15 @@ dependencies = [ "url", ] +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -754,6 +757,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -841,7 +853,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" dependencies = [ - "autocfg", + "autocfg 1.1.0", "cfg-if", "libc", "scopeguard", @@ -996,7 +1008,7 @@ version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ - "autocfg", + "autocfg 1.1.0", "cfg-if", "crossbeam-utils", "memoffset 0.9.0", @@ -1029,7 +1041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -1332,7 +1344,7 @@ checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ "curve25519-dalek", "ed25519", - "rand_core", + "rand_core 0.6.4", "serde", "sha2", "zeroize", @@ -1592,7 +1604,7 @@ dependencies = [ "parking_lot", "pav_regression", "pico-args", - "rand", + "rand 0.8.5", "rocksdb", "serde", "serde_json", @@ -1606,8 +1618,8 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "ulid", "unsigned-varint", - "uuid", "wasmer", "xz2", ] @@ -1637,7 +1649,7 @@ dependencies = [ "futures", "js-sys", "once_cell", - "rand", + "rand 0.8.5", "semver", "serde", "serde-wasm-bindgen 0.6.0", @@ -1663,6 +1675,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "2.0.0" @@ -2162,7 +2180,7 @@ dependencies = [ "http", "hyper", "log", - "rand", + "rand 0.8.5", "tokio", "url", "xmltree", @@ -2174,7 +2192,7 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "autocfg", + "autocfg 1.1.0", "hashbrown 0.12.3", "serde", ] @@ -2457,7 +2475,7 @@ dependencies = [ "libp2p-swarm", "log", "quick-protobuf", - "rand", + "rand 0.8.5", ] [[package]] @@ -2492,7 +2510,7 @@ dependencies = [ "parking_lot", "pin-project", "quick-protobuf", - "rand", + "rand 0.8.5", "rw-stream-sink", "smallvec", "thiserror", @@ -2551,7 +2569,7 @@ dependencies = [ "log", "multihash", "quick-protobuf", - "rand", + "rand 0.8.5", "sha2", "thiserror", "zeroize", @@ -2570,7 +2588,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand", + "rand 0.8.5", "smallvec", "socket2 0.5.4", "tokio", @@ -2610,7 +2628,7 @@ dependencies = [ "multihash", "once_cell", "quick-protobuf", - "rand", + "rand 0.8.5", "sha2", "snow", "static_assertions", @@ -2633,7 +2651,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand", + "rand 0.8.5", "void", ] @@ -2653,7 +2671,7 @@ dependencies = [ "log", "parking_lot", "quinn", - "rand", + "rand 0.8.5", "ring", "rustls", "socket2 0.5.4", @@ -2674,7 +2692,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand", + "rand 0.8.5", "smallvec", "void", ] @@ -2696,7 +2714,7 @@ dependencies = [ "log", "multistream-select", "once_cell", - "rand", + "rand 0.8.5", "smallvec", "tokio", "void", @@ -2841,7 +2859,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ - "autocfg", + "autocfg 1.1.0", "scopeguard", ] @@ -2956,7 +2974,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -2965,7 +2983,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -3210,7 +3228,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-integer", "num-traits", ] @@ -3227,7 +3245,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -3238,7 +3256,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-traits", ] @@ -3248,7 +3266,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-integer", "num-traits", ] @@ -3259,7 +3277,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ - "autocfg", + "autocfg 1.1.0", "libm", ] @@ -3401,7 +3419,7 @@ dependencies = [ "opentelemetry_api", "ordered-float 3.9.2", "percent-encoding", - "rand", + "rand 0.8.5", "regex", "thiserror", "tokio", @@ -3651,7 +3669,7 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ - "autocfg", + "autocfg 1.1.0", "bitflags 1.3.2", "cfg-if", "concurrent-queue", @@ -3857,7 +3875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c78e758510582acc40acb90458401172d41f1016f8c9dde89e49677afb7eec1" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring", "rustc-hash", "rustls", @@ -3895,6 +3913,25 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -3902,8 +3939,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", ] [[package]] @@ -3913,9 +3960,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -3925,6 +3987,69 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rayon" version = "1.8.0" @@ -3957,6 +4082,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -4162,7 +4296,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", @@ -4535,7 +4669,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4550,7 +4684,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -4586,7 +4720,7 @@ dependencies = [ "blake2", "chacha20poly1305 0.9.1", "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "ring", "rustc_version", "sha2", @@ -4772,7 +4906,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.5", "rsa", "serde", "sha1", @@ -4811,7 +4945,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha1", @@ -4866,12 +5000,12 @@ checksum = "51481a2abc8af47b7447b39b221ce806728e1dcbbb1354b0d5bacb1e5e1f72de" dependencies = [ "async-channel", "async-io", - "atomic 0.6.0", + "atomic", "crossbeam-channel", "futures", "getrandom", "parking_lot", - "rand", + "rand 0.8.5", "seahash", "thiserror", "tracing", @@ -5363,7 +5497,7 @@ dependencies = [ "idna 0.2.3", "ipnet", "lazy_static", - "rand", + "rand 0.8.5", "smallvec", "socket2 0.4.9", "thiserror", @@ -5389,7 +5523,7 @@ dependencies = [ "idna 0.4.0", "ipnet", "once_cell", - "rand", + "rand 0.8.5", "smallvec", "thiserror", "tinyvec", @@ -5410,7 +5544,7 @@ dependencies = [ "lru-cache", "once_cell", "parking_lot", - "rand", + "rand 0.8.5", "resolv-conf", "smallvec", "thiserror", @@ -5437,7 +5571,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -5456,6 +5590,18 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +[[package]] +name = "ulid" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e95a59b292ca0cf9b45be2e52294d1ca6cb24eb11b08ef4376f73f1a00c549" +dependencies = [ + "chrono", + "lazy_static", + "rand 0.6.5", + "serde", +] + [[package]] name = "unicase" version = "2.7.0" @@ -5580,11 +5726,6 @@ name = "uuid" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" -dependencies = [ - "atomic 0.5.3", - "getrandom", - "serde", -] [[package]] name = "valuable" @@ -6125,7 +6266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "serde", "zeroize", ] @@ -6206,7 +6347,7 @@ dependencies = [ "nohash-hasher", "parking_lot", "pin-project", - "rand", + "rand 0.8.5", "static_assertions", ] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0a61fc330..61560a371 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -59,7 +59,8 @@ thiserror = "1" tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "fs"] } tower-http = { version = "0.4", features = ["trace", "fs"] } unsigned-varint = "0.7" -uuid = { version = "1", features = ["serde", "v4", "v1"] } +# uuid = { version = "1", features = ["serde", "v4", "v1"] } +ulid = { version = "0.4", features = ["serde"] } sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio-rustls"], optional = true } # TODO(kakoc): clang should be installed for rocksdb; write about that in prerequisites/dev guide rocksdb = { version = "0.21.0", default-features = false, optional = true } diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index f4198e5a7..28bf08907 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -25,7 +25,6 @@ use executor::ContractExecutor; #[tracing::instrument(skip_all)] pub(crate) async fn contract_handling<'a, CH>(mut contract_handler: CH) -> Result<(), ContractError> -// todo: remove result where CH: ContractHandler + Send + 'static, { diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index e5fc49686..c37c5c5f8 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -197,8 +197,8 @@ struct GetContract { #[async_trait::async_trait] impl ComposeNetworkMessage for GetContract { - fn initiate_op(self, op_manager: &OpManager) -> operations::get::GetOp { - operations::get::start_op(self.key, self.fetch_contract, &op_manager.ring.peer_key) + fn initiate_op(self, _op_manager: &OpManager) -> operations::get::GetOp { + operations::get::start_op(self.key, self.fetch_contract) } async fn resume_op( diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index e4b94593d..c2dcc92f7 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -289,7 +289,6 @@ impl ContractHandlerToEventLoopChannel { } } - // todo: use pub async fn recv_from_handler(&mut self) -> (EventId, ContractHandlerEvent) { todo!() } @@ -424,55 +423,56 @@ pub mod test { use crate::runtime::ContractStore; use freenet_stdlib::{ - client_api::{ClientRequest, HostResponse}, + client_api::{ClientRequest, ContractRequest, HostResponse}, prelude::*, }; use super::*; use crate::{config::GlobalExecutor, contract::MockRuntime}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn channel_test() -> Result<(), anyhow::Error> { let (mut send_halve, mut rcv_halve) = contract_handler_channel(); + let contract = ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( + Arc::new(ContractCode::from(vec![0, 1, 2, 3])), + Parameters::from(vec![4, 5]), + ))); + + let contract_cp = contract.clone(); let h = GlobalExecutor::spawn(async move { - let contract = - ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(vec![0, 1, 2, 3])), - Parameters::from(vec![]), - ))); send_halve - .send_to_handler(ContractHandlerEvent::Cache(contract), None) + .send_to_handler(ContractHandlerEvent::Cache(contract_cp), None) .await }); - let (id, ev) = tokio::time::timeout(Duration::from_millis(100), rcv_halve.recv_from_event_loop()) .await??; - if let ContractHandlerEvent::Cache(contract) = ev { - let data: Vec = contract.data(); - assert_eq!(data, vec![0, 1, 2, 3]); - let contract = ContractContainer::Wasm(ContractWasmAPIVersion::V1( - WrappedContract::new(Arc::new(ContractCode::from(data)), Parameters::from(vec![])), - )); - tokio::time::timeout( - Duration::from_millis(100), - rcv_halve.send_to_event_loop(id, ContractHandlerEvent::Cache(contract)), - ) - .await??; - } else { + let ContractHandlerEvent::Cache(contract) = ev else { anyhow::bail!("invalid event"); - } - - if let ContractHandlerEvent::Cache(contract) = h.await?? { - let data: Vec = contract.data(); - assert_eq!(data, vec![0, 1, 2, 3]); - } else { + }; + assert_eq!(contract.data(), vec![0, 1, 2, 3]); + + tokio::time::timeout( + Duration::from_millis(100), + rcv_halve.send_to_event_loop(id, ContractHandlerEvent::Cache(contract)), + ) + .await??; + let ContractHandlerEvent::Cache(contract) = h.await?? else { anyhow::bail!("invalid event!"); - } + }; + assert_eq!(contract.data(), vec![0, 1, 2, 3]); Ok(()) } + + // Prepare and get handler for an in-memory sqlite db + async fn get_handler(test: &str) -> Result, DynError> { + let (_, ch_handler) = contract_handler_channel(); + let (_, executor_sender) = super::super::executor::executor_channel_test(); + let handler = + NetworkContractHandler::build(ch_handler, executor_sender, test.to_owned()).await?; + Ok(handler) + } } diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs index 5374e65f2..8fa030941 100644 --- a/crates/core/src/contract/in_memory.rs +++ b/crates/core/src/contract/in_memory.rs @@ -88,7 +88,6 @@ impl ContractHandler for MemoryContractHandler { } } -#[ignore] #[test] fn serialization() -> Result<(), anyhow::Error> { let bytes = crate::util::test::random_bytes_1024(); diff --git a/crates/core/src/contract/storages/rocks_db.rs b/crates/core/src/contract/storages/rocks_db.rs index 69d5be411..5c77e20d6 100644 --- a/crates/core/src/contract/storages/rocks_db.rs +++ b/crates/core/src/contract/storages/rocks_db.rs @@ -92,95 +92,3 @@ impl StateStorage for RocksDb { } } } - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use freenet_stdlib::{client_api::ContractRequest, prelude::*}; - - use crate::{ - client_events::ClientId, - contract::{ - contract_handler_channel, executor::executor_channel_test, ContractHandler, - MockRuntime, NetworkContractHandler, - }, - DynError, - }; - - // Prepare and get handler for rocksdb - async fn get_handler(test: &str) -> Result, DynError> { - let (_, ch_handler) = contract_handler_channel(); - let (_, executor_sender) = executor_channel_test(); - let handler = - NetworkContractHandler::build(ch_handler, executor_sender, test.to_string()).await?; - Ok(handler) - } - - #[ignore] - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn contract_handler() -> Result<(), DynError> { - // Create a rocksdb handler and initialize the database - let mut handler = get_handler("contract_handler").await?; - - // Generate a contract - let contract_bytes = b"Test contract value".to_vec(); - let contract: ContractContainer = - ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(contract_bytes.clone())), - Parameters::from(vec![]), - ))); - - // Get contract parts - let state = WrappedState::new(contract_bytes.clone()); - handler - .handle_request( - ContractRequest::Put { - contract: contract.clone(), - state: state.clone(), - related_contracts: Default::default(), - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_put(); - let (get_result_value, _) = handler - .handle_request( - ContractRequest::Get { - key: contract.key().clone(), - fetch_contract: false, - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_get(); - assert_eq!(state, get_result_value); - - // Update the contract state with a new delta - let delta = StateDelta::from(b"New test contract value".to_vec()); - handler - .handle_request( - ContractRequest::Update { - key: contract.key().clone(), - data: delta.into(), - } - .into(), - ClientId::FIRST, - None, - ) - .await?; - // let (new_get_result_value, _) = handler - // .handle_request(ContractOps::Get { - // key: *contract.key(), - // contract: false, - // }) - // .await? - // .unwrap_summary(); - // assert_eq!(delta, new_get_result_value); - todo!("get summary and compare with delta"); - } -} diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index a51249ef8..e70f3bc71 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -134,96 +134,3 @@ pub enum SqlDbError { #[error(transparent)] StateStore(#[from] StateStoreError), } - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use freenet_stdlib::client_api::ContractRequest; - use freenet_stdlib::prelude::*; - - use crate::{ - client_events::ClientId, - contract::{ - contract_handler_channel, executor::executor_channel_test, ContractHandler, - MockRuntime, NetworkContractHandler, - }, - DynError, - }; - - // Prepare and get handler for an in-memory sqlite db - async fn get_handler(test: &str) -> Result, DynError> { - let (_, ch_handler) = contract_handler_channel(); - let (_, executor_sender) = executor_channel_test(); - let handler = - NetworkContractHandler::build(ch_handler, executor_sender, test.to_owned()).await?; - Ok(handler) - } - - #[ignore] - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn contract_handler() -> Result<(), DynError> { - // Create a sqlite handler and initialize the database - let mut handler = get_handler("contract_handler").await?; - - // Generate a contract - let contract_bytes = b"test contract value".to_vec(); - let contract: ContractContainer = - ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(contract_bytes.clone())), - Parameters::from(vec![]), - ))); - - // Get contract parts - let state = WrappedState::new(contract_bytes.clone()); - handler - .handle_request( - ContractRequest::Put { - contract: contract.clone(), - state: state.clone(), - related_contracts: Default::default(), - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_put(); - let (get_result_value, _) = handler - .handle_request( - ContractRequest::Get { - key: contract.key().clone(), - fetch_contract: false, - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_get(); - assert_eq!(state, get_result_value); - - // Update the contract state with a new delta - let delta = StateDelta::from(b"New test contract value".to_vec()); - handler - .handle_request( - ContractRequest::Update { - key: contract.key().clone(), - data: delta.into(), - } - .into(), - ClientId::FIRST, - None, - ) - .await?; - // let (new_get_result_value, _) = handler - // .handle_request(ContractOps::Get { - // key: *contract.key(), - // contract: false, - // }) - // .await? - // .unwrap_summary(); - // assert_eq!(delta, new_get_result_value); - todo!("get summary and compare with delta"); - } -} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 5a1f0641a..e26ed2240 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -24,7 +24,6 @@ pub mod local_node { /// Exports to build a running network simulation. pub mod network_sim { - // todo: streamline this use super::*; pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; pub use node::{InitPeerNode, NodeBuilder, NodeConfig}; diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 4f4bd3b7a..551339abb 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -3,10 +3,7 @@ use std::{fmt::Display, time::Duration}; use serde::{Deserialize, Serialize}; -use uuid::{ - v1::{Context, Timestamp}, - Uuid, -}; +use ulid::Ulid; use crate::{ node::{ConnectionError, PeerKey}, @@ -19,7 +16,7 @@ use crate::{ pub(crate) use sealed_msg_type::{TransactionType, TransactionTypeId}; /// An transaction is a unique, universal and efficient identifier for any -/// roundtrip transaction as it is broadcasted around the F2 network. +/// roundtrip transaction as it is broadcasted around the Freenet network. /// /// The identifier conveys all necessary information to identify and classify the /// transaction: @@ -31,27 +28,14 @@ pub(crate) use sealed_msg_type::{TransactionType, TransactionTypeId}; /// A transaction may span different messages sent across the network. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy)] pub(crate) struct Transaction { - /// UUID V1, can retrieve timestamp information later to check for possible out of time - /// expired transactions which have been clean up already. - id: Uuid, + id: Ulid, ty: TransactionTypeId, } -static UUID_CONTEXT: Context = Context::new(14); - impl Transaction { - pub fn new(ty: TransactionTypeId, initial_peer: &PeerKey) -> Transaction { - // using v1 UUID to keep to keep track of the creation ts - let ts: Timestamp = uuid::timestamp::Timestamp::now(&UUID_CONTEXT); - - // event in the net this UUID should be unique since peer keys are unique - // however some id collision may be theoretically possible if two transactions - // are created at the same exact time and the first 6 bytes of the key coincide; - // in practice the chance of this happening is astronomically low - - let b = &mut [0; 6]; - b.copy_from_slice(&initial_peer.to_bytes()[0..6]); - let id = Uuid::new_v1(ts, b); + pub fn new() -> Transaction { + let ty = ::tx_type_id(); + let id = Ulid::new(); // 3 word size for 64-bits platforms Self { id, ty } @@ -111,6 +95,19 @@ mod sealed_msg_type { Canceled, } + impl Display for TransactionType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TransactionType::JoinRing => write!(f, "join ring"), + TransactionType::Put => write!(f, "put"), + TransactionType::Get => write!(f, "get"), + TransactionType::Subscribe => write!(f, "subscribe"), + TransactionType::Update => write!(f, "update"), + TransactionType::Canceled => write!(f, "canceled"), + } + } + } + macro_rules! transaction_type_enumeration { (decl struct { $( $var:tt -> $ty:tt),+ }) => { $( diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 28c6b74c3..f9b2e1156 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -29,7 +29,7 @@ use crate::{ ClientResponses, ClientResponsesSender, ContractError, ExecutorToEventLoopChannel, NetworkContractHandler, NetworkEventListenerHalve, OperationMode, }, - message::{InnerMessage, Message, Transaction, TransactionType, TxType}, + message::{InnerMessage, Message, Transaction, TransactionType}, operations::{ get, join_ring::{self, JoinRingMsg, JoinRingOp}, @@ -37,13 +37,11 @@ use crate::{ }, ring::{Location, PeerKeyLocation}, router::{RouteEvent, RouteOutcome}, - util::{ExponentialBackoff, IterExt}, + util::ExponentialBackoff, }; use crate::operations::handle_op_request; pub(crate) use conn_manager::{ConnectionBridge, ConnectionError}; -#[cfg(test)] -pub(crate) use event_log::test_utils::TestEventListener; pub(crate) use event_log::{EventLogRegister, EventRegister}; pub(crate) use op_state::OpManager; @@ -77,7 +75,6 @@ pub struct Node(NodeP2P); impl Node { pub async fn run(self) -> Result<(), Box> { - //TODO: Initialize tracer self.0.run_node().await?; Ok(()) } @@ -284,7 +281,7 @@ async fn join_ring_request( where CM: ConnectionBridge + Send + Sync, { - let tx_id = Transaction::new(::tx_type_id(), &peer_key); + let tx_id = Transaction::new::(); let mut op = join_ring::initial_request(peer_key, *gateway, op_storage.ring.max_hops_to_live, tx_id); if let Some(mut backoff) = backoff { @@ -295,10 +292,7 @@ where ); if backoff.sleep_async().await.is_none() { tracing::error!("Max number of retries reached"); - return Err(OpError::MaxRetriesExceeded( - tx_id, - format!("{:?}", tx_id.tx_type()), - )); + return Err(OpError::MaxRetriesExceeded(tx_id, tx_id.tx_type())); } op.backoff = Some(backoff); } @@ -367,12 +361,7 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc, op_storage: Arc, op_storage: Arc { tracing::warn!("Trying to subscribe to a contract not present: {}, requesting it first", key); - let get_op = - get::start_op(key.clone(), true, &op_storage_cp.ring.peer_key); + let get_op = get::start_op(key.clone(), true); if let Err(err) = get::request_get(&op_storage_cp, get_op, Some(client_id)).await { @@ -446,7 +434,7 @@ macro_rules! log_handling_msg { ($op:expr, $id:expr, $op_storage:ident) => { tracing::debug!( tx = %$id, - concat!("Handling ", $op, " get request @ {}"), + concat!("Handling ", $op, " request @ {}"), $op_storage.ring.peer_key, ); }; @@ -629,53 +617,27 @@ async fn process_message( async fn handle_cancelled_op( tx: Transaction, peer_key: PeerKey, - gateways: impl Iterator, op_storage: &OpManager, conn_manager: &mut CM, ) -> Result<(), OpError> where CM: ConnectionBridge + Send + Sync, { - tracing::warn!(%tx, "Failed transaction, potentially attempting a retry"); if let TransactionType::JoinRing = tx.tx_type() { - const MSG: &str = "Fatal error: unable to connect to the network"; // the attempt to join the network failed, this could be a fatal error since the node // is useless without connecting to the network, we will retry with exponential backoff match op_storage.pop(&tx) { Some(OpEnum::JoinRing(op)) if op.has_backoff() => { - if let JoinRingOp { - backoff: Some(backoff), - gateway, - .. - } = *op - { - if cfg!(test) { - join_ring_request(None, peer_key, &gateway, op_storage, conn_manager) - .await?; - } else { - join_ring_request( - Some(backoff), - peer_key, - &gateway, - op_storage, - conn_manager, - ) - .await?; - } - } + let JoinRingOp { + gateway, backoff, .. + } = *op; + let backoff = backoff.expect("infallible"); + tracing::warn!("Retry connecting to gateway {}", gateway.peer); + join_ring_request(Some(backoff), peer_key, &gateway, op_storage, conn_manager) + .await?; } - None | Some(OpEnum::JoinRing(_)) => { - let rand_gw = gateways - .shuffle() - .take(1) - .next() - .expect("at least one gateway"); - if !cfg!(test) { - tracing::error!("{}", MSG); - } else { - tracing::debug!("{}", MSG); - } - join_ring_request(None, peer_key, rand_gw, op_storage, conn_manager).await?; + Some(OpEnum::JoinRing(_)) => { + return Err(OpError::MaxRetriesExceeded(tx, tx.tx_type())); } _ => {} } @@ -693,6 +655,7 @@ impl PeerKey { PeerKey::from(Keypair::generate_ed25519().public()) } + #[cfg(test)] pub fn to_bytes(self) -> Vec { self.0.to_bytes() } diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index cef6b86ee..18d0ca3de 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -359,7 +359,6 @@ impl P2pConnManager { let res = handle_cancelled_op( tx, op_manager.ring.peer_key, - self.gateways.iter(), &op_manager, &mut self.bridge, ) diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index f27e3d8bc..b9caa4b6f 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -411,7 +411,7 @@ pub(super) mod test_utils { use parking_lot::RwLock; use super::*; - use crate::{message::TxType, node::tests::NodeLabel, ring::Distance}; + use crate::{node::tests::NodeLabel, ring::Distance}; static LOG_ID: AtomicUsize = AtomicUsize::new(0); @@ -570,12 +570,11 @@ pub(super) mod test_utils { } } - #[ignore] #[test] fn test_get_connections() -> Result<(), anyhow::Error> { let peer_id = PeerKey::random(); let loc = Location::try_from(0.5)?; - let tx = Transaction::new(::tx_type_id(), &peer_id); + let tx = Transaction::new::(); let locations = [ (PeerKey::random(), Location::try_from(0.5)?), (PeerKey::random(), Location::try_from(0.75)?), diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index 704ab10ea..e445eb401 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -145,6 +145,7 @@ impl NodeInMemory { } /// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. + #[tracing::instrument(name = "memory_event_listener", skip_all)] async fn run_event_listener( &mut self, _client_responses: ClientResponsesSender, // fixme: use this @@ -164,7 +165,6 @@ impl NodeInMemory { let res = handle_cancelled_op( tx, self.peer_key, - self.gateways.iter(), &self.op_storage, &mut self.conn_manager, ) @@ -174,15 +174,18 @@ impl NodeInMemory { if tx_type == TransactionType::JoinRing && !self.is_gateway => { tracing::warn!("Retrying joining the ring with an other peer"); - let gateway = self.gateways.iter().shuffle().next().unwrap(); - join_ring_request( - None, - self.peer_key, - gateway, - &self.op_storage, - &mut self.conn_manager, - ) - .await? + if let Some(gateway) = self.gateways.iter().shuffle().next() { + join_ring_request( + None, + self.peer_key, + gateway, + &self.op_storage, + &mut self.conn_manager, + ) + .await? + } else { + anyhow::bail!("requires at least one gateway"); + } } Err(err) => return Err(anyhow::anyhow!(err)), Ok(_) => {} diff --git a/crates/core/src/node/op_state.rs b/crates/core/src/node/op_state.rs index 047bcc1cc..8b7860e91 100644 --- a/crates/core/src/node/op_state.rs +++ b/crates/core/src/node/op_state.rs @@ -62,8 +62,7 @@ impl OpManager { } /// An early, fast path, return for communicating back changes of on-going operations - /// in the node to the main message handler receiving loop, without any transmission in - /// the network whatsoever. + /// in the node to the main message handler, without any transmission in the network whatsoever. /// /// Useful when transitioning between states that do not require any network communication /// with other nodes, like intermediate states before returning. diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index 256a8c913..b6a3c5b35 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -62,7 +62,6 @@ impl NodeP2P { } // start the p2p event loop - // todo: pass `cli_response_sender` self.conn_manager .run_event_listener( self.op_manager.clone(), diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index c4ca5e028..0ec5df0ed 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -122,6 +122,7 @@ pub(crate) struct SimNetwork { rnd_if_htl_above: usize, max_connections: usize, min_connections: usize, + init_backoff: Duration, } impl SimNetwork { @@ -149,6 +150,7 @@ impl SimNetwork { rnd_if_htl_above, max_connections, min_connections, + init_backoff: Duration::from_millis(1), }; net.build_gateways(gateways).await; net.build_nodes(nodes).await; @@ -272,11 +274,11 @@ impl SimNetwork { } } - pub async fn build(&mut self) { - self.build_with_specs(HashMap::new()).await + pub async fn start(&mut self) { + self.start_with_spec(HashMap::new()).await } - pub async fn build_with_specs(&mut self, mut specs: HashMap) { + pub async fn start_with_spec(&mut self, mut specs: HashMap) { let mut gw_not_init = self.gateways.len(); let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { @@ -284,8 +286,9 @@ impl SimNetwork { self.initialize_peer(node, label, node_spec); if gw_not_init != 0 { gw_not_init -= 1; + tokio::time::sleep(self.init_backoff).await; } else { - tokio::time::sleep(Duration::from_millis(1)).await; + tokio::time::sleep(self.init_backoff).await; } } } @@ -302,6 +305,7 @@ impl SimNetwork { user_events.request_contracts(specs.non_owned_contracts); user_events.generate_events(specs.events_to_generate); } + tracing::debug!(peer = %label, "initializing"); self.labels.insert(label, peer.peer_key); GlobalExecutor::spawn(async move { if let Some(specs) = node_specs { @@ -369,17 +373,16 @@ impl SimNetwork { /// Returns the connectivity in the network per peer (that is all the connections /// this peers has registered). - pub fn node_connectivity(&self) -> HashMap> { + pub fn node_connectivity(&self) -> HashMap)> { let mut peers_connections = HashMap::with_capacity(self.labels.len()); let key_to_label: HashMap<_, _> = self.labels.iter().map(|(k, v)| (v, k)).collect(); for (label, key) in &self.labels { - peers_connections.insert( - label.clone(), - self.event_listener - .connections(*key) - .map(|(k, d)| (key_to_label[&k].clone(), d)) - .collect::>(), - ); + let conns = self + .event_listener + .connections(*key) + .map(|(k, d)| (key_to_label[&k].clone(), d)) + .collect::>(); + peers_connections.insert(label.clone(), (*key, conns)); } peers_connections } @@ -478,7 +481,7 @@ pub(crate) async fn check_connectivity( let mut connections_per_peer: Vec<_> = node_connectivity .iter() - .map(|(k, v)| (k, v.len())) + .map(|(k, v)| (k, v.1.len())) .filter_map(|(k, v)| if !k.is_gateway() { Some(v) } else { None }) .collect(); @@ -497,15 +500,17 @@ pub(crate) async fn check_connectivity( Ok(()) } -fn pretty_print_connections(conns: &HashMap>) -> String { +fn pretty_print_connections( + conns: &HashMap)>, +) -> String { let mut connections = String::from("Node connections:\n"); let mut conns = conns.iter().collect::>(); conns.sort_by(|(a, _), (b, _)| a.cmp(b)); - for (peer, conns) in conns { + for (peer, (key, conns)) in conns { if peer.is_gateway() { continue; } - writeln!(&mut connections, "{peer}:").unwrap(); + writeln!(&mut connections, "{peer} ({key}):").unwrap(); for (conn, dist) in conns { let dist = dist.as_f64(); writeln!(&mut connections, " {conn} (dist: {dist:.3})").unwrap(); @@ -514,7 +519,6 @@ fn pretty_print_connections(conns: &HashMap Result<(), anyhow::Error> { let locations = vec![0.5356, 0.5435, 0.5468, 0.5597, 0.6745, 0.7309, 0.7412]; diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index aefe865e3..a2ee51e35 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -197,8 +197,8 @@ pub(crate) enum OpError { IncorrectTxType(TransactionType, TransactionType), #[error("op not present: {0}")] OpNotPresent(Transaction), - #[error("max number of retries for tx {0} of op type {1} reached")] - MaxRetriesExceeded(Transaction, String), + #[error("max number of retries for tx {0} of op type `{1}` reached")] + MaxRetriesExceeded(Transaction, TransactionType), // user for control flow /// This is used as an early interrumpt of an op update when an op diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 30e0d8fc7..0c5129470 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -8,7 +8,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::{ContractError, ContractHandlerEvent, StoreResponse}, - message::{InnerMessage, Message, Transaction, TxType}, + message::{InnerMessage, Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{Location, PeerKeyLocation, RingError}, @@ -409,7 +409,7 @@ impl Operation for GetOp { "Failed getting a value for contract {}, reached max retries", key ); - return Err(OpError::MaxRetriesExceeded(id, "get".to_owned())); + return Err(OpError::MaxRetriesExceeded(id, id.tx_type())); } } Some(GetState::ReceivedRequest) => { @@ -620,11 +620,11 @@ fn check_contract_found( } } -pub(crate) fn start_op(key: ContractKey, fetch_contract: bool, this_peer: &PeerKey) -> GetOp { +pub(crate) fn start_op(key: ContractKey, fetch_contract: bool) -> GetOp { let contract_location = Location::from(&key); tracing::debug!("Requesting get contract {} @ loc({contract_location})", key,); - let id = Transaction::new(::tx_type_id(), this_peer); + let id = Transaction::new::(); let state = Some(GetState::PrepareRequest { key, id, @@ -861,7 +861,7 @@ mod test { 2, ) .await; - sim_nodes.build_with_specs(get_specs).await; + sim_nodes.start_with_spec(get_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; // trigger get @ node-0, which does not own the contract @@ -900,7 +900,7 @@ mod test { // establish network let mut sim_nodes = SimNetwork::new("get_contract_not_found", NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - sim_nodes.build_with_specs(get_specs).await; + sim_nodes.start_with_spec(get_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; // trigger get @ node-1, which does not own the contract @@ -969,7 +969,7 @@ mod test { 3, ) .await; - sim_nodes.build_with_specs(get_specs).await; + sim_nodes.start_with_spec(get_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; sim_nodes diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index 03c5ba263..11ab357f5 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -103,7 +103,7 @@ impl Operation for JoinRingOp { _client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let mut return_msg = None; + let return_msg; let mut new_state = None; match input { @@ -333,45 +333,76 @@ impl Operation for JoinRingOp { peer: your_peer_id, }; - match self.state { - Some(JRState::Connecting(ConnectionInfo { gateway, .. })) => { - if !accepted_by.clone().is_empty() { - tracing::debug!( - tx = %id, - "OC received and acknowledged at requesting peer {} from gateway {}", - your_peer_id, - gateway.peer - ); - new_state = Some(JRState::OCReceived); - return_msg = Some(JoinRingMsg::Response { + // fixme: remove + tracing::debug!( + "accepted by: \nstate: {:?} \nlist: {:?}", + self.state, + accepted_by + ); + let Some(JRState::Connecting(ConnectionInfo { gateway, .. })) = self.state + else { + return Err(OpError::InvalidStateTransition(self.id)); + }; + if !accepted_by.is_empty() { + tracing::debug!( + tx = %id, + "OC received and acknowledged at requesting peer {} from gateway {}", + your_peer_id, + gateway.peer + ); + new_state = Some(JRState::OCReceived); + return_msg = Some(JoinRingMsg::Response { + id, + msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, + sender: pk_loc, + target: sender, + }); + tracing::debug!( + tx = %id, + this_peer = %your_peer_id, + location = %your_location, + "Updating assigned location" + ); + op_storage.ring.update_location(Some(your_location)); + + for other_peer in accepted_by { + let _ = propagate_oc_to_accepted_peers( + conn_manager, + op_storage, + sender, + &other_peer, + JoinRingMsg::Response { id, - msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, + target: other_peer, sender: pk_loc, - target: sender, - }); - } + msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, + }, + ) + .await; } - _ => return Err(OpError::InvalidStateTransition(self.id)), - }; - - op_storage.ring.update_location(Some(your_location)); - - for other_peer in accepted_by { - let _ = propagate_oc_to_accepted_peers( - conn_manager, - op_storage, - sender, - &other_peer, - JoinRingMsg::Response { - id, - target: other_peer, - sender: pk_loc, - msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, - }, - ) - .await; + } else { + // no connections accepted, failed + tracing::debug!( + tx = %id, + peer = %your_peer_id, + "No accepted connections, failed" + ); + let op = JoinRingOp { + id, + state: None, + gateway: self.gateway, + backoff: self.backoff, + _ttl: self._ttl, + }; + op_storage + .notify_op_change( + Message::Canceled(id), + OpEnum::JoinRing(op.into()), + None, + ) + .await?; + return Err(OpError::StatePushed); } - op_storage.ring.update_location(Some(your_location)); } JoinRingMsg::Response { id, @@ -520,9 +551,9 @@ impl Operation for JoinRingOp { } else { tracing::info!( tx = %id, - "Successfully completed connection @ {}, new location = {:?}", + assigned_location = ?op_storage.ring.own_location().location, + "Successfully completed connection @ {}", target.peer, - op_storage.ring.own_location().location ); conn_manager.add_connection(sender.peer).await?; op_storage.ring.add_connection( @@ -648,6 +679,7 @@ mod states { } } +#[derive(Debug)] enum JRState { Initializing, Connecting(ConnectionInfo), @@ -791,22 +823,36 @@ async fn forward_conn( where CM: ConnectionBridge, { - if left_htl == 0 || (ring.num_connections() == 0 && num_accepted == 0) { + if left_htl == 0 { + tracing::debug!( + tx = %id, + requester = %req_peer.peer, + "Couldn't forward join petition, no hops left or enough connections", + ); + return Ok(None); + } + + if ring.num_connections() == 0 { + tracing::warn!( + tx = %id, + requester = %req_peer.peer, + "Couldn't forward join petition, not enough connections", + ); return Ok(None); } let forward_to = if left_htl >= ring.rnd_if_htl_above { tracing::debug!( tx = %id, - "Randomly selecting peer to forward JoinRequest (requester: {})", - req_peer.peer + requester = %req_peer.peer, + "Randomly selecting peer to forward JoinRequest", ); - ring.random_peer(|p| p.peer != req_peer.peer) + ring.random_peer(|p| p != &req_peer.peer) } else { tracing::debug!( tx = %id, - "Selecting close peer to forward request (requester: {})", - req_peer.peer + requester = %req_peer.peer, + "Selecting close peer to forward request", ); ring.routing(&new_peer_loc.location.unwrap(), Some(&req_peer.peer), &[]) .and_then(|pkl| (pkl.peer != new_peer_loc.peer).then_some(pkl)) @@ -840,7 +886,7 @@ where if num_accepted != 0 { tracing::warn!( tx = %id, - "Unable to forward, will only be connected to one peer", + "Unable to forward, will only be connecting to one peer", ); } else { tracing::warn!(tx = %id, "Unable to forward or accept any connections"); @@ -1008,29 +1054,28 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn one_node_connects_to_gw() { let mut sim_nodes = SimNetwork::new("join_one_node_connects_to_gw", 1, 1, 1, 1, 2, 2).await; - sim_nodes.build().await; + sim_nodes.start().await; tokio::time::sleep(Duration::from_secs(3)).await; assert!(sim_nodes.connected(&"node-0".into())); } /// Once a gateway is left without remaining open slots, ensure forwarding connects - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn forward_connection_to_node() -> Result<(), anyhow::Error> { - // crate::config::set_logger(); + crate::config::set_logger(); const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; let mut sim_nodes = SimNetwork::new( "join_forward_connection_to_node", NUM_GW, NUM_NODES, - 3, + 4, 2, 4, 2, ) .await; - sim_nodes.build().await; + sim_nodes.start().await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await } @@ -1050,7 +1095,7 @@ mod test { 2, ) .await; - sim_nodes.build().await; + sim_nodes.start().await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(10)).await } } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 14836a177..72102ce40 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -15,7 +15,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::ContractHandlerEvent, - message::{InnerMessage, Message, Transaction, TxType}, + message::{InnerMessage, Message, Transaction }, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{Location, PeerKeyLocation, RingError}, @@ -601,7 +601,6 @@ pub(crate) fn start_op( contract: ContractContainer, value: WrappedState, htl: usize, - peer: &PeerKey, ) -> PutOp { let key = contract.key(); let contract_location = Location::from(&key); @@ -610,7 +609,7 @@ pub(crate) fn start_op( key, ); - let id = Transaction::new(::tx_type_id(), peer); + let id = Transaction::new::(); // let payload_size = contract.data().len(); let state = Some(PutState::PrepareRequest { contract, @@ -929,9 +928,9 @@ mod test { use super::*; use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_put_op_between_nodes() -> Result<(), anyhow::Error> { + crate::config::set_logger(); const NUM_NODES: usize = 2usize; const NUM_GW: usize = 1usize; @@ -1001,11 +1000,11 @@ mod test { ("gateway-0".into(), gw_0), ]); - sim_nodes.build_with_specs(put_specs).await; + sim_nodes.start_with_spec(put_specs).await; tokio::time::sleep(Duration::from_secs(5)).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; - // trigger the put op @ gw-0, this + // trigger the put op @ gw-0 sim_nodes .trigger_event(&"gateway-0".into(), 1, Some(Duration::from_secs(3))) .await?; diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 14de3c74e..28f1276bc 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -9,7 +9,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::ContractError, - message::{Message, Transaction, TxType}, + message::{Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{PeerKeyLocation, RingError}, @@ -245,7 +245,7 @@ impl Operation for SubscribeOp { retries: retries + 1, }); } else { - return Err(OpError::MaxRetriesExceeded(id, "sub".to_owned())); + return Err(OpError::MaxRetriesExceeded(id, id.tx_type())); } } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -300,8 +300,8 @@ fn build_op_result( }) } -pub(crate) fn start_op(key: ContractKey, peer: &PeerKey) -> SubscribeOp { - let id = Transaction::new(::tx_type_id(), peer); +pub(crate) fn start_op(key: ContractKey) -> SubscribeOp { + let id = Transaction::new::(); let state = Some(SubscribeState::PrepareRequest { id, key }); SubscribeOp { id, @@ -519,7 +519,7 @@ mod test { 2, ) .await; - sim_nodes.build_with_specs(subscribe_specs).await; + sim_nodes.start_with_spec(subscribe_specs).await; check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; Ok(()) diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index abbf146be..01d900522 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -342,13 +342,32 @@ impl Ring { /// Get a random peer from the known ring connections. pub fn random_peer(&self, filter_fn: F) -> Option where - F: FnMut(&&PeerKeyLocation) -> bool, + F: Fn(&PeerKey) -> bool, { - self.connections_by_location - .read() - .values() - .find(filter_fn) - .copied() + use rand::Rng; + let peers = &*self.location_for_peer.read(); + let amount = peers.len(); + if amount == 0 { + return None; + } + let mut rng = rand::thread_rng(); + let mut attempts = 0; + loop { + if attempts >= amount { + return None; + } + let selected = rng.gen_range(0..amount); + let (peer, loc) = peers.iter().nth(selected).expect("infallible"); + if !filter_fn(peer) { + attempts += 1; + continue; + } else { + return Some(PeerKeyLocation { + peer: *peer, + location: Some(*loc), + }); + } + } } /// Will return an error in case the max number of subscribers has been added. @@ -456,8 +475,7 @@ impl From<&ContractKey> for Location { impl Display for Location { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.0.to_string().as_str())?; - Ok(()) + write!(f, "{}", self.0) } } @@ -558,10 +576,7 @@ pub(crate) enum RingError { #[cfg(test)] mod test { use super::*; - use crate::client_events::test::MemoryEventsGen; - use tokio::sync::watch::channel; - #[ignore] #[test] fn location_dist() { let l0 = Location(0.); @@ -572,58 +587,4 @@ mod test { let l1 = Location(0.50); assert!(l0.distance(l1) == Distance(0.25)); } - - #[ignore] - #[test] - fn find_closest() { - let peer_key: PeerKey = PeerKey::random(); - - let (_, receiver) = channel((0, peer_key)); - let user_events = MemoryEventsGen::new(receiver, peer_key); - let config = NodeBuilder::new([Box::new(user_events)]); - let ring = Ring::new::<1, node::TestEventListener>(&config, &[]).unwrap(); - - fn build_pk(loc: Location) -> PeerKeyLocation { - PeerKeyLocation { - peer: PeerKey::random(), - location: Some(loc), - } - } - - { - let conns = &mut *ring.connections_by_location.write(); - conns.insert(Location(0.3), build_pk(Location(0.3))); - conns.insert(Location(0.5), build_pk(Location(0.5))); - conns.insert(Location(0.0), build_pk(Location(0.0))); - } - - assert_eq!( - Location(0.0), - ring.routing(&Location(0.9), None, &[]) - .unwrap() - .location - .unwrap() - ); - assert_eq!( - Location(0.0), - ring.routing(&Location(0.1), None, &[]) - .unwrap() - .location - .unwrap() - ); - assert_eq!( - Location(0.5), - ring.routing(&Location(0.41), None, &[]) - .unwrap() - .location - .unwrap() - ); - assert_eq!( - Location(0.3), - ring.routing(&Location(0.39), None, &[]) - .unwrap() - .location - .unwrap() - ); - } } diff --git a/crates/core/src/runtime/contract_store.rs b/crates/core/src/runtime/contract_store.rs index bc9883a35..4204ef549 100644 --- a/crates/core/src/runtime/contract_store.rs +++ b/crates/core/src/runtime/contract_store.rs @@ -51,7 +51,7 @@ pub struct ContractStore { key_to_code_part: Arc>, } // TODO: add functionality to delete old contracts which have not been used for a while -// to keep the total speed used under a configured threshold +// to keep the total space used under a configured threshold static LOCK_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); static KEY_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); diff --git a/crates/core/src/runtime/wasm_runtime.rs b/crates/core/src/runtime/wasm_runtime.rs index 7c5d32e83..b7d2a6446 100644 --- a/crates/core/src/runtime/wasm_runtime.rs +++ b/crates/core/src/runtime/wasm_runtime.rs @@ -209,7 +209,6 @@ impl Runtime { let module = if let Some(module) = self.delegate_modules.get(key) { module } else { - // FIXME let delegate = self .delegate_store .fetch_delegate(key, params) diff --git a/crates/core/src/server/mod.rs b/crates/core/src/server/mod.rs index 730c2a4de..e9a56b4cf 100644 --- a/crates/core/src/server/mod.rs +++ b/crates/core/src/server/mod.rs @@ -130,7 +130,7 @@ pub mod local_node { if let Some(cause) = cause { tracing::info!("disconnecting cause: {cause}"); } - // todo: token must live for a bit to allow reconnections + // fixme: token must live for a bit to allow reconnections if let Some(rm_token) = gw .attested_contracts .iter() diff --git a/crates/fdev/src/build.rs b/crates/fdev/src/build.rs index a8c491d43..d16a6ef90 100644 --- a/crates/fdev/src/build.rs +++ b/crates/fdev/src/build.rs @@ -51,6 +51,7 @@ fn compile_options(cli_config: &BuildToolCliConfig) -> impl Iterator Result<(), DynError> { Error::CommandFailed("npm") })?; pipe_std_streams(child)?; - // todo: change pacakge.json: + // todo: change package.json: // - include dependencies: freenet-stdlib let child = Command::new(TSC) diff --git a/stdlib b/stdlib index 3d21697c2..cdb8ceaba 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit 3d21697c2393f7b563cf2242237c6fbb6ec08f44 +Subproject commit cdb8ceabaa6fcd2cd730b7be66da7fc2b57a50bf From 87da354155a122d20b3d4769acea083b9bb91c1f Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Tue, 17 Oct 2023 14:50:48 +0200 Subject: [PATCH 05/14] Actually return back the populated connection list --- crates/core/src/message.rs | 14 +++---- crates/core/src/node.rs | 5 +-- .../core/src/node/conn_manager/p2p_protoc.rs | 2 +- crates/core/src/node/in_memory_impl.rs | 2 +- crates/core/src/operations.rs | 2 +- crates/core/src/operations/join_ring.rs | 37 ++++++++++--------- crates/core/src/ring.rs | 20 +++++++--- 7 files changed, 46 insertions(+), 36 deletions(-) diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 551339abb..f284f7503 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -142,8 +142,8 @@ pub(crate) enum Message { Get(GetMsg), Subscribe(SubscribeMsg), Update(UpdateMsg), - /// Failed a transaction, informing of cancellation. - Canceled(Transaction), + /// Failed a transaction, informing of abortion. + Aborted(Transaction), } pub(crate) trait InnerMessage: Into { @@ -191,7 +191,7 @@ impl Message { Get(op) => op.id(), Subscribe(op) => op.id(), Update(_op) => todo!(), - Canceled(tx) => tx, + Aborted(tx) => tx, } } @@ -203,7 +203,7 @@ impl Message { Get(op) => op.target(), Subscribe(op) => op.target(), Update(_op) => todo!(), - Canceled(_) => None, + Aborted(_) => None, } } @@ -216,13 +216,13 @@ impl Message { Get(op) => op.terminal(), Subscribe(op) => op.terminal(), Update(_op) => todo!(), - Canceled(_) => true, + Aborted(_) => true, } } pub fn track_stats(&self) -> bool { use Message::*; - !matches!(self, JoinRing(_) | Subscribe(_) | Canceled(_)) + !matches!(self, JoinRing(_) | Subscribe(_) | Aborted(_)) } } @@ -236,7 +236,7 @@ impl Display for Message { Get(msg) => msg.fmt(f)?, Subscribe(msg) => msg.fmt(f)?, Update(_op) => todo!(), - Canceled(msg) => msg.fmt(f)?, + Aborted(msg) => msg.fmt(f)?, }; write!(f, "}}") } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index f9b2e1156..7617d8680 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -286,10 +286,7 @@ where join_ring::initial_request(peer_key, *gateway, op_storage.ring.max_hops_to_live, tx_id); if let Some(mut backoff) = backoff { // backoff to retry later in case it failed - tracing::warn!( - "Performing a new join attempt, attempt number: {}", - backoff.retries() - ); + tracing::warn!("Performing a new join, attempt {}", backoff.retries() + 1); if backoff.sleep_async().await.is_none() { tracing::error!("Max number of retries reached"); return Err(OpError::MaxRetriesExceeded(tx_id, tx_id.tx_type())); diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index 18d0ca3de..38d1f194b 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -354,7 +354,7 @@ impl P2pConnManager { Ok(Left((msg, client_id))) => { let cb = self.bridge.clone(); match msg { - Message::Canceled(tx) => { + Message::Aborted(tx) => { let tx_type = tx.tx_type(); let res = handle_cancelled_op( tx, diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index e445eb401..bdbc8b1b5 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -160,7 +160,7 @@ impl NodeInMemory { } }; - if let Ok(Either::Left(Message::Canceled(tx))) = msg { + if let Ok(Either::Left(Message::Aborted(tx))) = msg { let tx_type = tx.tx_type(); let res = handle_cancelled_op( tx, diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index a2ee51e35..410bff877 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -75,7 +75,7 @@ where } Err((err, tx_id)) => { if let Some(sender) = sender { - conn_manager.send(&sender, Message::Canceled(tx_id)).await?; + conn_manager.send(&sender, Message::Aborted(tx_id)).await?; } return Err(err); } diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index 11ab357f5..d33fd98ed 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -17,8 +17,6 @@ use crate::{ pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; -const MAX_JOIN_RETRIES: usize = 3; - pub(crate) struct JoinRingOp { id: Transaction, state: Option, @@ -132,7 +130,7 @@ impl Operation for JoinRingOp { tracing::debug!(tx = %id, "Accepting connection from {}", req_peer,); HashSet::from_iter([this_node_loc]) } else { - tracing::debug!(tx = %id, "Rejecting connection from peer {}", req_peer); + tracing::debug!(tx = %id, at_peer = %this_node_loc.peer, "Rejecting connection from peer {}", req_peer); HashSet::new() }; @@ -334,16 +332,13 @@ impl Operation for JoinRingOp { }; // fixme: remove - tracing::debug!( - "accepted by: \nstate: {:?} \nlist: {:?}", - self.state, - accepted_by - ); + tracing::debug!("accepted by state: {:?} ", self.state,); let Some(JRState::Connecting(ConnectionInfo { gateway, .. })) = self.state else { return Err(OpError::InvalidStateTransition(self.id)); }; if !accepted_by.is_empty() { + tracing::debug!("accepted by list: {:?} ", accepted_by); tracing::debug!( tx = %id, "OC received and acknowledged at requesting peer {} from gateway {}", @@ -385,7 +380,7 @@ impl Operation for JoinRingOp { tracing::debug!( tx = %id, peer = %your_peer_id, - "No accepted connections, failed" + "Failed to establish any connections, aborting" ); let op = JoinRingOp { id, @@ -396,7 +391,7 @@ impl Operation for JoinRingOp { }; op_storage .notify_op_change( - Message::Canceled(id), + Message::Aborted(id), OpEnum::JoinRing(op.into()), None, ) @@ -472,7 +467,7 @@ impl Operation for JoinRingOp { target: state_target, sender: target, msg: JoinResponse::AcceptedBy { - peers: accepted_by, + peers: previously_accepted, your_location: new_location, your_peer_id: new_peer_id, }, @@ -490,7 +485,9 @@ impl Operation for JoinRingOp { id, target: state_target, sender: target, - msg: JoinResponse::Proxy { accepted_by }, + msg: JoinResponse::Proxy { + accepted_by: previously_accepted, + }, }); } } @@ -733,19 +730,25 @@ pub(crate) fn initial_request( max_hops_to_live: usize, id: Transaction, ) -> JoinRingOp { + const MAX_JOIN_RETRIES: usize = 3; tracing::debug!(tx = %id, "Connecting to gw {} from {}", gateway.peer, this_peer); let state = JRState::Connecting(ConnectionInfo { gateway, this_peer, max_hops_to_live, }); + let ceiling = if cfg!(test) { + Duration::from_secs(1) + } else { + Duration::from_secs(120) + }; JoinRingOp { id, state: Some(state), gateway: Box::new(gateway), backoff: Some(ExponentialBackoff::new( Duration::from_secs(1), - Duration::from_secs(120), + ceiling, MAX_JOIN_RETRIES, )), _ttl: PEER_TIMEOUT, @@ -1069,14 +1072,14 @@ mod test { "join_forward_connection_to_node", NUM_GW, NUM_NODES, - 4, - 2, - 4, + 3, 2, + 8, + 5, ) .await; sim_nodes.start().await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await + check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(10)).await } /// Given a network of N peers all nodes should have connections. diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index 01d900522..f933b48d9 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -250,23 +250,27 @@ impl Ring { /// # Panic /// Will panic if the node checking for this condition has no location assigned. pub fn should_accept(&self, location: &Location) -> bool { + let cbl = &*self.connections_by_location.read(); let open_conn = self.open_connections.fetch_add(1, SeqCst) + 1; let my_location = &self .own_location() .location .expect("this node has no location assigned!"); - let cbl = &*self.connections_by_location.read(); let accepted = if location == my_location || cbl.contains_key(location) { false } else if open_conn < self.min_connections { true } else if open_conn >= self.max_connections { + tracing::debug!(peer = %self.peer_key, "max connections reached"); false } else { - my_location.distance(location) - < self - .median_distance_to(my_location) - .unwrap_or(Distance(0.5)) + let median_distance = self + .median_distance_to(my_location) + .unwrap_or(Distance(0.5)); + let dist_to_loc = my_location.distance(location); + let is_lower_than_median = dist_to_loc < median_distance; + tracing::debug!("dist to connection loc: {dist_to_loc}, median dist: {median_distance}, accepting: {is_lower_than_median}"); + is_lower_than_median }; if !accepted { self.open_connections.fetch_sub(1, SeqCst); @@ -563,6 +567,12 @@ impl Ord for Distance { } } +impl Display for Distance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + #[derive(thiserror::Error, Debug)] pub(crate) enum RingError { #[error(transparent)] From 574698a2819e3d5e1279f7ade4d59ff92c8f0c00 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Wed, 18 Oct 2023 15:13:21 +0200 Subject: [PATCH 06/14] Fixes to event log and sim connectivity report --- crates/core/src/config.rs | 23 +- crates/core/src/node.rs | 23 +- .../core/src/node/conn_manager/in_memory.rs | 64 ++++-- .../core/src/node/conn_manager/p2p_protoc.rs | 80 ++++++- crates/core/src/node/event_log.rs | 208 +++++++++++++----- crates/core/src/node/in_memory_impl.rs | 7 +- crates/core/src/node/p2p_impl.rs | 6 +- crates/core/src/node/tests.rs | 139 ++++++------ crates/core/src/operations/get.rs | 32 +-- crates/core/src/operations/join_ring.rs | 28 +-- crates/core/src/operations/put.rs | 24 +- crates/core/src/operations/subscribe.rs | 8 +- 12 files changed, 419 insertions(+), 223 deletions(-) diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index f5cd1167a..463414add 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -324,17 +324,18 @@ impl libp2p::swarm::Executor for GlobalExecutor { } pub fn set_logger() { - tracing_subscriber::fmt() - .with_level(true) - .with_file(true) - .with_line_number(true) - .with_env_filter( - tracing_subscriber::EnvFilter::builder() - .with_default_directive(tracing_subscriber::filter::LevelFilter::DEBUG.into()) - .from_env_lossy() - .add_directive("stretto=off".parse().unwrap()), - ) - .init(); + let sub = tracing_subscriber::fmt().with_level(true).with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing_subscriber::filter::LevelFilter::DEBUG.into()) + .from_env_lossy() + .add_directive("stretto=off".parse().unwrap()), + ); + + if cfg!(any(test, debug_assertions)) { + sub.with_file(true).with_line_number(true).init(); + } else { + sub.init(); + } } pub(super) mod tracer { diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 7617d8680..ebc372f60 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -15,6 +15,7 @@ use std::{ time::Duration, }; +use either::Either; use freenet_stdlib::client_api::{ClientRequest, ContractRequest}; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; @@ -469,12 +470,13 @@ async fn report_result( payload_transfer_time, }, }; - if let Err(err) = event_listener - .event_received(EventLog::route_event(op_res.id(), op_storage, &event)) - .await - { - tracing::warn!("failed logging event: {err}"); - } + event_listener + .register_events(Either::Left(EventLog::route_event( + op_res.id(), + op_storage, + &event, + ))) + .await; op_storage.ring.routing_finished(event); } // todo: handle failures, need to track timeouts and other potential failures @@ -516,12 +518,9 @@ async fn process_message( let cli_req = client_id.zip(client_req_handler_callback); match msg { Ok(msg) => { - if let Err(err) = event_listener - .event_received(EventLog::from_msg(&msg, &op_storage)) - .await - { - tracing::warn!("failed logging event: {err}"); - } + event_listener + .register_events(EventLog::from_inbound_msg(&msg, &op_storage)) + .await; match msg { Message::JoinRing(op) => { log_handling_msg!("join", op.id(), op_storage); diff --git a/crates/core/src/node/conn_manager/in_memory.rs b/crates/core/src/node/conn_manager/in_memory.rs index ca62c9144..f1fbda01f 100644 --- a/crates/core/src/node/conn_manager/in_memory.rs +++ b/crates/core/src/node/conn_manager/in_memory.rs @@ -8,23 +8,33 @@ use std::{ use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; -use parking_lot::Mutex; use rand::{prelude::StdRng, thread_rng, Rng, SeedableRng}; +use tokio::sync::Mutex; use super::{ConnectionBridge, ConnectionError, PeerKey}; -use crate::{config::GlobalExecutor, message::Message}; +use crate::{ + config::GlobalExecutor, + message::Message, + node::{event_log::EventLog, EventLogRegister, OpManager}, +}; static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = OnceCell::new(); pub(in crate::node) struct MemoryConnManager { pub transport: InMemoryTransport, + log_register: Arc>>, + op_manager: Arc, msg_queue: Arc>>, peer: PeerKey, } impl MemoryConnManager { - pub fn new(peer: PeerKey) -> Self { + pub fn new( + peer: PeerKey, + log_register: Box, + op_manager: Arc, + ) -> Self { let transport = InMemoryTransport::new(peer); let msg_queue = Arc::new(Mutex::new(Vec::new())); @@ -33,21 +43,21 @@ impl MemoryConnManager { GlobalExecutor::spawn(async move { // evaluate the messages as they arrive loop { - let msg = { tr_cp.msg_stack_queue.lock().pop() }; - if let Some(msg) = msg { - let msg_data: Message = - bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); - if let Some(mut queue) = msg_queue_cp.try_lock() { - queue.push(msg_data); - std::mem::drop(queue); - } + let Some(msg) = tr_cp.msg_stack_queue.lock().await.pop() else { + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + }; + let msg_data: Message = bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); + if let Ok(mut queue) = msg_queue_cp.try_lock() { + queue.push(msg_data); } - tokio::time::sleep(Duration::from_millis(10)).await; } }); Self { transport, + log_register: Arc::new(Mutex::new(log_register)), + op_manager, msg_queue, peer, } @@ -55,22 +65,27 @@ impl MemoryConnManager { pub async fn recv(&self) -> Result { loop { - if let Some(mut queue) = self.msg_queue.try_lock() { - let msg = queue.pop(); - std::mem::drop(queue); - if let Some(msg) = msg { - return Ok(msg); - } - } - tokio::time::sleep(Duration::from_millis(10)).await; + let Some(msg) = self.msg_queue.try_lock().ok().and_then(|mut l| l.pop()) else { + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + }; + return Ok(msg); } } } impl Clone for MemoryConnManager { fn clone(&self) -> Self { + let log_register = loop { + if let Ok(lr) = self.log_register.try_lock() { + break lr.trait_clone(); + } + std::thread::sleep(Duration::from_nanos(50)); + }; Self { transport: self.transport.clone(), + log_register: Arc::new(Mutex::new(log_register)), + op_manager: self.op_manager.clone(), msg_queue: self.msg_queue.clone(), peer: self.peer, } @@ -80,6 +95,11 @@ impl Clone for MemoryConnManager { #[async_trait::async_trait] impl ConnectionBridge for MemoryConnManager { async fn send(&self, target: &PeerKey, msg: Message) -> super::ConnResult<()> { + self.log_register + .try_lock() + .expect("unique lock") + .register_events(EventLog::from_outbound_msg(&msg, &self.op_manager)) + .await; let msg = bincode::serialize(&msg)?; self.transport.send(*target, msg); Ok(()) @@ -139,7 +159,7 @@ impl InMemoryTransport { delayed.entry(msg.target).or_default().push(msg); tokio::time::sleep(Duration::from_millis(10)).await; } else { - rcv_msg_c.lock().push(msg); + rcv_msg_c.lock().await.push(msg); } } Ok(msg) => { @@ -156,7 +176,7 @@ impl InMemoryTransport { && !delayed.is_empty()) || delayed.len() == MAX_DELAYED_MSG { - let mut queue = rcv_msg_c.lock(); + let mut queue = rcv_msg_c.lock().await; for (_, msgs) in delayed.drain() { queue.extend(msgs); } diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index 38d1f194b..674c7f235 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -5,6 +5,7 @@ use std::{ pin::Pin, sync::Arc, task::Poll, + time::Duration, }; use asynchronous_codec::{BytesMut, Framed}; @@ -31,7 +32,10 @@ use libp2p::{ }, InboundUpgrade, Multiaddr, OutboundUpgrade, PeerId, Swarm, }; -use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::sync::{ + mpsc::{self, Receiver, Sender}, + Mutex, +}; use unsigned_varint::codec::UviBytes; use super::{ConnectionBridge, ConnectionError, EventLoopNotifications}; @@ -41,8 +45,8 @@ use crate::{ contract::{ClientResponsesSender, ExecutorToEventLoopChannel, NetworkEventListenerHalve}, message::{Message, NodeEvent, Transaction, TransactionType}, node::{ - handle_cancelled_op, join_ring_request, process_message, EventLogRegister, InitPeerNode, - NodeBuilder, OpManager, PeerKey, + event_log::EventLog, handle_cancelled_op, join_ring_request, process_message, + EventLogRegister, InitPeerNode, NodeBuilder, OpManager, PeerKey, }, operations::OpError, ring::PeerKeyLocation, @@ -117,19 +121,73 @@ fn multiaddr_from_connection(conn: (IpAddr, u16)) -> Multiaddr { type P2pBridgeEvent = Either<(PeerKey, Box), NodeEvent>; -#[derive(Clone)] pub(crate) struct P2pBridge { active_net_connections: Arc>, accepted_peers: Arc>, ev_listener_tx: Sender, + op_manager: Arc, + log_register: Arc>>, } impl P2pBridge { - fn new(sender: Sender) -> Self { + fn new( + sender: Sender, + op_manager: Arc, + event_register: EL, + ) -> Self + where + EL: EventLogRegister, + { Self { active_net_connections: Arc::new(DashMap::new()), accepted_peers: Arc::new(DashSet::new()), ev_listener_tx: sender, + op_manager, + log_register: Arc::new(Mutex::new(Box::new(event_register))), + } + } +} + +#[cfg(any(debug_assertions, test))] +static CONTESTED_BRIDGE_CLONES: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +#[cfg(any(debug_assertions, test))] +static TOTAL_BRIDGE_CLONES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +impl Clone for P2pBridge { + fn clone(&self) -> Self { + let log_register = loop { + if let Ok(lr) = self.log_register.try_lock() { + #[cfg(any(debug_assertions, test))] + { + TOTAL_BRIDGE_CLONES.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } + break lr.trait_clone(); + } + #[cfg(any(debug_assertions, test))] + { + let contested = + CONTESTED_BRIDGE_CLONES.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + + if contested % 100 == 0 { + let total = TOTAL_BRIDGE_CLONES.load(std::sync::atomic::Ordering::Acquire); + if total > 0 { + let threshold = (total as f64 * 0.01) as usize; + if contested / total > threshold { + tracing::debug!("p2p bridge clone contested more than 1% of the time"); + } + } + } + } + std::thread::sleep(Duration::from_nanos(50)); + }; + Self { + active_net_connections: self.active_net_connections.clone(), + accepted_peers: self.accepted_peers.clone(), + ev_listener_tx: self.ev_listener_tx.clone(), + op_manager: self.op_manager.clone(), + log_register: Arc::new(Mutex::new(log_register)), } } } @@ -157,6 +215,10 @@ impl ConnectionBridge for P2pBridge { } async fn send(&self, target: &PeerKey, msg: Message) -> super::ConnResult<()> { + self.log_register + .lock() + .await + .register_events(EventLog::from_outbound_msg(&msg, &self.op_manager)); self.ev_listener_tx .send(Left((*target, Box::new(msg)))) .await @@ -180,7 +242,7 @@ impl P2pConnManager { transport: transport::Boxed<(PeerId, muxing::StreamMuxerBox)>, config: &NodeBuilder, op_manager: Arc, - event_listener: &dyn EventLogRegister, + event_listener: impl EventLogRegister + Clone, ) -> Result { // We set a global executor which is virtually the Tokio multi-threaded executor // to reuse it's thread pool and scheduler in order to drive futures. @@ -197,7 +259,7 @@ impl P2pConnManager { &config.local_key, &config.remote_nodes, &public_addr, - op_manager, + op_manager.clone(), ); let mut swarm = Swarm::new( transport, @@ -212,7 +274,7 @@ impl P2pConnManager { } let (tx_bridge_cmd, rx_bridge_cmd) = mpsc::channel(100); - let bridge = P2pBridge::new(tx_bridge_cmd); + let bridge = P2pBridge::new(tx_bridge_cmd, op_manager, event_listener.clone()); let gateways = config.get_gateways()?; Ok(P2pConnManager { @@ -221,7 +283,7 @@ impl P2pConnManager { bridge, conn_bridge_rx: rx_bridge_cmd, public_addr, - event_listener: event_listener.trait_clone(), + event_listener: Box::new(event_listener), }) } diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index b9caa4b6f..08f0ecab0 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -1,6 +1,7 @@ use std::{io, path::Path, time::SystemTime}; use chrono::{DateTime, Utc}; +use either::Either; use freenet_stdlib::prelude::*; use futures::{future::BoxFuture, FutureExt}; use serde::{Deserialize, Serialize}; @@ -14,8 +15,8 @@ use crate::{ config::GlobalExecutor, contract::StoreResponse, message::{Message, Transaction}, - operations::{get::GetMsg, join_ring::JoinRingMsg, put::PutMsg}, - ring::{Location, PeerKeyLocation}, + operations::{get::GetMsg, join_ring, put::PutMsg}, + ring::PeerKeyLocation, router::RouteEvent, DynError, }; @@ -34,7 +35,10 @@ struct ListenerLogId(usize); /// This type then can emit it's own information to adjacent systems /// or is a no-op. pub(crate) trait EventLogRegister: std::any::Any + Send + Sync + 'static { - fn event_received<'a>(&'a mut self, ev: EventLog) -> BoxFuture<'a, Result<(), DynError>>; + fn register_events<'a>( + &'a mut self, + events: Either, Vec>>, + ) -> BoxFuture<'a, ()>; fn trait_clone(&self) -> Box; fn as_any(&self) -> &dyn std::any::Any where @@ -44,7 +48,6 @@ pub(crate) trait EventLogRegister: std::any::Any + Send + Sync + 'static { } } -#[allow(dead_code)] // fixme: remove this pub(crate) struct EventLog<'a> { tx: &'a Transaction, peer_id: &'a PeerKey, @@ -64,15 +67,76 @@ impl<'a> EventLog<'a> { } } - pub fn from_msg(msg: &'a Message, op_storage: &'a OpManager) -> Self { + pub fn from_outbound_msg( + msg: &'a Message, + op_storage: &'a OpManager, + ) -> Either> { let kind = match msg { - Message::JoinRing(JoinRingMsg::Connected { sender, target, .. }) => { - EventKind::Connected { - loc: target.location.unwrap(), - from: target.peer, - to: *sender, + Message::JoinRing(join_ring::JoinRingMsg::Response { + msg: + join_ring::JoinResponse::AcceptedBy { + peers, + your_location, + your_peer_id, + }, + .. + }) => { + let this_peer = op_storage.ring.own_location(); + if peers.contains(&this_peer) { + EventKind::Connected { + this: this_peer, + connected: PeerKeyLocation { + peer: *your_peer_id, + location: Some(*your_location), + }, + } + } else { + EventKind::Ignored } } + _ => EventKind::Ignored, + }; + Either::Left(EventLog { + tx: msg.id(), + peer_id: &op_storage.ring.peer_key, + kind, + }) + } + + pub fn from_inbound_msg( + msg: &'a Message, + op_storage: &'a OpManager, + ) -> Either> { + let kind = match msg { + Message::JoinRing(join_ring::JoinRingMsg::Response { + msg: + join_ring::JoinResponse::AcceptedBy { + peers, + your_location, + your_peer_id, + }, + .. + }) => { + return Either::Right( + peers + .iter() + .map(|peer| { + let kind: EventKind = EventKind::Connected { + this: PeerKeyLocation { + peer: *your_peer_id, + location: Some(*your_location), + }, + connected: *peer, + }; + EventLog { + tx: msg.id(), + peer_id: &op_storage.ring.peer_key, + kind, + } + }) + .collect(), + ); + } Message::Put(PutMsg::RequestPut { contract, target, .. }) => { @@ -113,13 +177,13 @@ impl<'a> EventLog<'a> { value: StoreResponse { state: Some(_), .. }, .. }) => EventKind::Get { key: key.clone() }, - _ => EventKind::Unknown, + _ => EventKind::Ignored, }; - EventLog { + Either::Left(EventLog { tx: msg.id(), peer_id: &op_storage.ring.peer_key, kind, - } + }) } } @@ -339,14 +403,35 @@ impl EventRegister { } impl EventLogRegister for EventRegister { - fn event_received<'a>(&'a mut self, log: EventLog) -> BoxFuture<'a, Result<(), DynError>> { - let log_msg = LogMessage { - datetime: Utc::now(), - tx: *log.tx, - kind: log.kind, - peer_id: *log.peer_id, - }; - async { Ok(self.log_sender.send(log_msg).await?) }.boxed() + fn register_events<'a>( + &'a mut self, + logs: Either, Vec>>, + ) -> BoxFuture<'a, ()> { + async { + match logs { + Either::Left(log) => { + let log_msg = LogMessage { + datetime: Utc::now(), + tx: *log.tx, + kind: log.kind, + peer_id: *log.peer_id, + }; + let _ = self.log_sender.send(log_msg).await; + } + Either::Right(logs) => { + for log in logs { + let log_msg = LogMessage { + datetime: Utc::now(), + tx: *log.tx, + kind: log.kind, + peer_id: *log.peer_id, + }; + let _ = self.log_sender.send(log_msg).await; + } + } + } + } + .boxed() } fn trait_clone(&self) -> Box { @@ -355,18 +440,18 @@ impl EventLogRegister for EventRegister { } #[derive(Serialize, Deserialize)] +// todo: make this take by ref instead enum EventKind { Connected { - loc: Location, - from: PeerKey, - to: PeerKeyLocation, + this: PeerKeyLocation, + connected: PeerKeyLocation, }, Put(PutEvent), Get { key: ContractKey, }, Route(RouteEvent), - Unknown, + Ignored, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] @@ -408,7 +493,7 @@ pub(super) mod test_utils { }; use dashmap::DashMap; - use parking_lot::RwLock; + use parking_lot::Mutex; use super::*; use crate::{node::tests::NodeLabel, ring::Distance}; @@ -419,7 +504,7 @@ pub(super) mod test_utils { pub(crate) struct TestEventListener { node_labels: Arc>, tx_log: Arc>>, - logs: Arc>>, + logs: Arc>>, } impl TestEventListener { @@ -427,7 +512,7 @@ pub(super) mod test_utils { TestEventListener { node_labels: Arc::new(DashMap::new()), tx_log: Arc::new(DashMap::new()), - logs: Arc::new(RwLock::new(Vec::new())), + logs: Arc::new(Mutex::new(Vec::new())), } } @@ -436,7 +521,7 @@ pub(super) mod test_utils { } pub fn is_connected(&self, peer: &PeerKey) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); logs.iter() .any(|log| &log.peer_id == peer && matches!(log.kind, EventKind::Connected { .. })) } @@ -447,7 +532,7 @@ pub(super) mod test_utils { for_key: &ContractKey, expected_value: &WrappedState, ) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); let put_ops = logs.iter().filter_map(|l| match &l.kind { EventKind::Put(ev) => Some((&l.tx, ev)), _ => None, @@ -485,7 +570,7 @@ pub(super) mod test_utils { /// The contract was broadcasted from one peer to an other successfully. pub fn contract_broadcasted(&self, for_key: &ContractKey) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); let put_broadcast_ops = logs.iter().filter_map(|l| match &l.kind { EventKind::Put(ev @ PutEvent::BroadcastEmitted { .. }) | EventKind::Put(ev @ PutEvent::BroadcastReceived { .. }) => Some((&l.tx, ev)), @@ -518,7 +603,7 @@ pub(super) mod test_utils { } pub fn has_got_contract(&self, peer: &PeerKey, expected_key: &ContractKey) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); logs.iter().any(|log| { &log.peer_id == peer && matches!(log.kind, EventKind::Get { ref key } if key == expected_key ) @@ -527,12 +612,18 @@ pub(super) mod test_utils { /// Unique connections for a given peer and their relative distance to other peers. pub fn connections(&self, peer: PeerKey) -> impl Iterator { - let logs = self.logs.read(); + let logs = self.logs.lock(); logs.iter() .filter_map(|l| { - if let EventKind::Connected { loc, from, to } = l.kind { - if from == peer { - return Some((to.peer, loc.distance(to.location.unwrap()))); + if let EventKind::Connected { this, connected } = l.kind { + if this.peer == peer { + return Some(( + connected.peer, + connected + .location + .expect("set location") + .distance(this.location.unwrap()), + )); } } None @@ -555,14 +646,28 @@ pub(super) mod test_utils { } impl super::EventLogRegister for TestEventListener { - fn event_received<'a>(&'a mut self, log: EventLog) -> BoxFuture<'a, Result<(), DynError>> { - let tx = log.tx; - let mut logs = self.logs.write(); - let (msg_log, log_id) = Self::create_log(log); - logs.push(msg_log); - std::mem::drop(logs); - self.tx_log.entry(*tx).or_default().push(log_id); - async { Ok(()) }.boxed() + fn register_events<'a>( + &'a mut self, + logs: Either, Vec>>, + ) -> BoxFuture<'a, ()> { + match logs { + Either::Left(log) => { + let tx = log.tx; + let (msg_log, log_id) = Self::create_log(log); + self.logs.lock().push(msg_log); + self.tx_log.entry(*tx).or_default().push(log_id); + } + Either::Right(logs) => { + let logs_list = &mut *self.logs.lock(); + for log in logs { + let tx = log.tx; + let (msg_log, log_id) = Self::create_log(log); + logs_list.push(msg_log); + self.tx_log.entry(*tx).or_default().push(log_id); + } + } + } + async {}.boxed() } fn trait_clone(&self) -> Box { @@ -572,9 +677,10 @@ pub(super) mod test_utils { #[test] fn test_get_connections() -> Result<(), anyhow::Error> { + use crate::ring::Location; let peer_id = PeerKey::random(); let loc = Location::try_from(0.5)?; - let tx = Transaction::new::(); + let tx = Transaction::new::(); let locations = [ (PeerKey::random(), Location::try_from(0.5)?), (PeerKey::random(), Location::try_from(0.75)?), @@ -583,18 +689,20 @@ pub(super) mod test_utils { let mut listener = TestEventListener::new(); locations.iter().for_each(|(other, location)| { - listener.event_received(EventLog { + listener.register_events(Either::Left(EventLog { tx: &tx, peer_id: &peer_id, kind: EventKind::Connected { - loc, - from: peer_id, - to: PeerKeyLocation { + this: PeerKeyLocation { + peer: peer_id, + location: Some(loc), + }, + connected: PeerKeyLocation { peer: *other, location: Some(*location), }, }, - }); + })); }); let distances: Vec<_> = listener.connections(peer_id).collect(); diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index bdbc8b1b5..da2d4ce75 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -43,8 +43,8 @@ impl NodeInMemory { event_listener: EL, ch_builder: String, ) -> Result { + let event_listener = Box::new(event_listener); let peer_key = PeerKey::from(builder.local_key.public()); - let conn_manager = MemoryConnManager::new(peer_key); let gateways = builder.get_gateways()?; let is_gateway = builder.local_ip.zip(builder.local_port).is_some(); @@ -58,6 +58,9 @@ impl NodeInMemory { .await .map_err(|e| anyhow::anyhow!(e))?; + let conn_manager = + MemoryConnManager::new(peer_key, event_listener.trait_clone(), op_storage.clone()); + GlobalExecutor::spawn(contract::contract_handling(contract_handler)); Ok(NodeInMemory { @@ -66,7 +69,7 @@ impl NodeInMemory { op_storage, gateways, notification_channel, - event_listener: Box::new(event_listener), + event_listener, is_gateway, _executor_listener, }) diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index b6a3c5b35..2ea78a3bb 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -72,18 +72,18 @@ impl NodeP2P { .await } - pub(crate) async fn build( + pub(crate) async fn build( builder: NodeBuilder, event_listener: EL, ch_builder: CH::Builder, ) -> Result where CH: ContractHandler + Send + Sync + 'static, + EL: EventLogRegister + Clone, { let peer_key = PeerKey::from(builder.local_key.public()); let gateways = builder.get_gateways()?; - let event_listener: Box = Box::new(event_listener); let ring = Ring::new::(&builder, &gateways)?; let (notification_channel, notification_tx) = EventLoopNotifications::channel(); let (ch_outbound, ch_inbound) = contract::contract_handler_channel(); @@ -96,7 +96,7 @@ impl NodeP2P { let conn_manager = { let transport = Self::config_transport(&builder.local_key)?; - P2pConnManager::build(transport, &builder, op_storage.clone(), &*event_listener)? + P2pConnManager::build(transport, &builder, op_storage.clone(), event_listener)? }; GlobalExecutor::spawn(contract::contract_handling(contract_handler)); diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index 0ec5df0ed..d9f0d3416 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -56,6 +56,7 @@ impl NodeLabel { Self(format!("node-{id}").into()) } + #[allow(dead_code)] fn is_gateway(&self) -> bool { self.0.starts_with("gateway") } @@ -360,15 +361,19 @@ impl SimNetwork { } /// Builds an histogram of the distribution in the ring of each node relative to each other. - pub fn ring_distribution(&self, scale: i32) -> impl Iterator { + pub fn ring_distribution(&self, scale: i32) -> Vec<(f64, usize)> { let mut all_dists = Vec::with_capacity(self.labels.len()); for (.., key) in &self.labels { all_dists.push(self.event_listener.connections(*key)); } - group_locations_in_buckets( + let mut dist_buckets = group_locations_in_buckets( all_dists.into_iter().flatten().map(|(_, l)| l.as_f64()), scale, ) + .collect::>(); + dist_buckets + .sort_by(|(d0, _), (d1, _)| d0.partial_cmp(d1).unwrap_or(std::cmp::Ordering::Equal)); + dist_buckets } /// Returns the connectivity in the network per peer (that is all the connections @@ -405,6 +410,72 @@ impl SimNetwork { } Ok(()) } + + pub async fn check_connectivity(&self, time_out: Duration) -> Result<(), anyhow::Error> { + let num_nodes = self.nodes.capacity(); + let mut connected = HashSet::new(); + let elapsed = Instant::now(); + while elapsed.elapsed() < time_out && connected.len() < num_nodes { + for node in 0..num_nodes { + if !connected.contains(&node) && self.connected(&NodeLabel::node(node)) { + connected.insert(node); + } + } + } + tokio::time::sleep(Duration::from_millis(1_000)).await; + let expected = HashSet::from_iter(0..num_nodes); + let mut missing: Vec<_> = expected + .difference(&connected) + .map(|n| format!("node-{}", n)) + .collect(); + + let node_connectivity = self.node_connectivity(); + let connections = pretty_print_connections(&node_connectivity); + tracing::info!("Number of simulated nodes: {num_nodes}"); + tracing::info!("{connections}"); + + if !missing.is_empty() { + missing.sort(); + tracing::error!("Nodes without connection: {:?}", missing); + tracing::error!("Total nodes without connection: {:?}", missing.len()); + anyhow::bail!("found disconnected nodes"); + } + + tracing::info!( + "Required time for connecting all peers: {} secs", + elapsed.elapsed().as_secs() + ); + + let hist = self.ring_distribution(1); + tracing::info!("Ring distribution: {:?}", hist); + + Ok(()) + } + + pub fn connections_per_peer(&self) -> Result<(), anyhow::Error> { + let num_nodes = self.nodes.capacity(); + let node_connectivity = self.node_connectivity(); + + let mut connections_per_peer: Vec<_> = node_connectivity + .iter() + .map(|(k, v)| (k, v.1.len())) + .filter_map(|(k, v)| if !k.is_gateway() { Some(v) } else { None }) + .collect(); + + // ensure at least normal nodes have more than one connection + connections_per_peer.sort_unstable_by_key(|num_conn| *num_conn); + if *connections_per_peer.iter().last().unwrap() < 2 { + anyhow::bail!("low connectivy; some nodes didn't connect beyond the gateway"); + } + + // ensure the average number of connections per peer is above N + let avg_connections: usize = connections_per_peer.iter().sum::() / num_nodes; + tracing::info!("Average connections: {}", avg_connections); + if avg_connections < 1 { + anyhow::bail!("average number of connections is low"); + } + Ok(()) + } } impl Drop for SimNetwork { @@ -439,67 +510,6 @@ fn group_locations_in_buckets( .map(move |(k, v)| ((k as f64 / (10.0f64).powi(scale)), v)) } -pub(crate) async fn check_connectivity( - sim_nodes: &SimNetwork, - num_nodes: usize, - time_out: Duration, -) -> Result<(), anyhow::Error> { - let mut connected = HashSet::new(); - let elapsed = Instant::now(); - while elapsed.elapsed() < time_out && connected.len() < num_nodes { - for node in 0..num_nodes { - if !connected.contains(&node) && sim_nodes.connected(&NodeLabel::node(node)) { - connected.insert(node); - } - } - } - tokio::time::sleep(Duration::from_millis(1_000)).await; - let expected = HashSet::from_iter(0..num_nodes); - let mut missing: Vec<_> = expected - .difference(&connected) - .map(|n| format!("node-{}", n)) - .collect(); - - let node_connectivity = sim_nodes.node_connectivity(); - let connections = pretty_print_connections(&node_connectivity); - tracing::info!("{connections}"); - - if !missing.is_empty() { - missing.sort(); - tracing::error!("Nodes without connection: {:?}", missing); - tracing::error!("Total nodes without connection: {:?}", missing.len()); - anyhow::bail!("found disconnected nodes"); - } - - tracing::info!( - "Required time for connecting all peers: {} secs", - elapsed.elapsed().as_secs() - ); - - let hist: Vec<_> = sim_nodes.ring_distribution(1).collect(); - tracing::info!("Ring distribution: {:?}", hist); - - let mut connections_per_peer: Vec<_> = node_connectivity - .iter() - .map(|(k, v)| (k, v.1.len())) - .filter_map(|(k, v)| if !k.is_gateway() { Some(v) } else { None }) - .collect(); - - // ensure at least some normal nodes have more than one connection - connections_per_peer.sort_unstable_by_key(|num_conn| *num_conn); - if *connections_per_peer.iter().last().unwrap() < 1 { - anyhow::bail!("low connectivy; nodes didn't connect beyond the gateway"); - } - - // ensure the average number of connections per peer is above N - let avg_connections: usize = connections_per_peer.iter().sum::() / num_nodes; - tracing::info!("Average connections: {}", avg_connections); - if avg_connections < 1 { - anyhow::bail!("average number of connections is low"); - } - Ok(()) -} - fn pretty_print_connections( conns: &HashMap)>, ) -> String { @@ -507,9 +517,6 @@ fn pretty_print_connections( let mut conns = conns.iter().collect::>(); conns.sort_by(|(a, _), (b, _)| a.cmp(b)); for (peer, (key, conns)) in conns { - if peer.is_gateway() { - continue; - } writeln!(&mut connections, "{peer} ({key}):").unwrap(); for (conn, dist) in conns { let dist = dist.as_f64(); diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 0c5129470..aa2731816 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -814,7 +814,7 @@ mod test { use std::collections::HashMap; use super::*; - use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; + use crate::node::tests::{NodeSpecification, SimNetwork}; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_get_op_between_nodes() -> Result<(), anyhow::Error> { @@ -851,7 +851,7 @@ mod test { let get_specs = HashMap::from_iter([("node-0".into(), node_0), ("gateway-0".into(), gw_0)]); // establish network - let mut sim_nodes = SimNetwork::new( + let mut sim_nw = SimNetwork::new( "successful_get_op_between_nodes", NUM_GW, NUM_NODES, @@ -861,15 +861,15 @@ mod test { 2, ) .await; - sim_nodes.start_with_spec(get_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + sim_nw.start_with_spec(get_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; // trigger get @ node-0, which does not own the contract - sim_nodes + sim_nw .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(100))) .await?; tokio::time::sleep(Duration::from_millis(200)).await; - assert!(sim_nodes.has_got_contract(&"node-0".into(), &key)); + assert!(sim_nw.has_got_contract(&"node-0".into(), &key)); Ok(()) } @@ -898,16 +898,16 @@ mod test { let get_specs = HashMap::from_iter([("node-1".into(), node_1)]); // establish network - let mut sim_nodes = + let mut sim_nw = SimNetwork::new("get_contract_not_found", NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - sim_nodes.start_with_spec(get_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + sim_nw.start_with_spec(get_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; // trigger get @ node-1, which does not own the contract - sim_nodes + sim_nw .trigger_event(&"node-1".into(), 1, Some(Duration::from_millis(100))) .await?; - assert!(!sim_nodes.has_got_contract(&"node-1".into(), &key)); + assert!(!sim_nw.has_got_contract(&"node-1".into(), &key)); Ok(()) } @@ -959,7 +959,7 @@ mod test { ]); // establish network - let mut sim_nodes = SimNetwork::new( + let mut sim_nw = SimNetwork::new( "get_contract_found_after_retry", NUM_GW, NUM_NODES, @@ -969,13 +969,13 @@ mod test { 3, ) .await; - sim_nodes.start_with_spec(get_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + sim_nw.start_with_spec(get_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; - sim_nodes + sim_nw .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(500))) .await?; - assert!(sim_nodes.has_got_contract(&"node-0".into(), &key)); + assert!(sim_nw.has_got_contract(&"node-0".into(), &key)); Ok(()) } } diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index d33fd98ed..c8f70ba2e 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -432,13 +432,13 @@ impl Operation for JoinRingOp { Some(JRState::AwaitingProxyResponse { accepted_by: mut previously_accepted, new_peer_id, - target: state_target, + target: original_target, new_location, }) => { // Check if the response reached the target node and if the request // has been accepted by any node let is_accepted = !accepted_by.is_empty(); - let is_target_peer = new_peer_id == state_target.peer; + let is_target_peer = new_peer_id == original_target.peer; if is_accepted { previously_accepted.extend(accepted_by.drain()); @@ -460,11 +460,11 @@ impl Operation for JoinRingOp { "Sending response to join request with all the peers that accepted \ connection from gateway {} to peer {}", target.peer, - state_target.peer + original_target.peer ); return_msg = Some(JoinRingMsg::Response { id, - target: state_target, + target: original_target, sender: target, msg: JoinResponse::AcceptedBy { peers: previously_accepted, @@ -478,12 +478,12 @@ impl Operation for JoinRingOp { "Sending response to join request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", target.peer, - state_target.peer + original_target.peer ); return_msg = Some(JoinRingMsg::Response { id, - target: state_target, + target: original_target, sender: target, msg: JoinResponse::Proxy { accepted_by: previously_accepted, @@ -1051,7 +1051,7 @@ mod messages { mod test { use std::time::Duration; - use crate::node::tests::{check_connectivity, SimNetwork}; + use crate::node::tests::SimNetwork; /// Given a network of one node and one gateway test that both are connected. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1068,18 +1068,18 @@ mod test { crate::config::set_logger(); const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; - let mut sim_nodes = SimNetwork::new( + let mut sim_nw = SimNetwork::new( "join_forward_connection_to_node", NUM_GW, NUM_NODES, 3, 2, - 8, 5, + 3, ) .await; - sim_nodes.start().await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(10)).await + sim_nw.start().await; + sim_nw.check_connectivity(Duration::from_secs(10)).await } /// Given a network of N peers all nodes should have connections. @@ -1088,7 +1088,7 @@ mod test { async fn all_nodes_should_connect() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 10usize; const NUM_GW: usize = 1usize; - let mut sim_nodes = SimNetwork::new( + let mut sim_nw = SimNetwork::new( "join_all_nodes_should_connect", NUM_GW, NUM_NODES, @@ -1098,7 +1098,7 @@ mod test { 2, ) .await; - sim_nodes.start().await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(10)).await + sim_nw.start().await; + sim_nw.check_connectivity(Duration::from_secs(10)).await } } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 72102ce40..b1c00fa67 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -15,7 +15,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::ContractHandlerEvent, - message::{InnerMessage, Message, Transaction }, + message::{InnerMessage, Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{Location, PeerKeyLocation, RingError}, @@ -597,11 +597,7 @@ async fn try_to_broadcast( Ok((new_state, return_msg)) } -pub(crate) fn start_op( - contract: ContractContainer, - value: WrappedState, - htl: usize, -) -> PutOp { +pub(crate) fn start_op(contract: ContractContainer, value: WrappedState, htl: usize) -> PutOp { let key = contract.key(); let contract_location = Location::from(&key); tracing::debug!( @@ -926,7 +922,7 @@ mod test { use freenet_stdlib::prelude::*; use super::*; - use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; + use crate::node::tests::{NodeSpecification, SimNetwork}; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_put_op_between_nodes() -> Result<(), anyhow::Error> { @@ -941,7 +937,7 @@ mod test { let contract_val: WrappedState = gen.arbitrary()?; let new_value = WrappedState::new(Vec::from_iter(gen.arbitrary::<[u8; 20]>().unwrap())); - let mut sim_nodes = SimNetwork::new( + let mut sim_nw = SimNetwork::new( "successful_put_op_between_nodes", NUM_GW, NUM_NODES, @@ -951,7 +947,7 @@ mod test { 2, ) .await; - let mut locations = sim_nodes.get_locations_by_node(); + let mut locations = sim_nw.get_locations_by_node(); let node0_loc = locations.remove(&"node-0".into()).unwrap(); let node1_loc = locations.remove(&"node-1".into()).unwrap(); @@ -1000,16 +996,16 @@ mod test { ("gateway-0".into(), gw_0), ]); - sim_nodes.start_with_spec(put_specs).await; + sim_nw.start_with_spec(put_specs).await; tokio::time::sleep(Duration::from_secs(5)).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; // trigger the put op @ gw-0 - sim_nodes + sim_nw .trigger_event(&"gateway-0".into(), 1, Some(Duration::from_secs(3))) .await?; - assert!(sim_nodes.has_put_contract(&"gateway-0".into(), &key, &new_value)); - assert!(sim_nodes.event_listener.contract_broadcasted(&key)); + assert!(sim_nw.has_put_contract(&"gateway-0".into(), &key, &new_value)); + assert!(sim_nw.event_listener.contract_broadcasted(&key)); Ok(()) } } diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 28f1276bc..3f443ce3a 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -469,7 +469,7 @@ mod test { use freenet_stdlib::client_api::ContractRequest; use super::*; - use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; + use crate::node::tests::{NodeSpecification, SimNetwork}; #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -509,7 +509,7 @@ mod test { ("node-0".into(), first_node), ("node-1".into(), second_node), ]); - let mut sim_nodes = SimNetwork::new( + let mut sim_nw = SimNetwork::new( "successful_subscribe_op_between_nodes", NUM_GW, NUM_NODES, @@ -519,8 +519,8 @@ mod test { 2, ) .await; - sim_nodes.start_with_spec(subscribe_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + sim_nw.start_with_spec(subscribe_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; Ok(()) } From 2d6697cc23c6e6cce2e4b01087516104ee6cc6f1 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 19 Oct 2023 13:19:12 +0200 Subject: [PATCH 07/14] Cleanup a bunch of unnecessary code and todos --- Cargo.lock | 11 +- apps/freenet-email-app/web/src/api.rs | 2 +- crates/core/Cargo.toml | 2 +- crates/core/src/client_events/websocket.rs | 2 +- crates/core/src/config.rs | 17 ++- crates/core/src/contract.rs | 2 +- crates/core/src/contract/executor.rs | 13 +- crates/core/src/contract/handler.rs | 68 ++++----- crates/core/src/contract/in_memory.rs | 31 +--- .../core/src/contract/storages/in_memory.rs | 41 ------ crates/core/src/contract/storages/mod.rs | 3 - crates/core/src/node.rs | 6 +- .../core/src/node/conn_manager/p2p_protoc.rs | 5 +- crates/core/src/node/in_memory_impl.rs | 2 +- .../node/{op_state.rs => op_state_manager.rs} | 9 +- crates/core/src/node/tests.rs | 65 ++++++--- crates/core/src/operations/get.rs | 2 +- crates/core/src/operations/join_ring.rs | 118 ++++++++++----- crates/core/src/ring.rs | 1 + crates/fdev/src/build.rs | 134 ++++++++---------- crates/fdev/src/inspect.rs | 2 - crates/fdev/src/local_node/commands.rs | 10 -- crates/fdev/src/new_package.rs | 4 +- 23 files changed, 268 insertions(+), 282 deletions(-) delete mode 100644 crates/core/src/contract/storages/in_memory.rs rename crates/core/src/node/{op_state.rs => op_state_manager.rs} (97%) diff --git a/Cargo.lock b/Cargo.lock index 15bb518e4..9e82e978f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1600,7 +1600,7 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry-jaeger", - "ordered-float 3.9.2", + "ordered-float 4.1.1", "parking_lot", "pav_regression", "pico-args", @@ -3450,6 +3450,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" diff --git a/apps/freenet-email-app/web/src/api.rs b/apps/freenet-email-app/web/src/api.rs index 8a5e21b00..642ba9ab3 100644 --- a/apps/freenet-email-app/web/src/api.rs +++ b/apps/freenet-email-app/web/src/api.rs @@ -39,7 +39,7 @@ impl WebApi { #[cfg(all(not(target_family = "wasm"), feature = "use-node"))] fn new() -> Result { - todo!() + unimplemented!() } #[cfg(all(target_family = "wasm", feature = "use-node"))] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 61560a371..4f4126651 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -66,7 +66,7 @@ sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio-rustls"], optiona rocksdb = { version = "0.21.0", default-features = false, optional = true } pav_regression = "0.4.0" itertools = "0.11" -ordered-float = "3.9.1" +ordered-float = "4.1.1" notify = "6" wasmer = { workspace = true, features = [ "sys"] } chrono = { workspace = true } diff --git a/crates/core/src/client_events/websocket.rs b/crates/core/src/client_events/websocket.rs index c2840438a..47f7b4b12 100644 --- a/crates/core/src/client_events/websocket.rs +++ b/crates/core/src/client_events/websocket.rs @@ -474,7 +474,7 @@ impl ClientEventsProxy for WebSocketProxy { break Ok(reply.into_owned()); } } else { - todo!() + break Err(ClientError::from(ErrorKind::ChannelClosed)); } } } diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 463414add..f12094421 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -143,10 +143,6 @@ impl ConfigPaths { } impl Config { - pub fn set_from_cli() -> Result<(), DynError> { - todo!() - } - pub fn set_op_mode(mode: OperationMode) { let local_mode = matches!(mode, OperationMode::Local); Self::conf() @@ -324,6 +320,19 @@ impl libp2p::swarm::Executor for GlobalExecutor { } pub fn set_logger() { + static LOGGER_SET: AtomicBool = AtomicBool::new(false); + if LOGGER_SET + .compare_exchange( + false, + true, + std::sync::atomic::Ordering::Acquire, + std::sync::atomic::Ordering::SeqCst, + ) + .is_err() + { + return; + } + let sub = tracing_subscriber::fmt().with_level(true).with_env_filter( tracing_subscriber::EnvFilter::builder() .with_default_directive(tracing_subscriber::filter::LevelFilter::DEBUG.into()) diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index 28bf08907..b018490fd 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -13,7 +13,7 @@ pub(crate) use executor::{ }; pub(crate) use handler::{ contract_handler_channel, ClientResponses, ClientResponsesSender, ContractHandler, - ContractHandlerEvent, ContractHandlerToEventLoopChannel, EventId, NetEventListener, + ContractHandlerEvent, ContractHandlerToEventLoopChannel, EventId, NetEventListenerHalve, NetworkContractHandler, StoreResponse, }; #[cfg(test)] diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index c37c5c5f8..82526414d 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -124,14 +124,6 @@ pub(crate) fn executor_channel( (listener_halve, sender_halve) } -#[cfg(test)] -pub(crate) fn executor_channel_test() -> ( - ExecutorToEventLoopChannel, - ExecutorToEventLoopChannel, -) { - todo!() -} - impl ExecutorToEventLoopChannel { async fn send_to_event_loop(&mut self, message: T) -> Result<(), DynError> where @@ -1200,6 +1192,7 @@ impl Executor { let contracts_data_dir = tmp_path.join("contracts"); let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; + // uses inmemory SQLite let state_store = StateStore::new(Storage::new().await?, u16::MAX as u32).unwrap(); let executor = Executor::new( @@ -1218,7 +1211,7 @@ impl Executor { _req: ClientRequest<'a>, _updates: Option>>, ) -> Response { - todo!() + unreachable!() } } @@ -1313,7 +1306,7 @@ impl ContractExecutor for Executor { .await?; return Ok(state); } - _ => todo!(), + _ => unreachable!(), } } } diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index c2dcc92f7..20a90c702 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -1,6 +1,5 @@ #![allow(unused)] // FIXME: remove this - -use std::collections::{BTreeMap, VecDeque}; +use std::collections::BTreeMap; use std::hash::Hash; use std::marker::PhantomData; use std::sync::atomic::{AtomicU64, Ordering::SeqCst}; @@ -19,15 +18,7 @@ use super::{ }; use crate::client_events::HostResult; use crate::message::Transaction; -use crate::node::OpManager; -use crate::{ - client_events::ClientId, - node::NodeConfig, - runtime::{ContractStore, Runtime, StateStorage, StateStore}, - DynError, -}; - -pub const MAX_MEM_CACHE: i64 = 10_000_000; +use crate::{client_events::ClientId, node::NodeConfig, runtime::Runtime, DynError}; pub(crate) struct ClientResponses(UnboundedReceiver<(ClientId, HostResult)>); @@ -188,12 +179,23 @@ impl ContractHandler for NetworkContractHandler { pub(crate) struct EventId { id: u64, client_id: Option, + transaction: Option, } impl EventId { pub fn client_id(&self) -> Option { self.client_id } + + pub fn transaction(&self) -> Option { + self.transaction + } + + // FIXME: this should be used somewhere to inform than an event is pending + // a transaction resolution + pub fn with_transaction(&mut self, transaction: Transaction) { + self.transaction = Some(transaction); + } } impl PartialEq for EventId { @@ -218,17 +220,17 @@ pub(crate) struct ContractHandlerToEventLoopChannel { } pub(crate) struct ContractHandlerHalve; -pub(crate) struct NetEventListener; +pub(crate) struct NetEventListenerHalve; mod sealed { - use super::{ContractHandlerHalve, NetEventListener}; + use super::{ContractHandlerHalve, NetEventListenerHalve}; pub(crate) trait ChannelHalve {} impl ChannelHalve for ContractHandlerHalve {} - impl ChannelHalve for NetEventListener {} + impl ChannelHalve for NetEventListenerHalve {} } pub(crate) fn contract_handler_channel() -> ( - ContractHandlerToEventLoopChannel, + ContractHandlerToEventLoopChannel, ContractHandlerToEventLoopChannel, ) { let (notification_tx, notification_channel) = mpsc::unbounded_channel(); @@ -258,7 +260,7 @@ static EV_ID: AtomicU64 = AtomicU64::new(0); // kind of event and can be optimized on a case basis const CH_EV_RESPONSE_TIME_OUT: Duration = Duration::from_secs(300); -impl ContractHandlerToEventLoopChannel { +impl ContractHandlerToEventLoopChannel { /// Send an event to the contract handler and receive a response event if successful. pub async fn send_to_handler( &mut self, @@ -289,7 +291,7 @@ impl ContractHandlerToEventLoopChannel { } } - pub async fn recv_from_handler(&mut self) -> (EventId, ContractHandlerEvent) { + pub async fn recv_from_handler(&mut self) -> EventId { todo!() } } @@ -317,6 +319,7 @@ impl ContractHandlerToEventLoopChannel { EventId { id: msg.id, client_id: msg.client_id, + transaction: None, }, msg.ev, )); @@ -369,9 +372,7 @@ impl std::fmt::Display for ContractHandlerEvent { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ContractHandlerEvent::PutQuery { - key, - state, - parameters, + key, parameters, .. } => { if let Some(params) = parameters { write!(f, "put query {{ {key}, params: {:?} }}", params.as_ref()) @@ -394,10 +395,10 @@ impl std::fmt::Display for ContractHandlerEvent { write!(f, "get query {{ {key}, fetch contract: {fetch_contract} }}",) } ContractHandlerEvent::GetResponse { key, response } => match response { - Ok(v) => { + Ok(_) => { write!(f, "get query response {{ {key} }}",) } - Err(e) => { + Err(_) => { write!(f, "get query failed {{ {key} }}",) } }, @@ -411,24 +412,14 @@ impl std::fmt::Display for ContractHandlerEvent { } } -impl ContractHandlerEvent { - pub async fn into_network_op(self, op_manager: &OpManager) -> Transaction { - todo!() - } -} - #[cfg(test)] pub mod test { use std::sync::Arc; - use crate::runtime::ContractStore; - use freenet_stdlib::{ - client_api::{ClientRequest, ContractRequest, HostResponse}, - prelude::*, - }; + use freenet_stdlib::prelude::*; use super::*; - use crate::{config::GlobalExecutor, contract::MockRuntime}; + use crate::config::GlobalExecutor; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn channel_test() -> Result<(), anyhow::Error> { @@ -466,13 +457,4 @@ pub mod test { Ok(()) } - - // Prepare and get handler for an in-memory sqlite db - async fn get_handler(test: &str) -> Result, DynError> { - let (_, ch_handler) = contract_handler_channel(); - let (_, executor_sender) = super::super::executor::executor_channel_test(); - let handler = - NetworkContractHandler::build(ch_handler, executor_sender, test.to_owned()).await?; - Ok(handler) - } } diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs index 8fa030941..1fc7149ac 100644 --- a/crates/core/src/contract/in_memory.rs +++ b/crates/core/src/contract/in_memory.rs @@ -1,7 +1,4 @@ -use crate::{ - client_events::ClientId, - runtime::{ContractStore, StateStorage, StateStore}, -}; +use crate::{client_events::ClientId, runtime::ContractStore}; use freenet_stdlib::{ client_api::{ClientError, ClientRequest, HostResponse}, prelude::WrappedContract, @@ -12,7 +9,6 @@ use tokio::sync::mpsc::UnboundedSender; use super::{ executor::{ExecutorHalve, ExecutorToEventLoopChannel}, handler::{ContractHandler, ContractHandlerHalve, ContractHandlerToEventLoopChannel}, - storages::in_memory::MemKVStore, Executor, }; use crate::DynError; @@ -21,34 +17,18 @@ pub(crate) struct MockRuntime { pub contract_store: ContractStore, } -pub(crate) struct MemoryContractHandler -where - KVStore: StateStorage, -{ +pub(crate) struct MemoryContractHandler { channel: ContractHandlerToEventLoopChannel, - _kv_store: StateStore, runtime: Executor, } -impl MemoryContractHandler -where - KVStore: StateStorage + Send + Sync + 'static, - ::Error: Into>, -{ +impl MemoryContractHandler { pub async fn new( channel: ContractHandlerToEventLoopChannel, - kv_store: KVStore, data_dir: &str, ) -> Self { - // let rt = MockRuntime { - // contract_store: ContractStore::new( - // Config::conf().contracts_dir(), - // Self::MAX_MEM_CACHE, - // ) - // .unwrap(); MemoryContractHandler { channel, - _kv_store: StateStore::new(kv_store, 10_000_000).unwrap(), runtime: Executor::new_mock(data_dir).await.unwrap(), } } @@ -66,8 +46,7 @@ impl ContractHandler for MemoryContractHandler { where Self: Sized + 'static, { - let store = MemKVStore::new(); - async move { Ok(MemoryContractHandler::new(channel, store, &config).await) }.boxed() + async move { Ok(MemoryContractHandler::new(channel, &config).await) }.boxed() } fn channel(&mut self) -> &mut ContractHandlerToEventLoopChannel { @@ -80,7 +59,7 @@ impl ContractHandler for MemoryContractHandler { _client_id: ClientId, _updates: Option>>, ) -> BoxFuture<'static, Result> { - todo!() + unreachable!() } fn executor(&mut self) -> &mut Self::ContractExecutor { diff --git a/crates/core/src/contract/storages/in_memory.rs b/crates/core/src/contract/storages/in_memory.rs deleted file mode 100644 index c9c66280e..000000000 --- a/crates/core/src/contract/storages/in_memory.rs +++ /dev/null @@ -1,41 +0,0 @@ -use dashmap::DashMap; -use freenet_stdlib::prelude::*; - -use crate::runtime::StateStorage; - -#[derive(Default, Clone)] -pub(crate) struct MemKVStore(DashMap); - -#[async_trait::async_trait] -impl StateStorage for MemKVStore { - type Error = String; - - async fn store(&mut self, _key: ContractKey, _state: WrappedState) -> Result<(), Self::Error> { - todo!() - } - - async fn get(&self, _key: &ContractKey) -> Result, Self::Error> { - todo!() - } - - async fn store_params( - &mut self, - _key: ContractKey, - _state: Parameters<'static>, - ) -> Result<(), Self::Error> { - todo!() - } - - async fn get_params<'a>( - &'a self, - _key: &'a ContractKey, - ) -> Result>, Self::Error> { - todo!() - } -} - -impl MemKVStore { - pub fn new() -> Self { - Self::default() - } -} diff --git a/crates/core/src/contract/storages/mod.rs b/crates/core/src/contract/storages/mod.rs index 48bc02864..e55e61c94 100644 --- a/crates/core/src/contract/storages/mod.rs +++ b/crates/core/src/contract/storages/mod.rs @@ -13,6 +13,3 @@ use self::rocks_db::RocksDb; #[cfg(all(feature = "rocks_db", not(feature = "sqlite")))] pub type Storage = RocksDb; - -#[cfg(test)] -pub(crate) mod in_memory; diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index ebc372f60..510b44d91 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -44,13 +44,13 @@ use crate::{ use crate::operations::handle_op_request; pub(crate) use conn_manager::{ConnectionBridge, ConnectionError}; pub(crate) use event_log::{EventLogRegister, EventRegister}; -pub(crate) use op_state::OpManager; +pub(crate) use op_state_manager::OpManager; mod conn_manager; mod event_log; #[cfg(test)] mod in_memory_impl; -mod op_state; +mod op_state_manager; mod p2p_impl; #[cfg(test)] pub(crate) mod tests; @@ -388,7 +388,7 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { // Initialize a subscribe op. loop { - // FIXME: this will block the event loop until the subscribe op succeeds + // FIXME: this will block the loop until the subscribe op succeeds // instead the op should be deferred for later execution let op = subscribe::start_op(key.clone()); match subscribe::request_subscribe(&op_storage_cp, op, Some(client_id)) diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index 674c7f235..d7c0c0b64 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -399,9 +399,8 @@ impl P2pConnManager { msg = network_msg => { msg } msg = notification_msg => { msg } msg = bridge_msg => { msg } - (event_id, contract_handler_event) = op_manager.recv_from_handler() => { - if let Some(client_id) = event_id.client_id() { - let transaction = contract_handler_event.into_network_op(&op_manager).await; + event_id = op_manager.recv_from_handler() => { + if let Some((client_id, transaction)) = event_id.client_id().zip(event_id.transaction()) { tx_to_client.insert(transaction, client_id); } continue; diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index da2d4ce75..47246894d 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -7,7 +7,7 @@ use super::{ client_event_handling, conn_manager::{in_memory::MemoryConnManager, EventLoopNotifications}, handle_cancelled_op, join_ring_request, - op_state::OpManager, + op_state_manager::OpManager, process_message, EventLogRegister, PeerKey, }; use crate::{ diff --git a/crates/core/src/node/op_state.rs b/crates/core/src/node/op_state_manager.rs similarity index 97% rename from crates/core/src/node/op_state.rs rename to crates/core/src/node/op_state_manager.rs index 8b7860e91..addb61193 100644 --- a/crates/core/src/node/op_state.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -7,7 +7,8 @@ use tokio::sync::{mpsc::error::SendError, Mutex}; use crate::{ contract::{ - ContractError, ContractHandlerEvent, ContractHandlerToEventLoopChannel, NetEventListener, + ContractError, ContractHandlerEvent, ContractHandlerToEventLoopChannel, + NetEventListenerHalve, }, dev_tool::ClientId, message::{Message, Transaction, TransactionType}, @@ -28,7 +29,7 @@ pub(crate) struct OpManager { subscribe: DashMap, to_event_listener: EventLoopNotificationsSender, // todo: remove the need for a mutex here - ch_outbound: Mutex>, + ch_outbound: Mutex>, // FIXME: think of an optimal strategy to check for timeouts and clean up garbage _ops_ttl: RwLock>>, pub ring: Ring, @@ -47,7 +48,7 @@ impl OpManager { pub(super) fn new( ring: Ring, notification_channel: EventLoopNotificationsSender, - contract_handler: ContractHandlerToEventLoopChannel, + contract_handler: ContractHandlerToEventLoopChannel, ) -> Self { Self { join_ring: DashMap::default(), @@ -101,7 +102,7 @@ impl OpManager { .await } - pub async fn recv_from_handler(&self) -> (crate::contract::EventId, ContractHandlerEvent) { + pub async fn recv_from_handler(&self) -> crate::contract::EventId { todo!() } diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index d9f0d3416..4bccb574b 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -56,10 +56,13 @@ impl NodeLabel { Self(format!("node-{id}").into()) } - #[allow(dead_code)] fn is_gateway(&self) -> bool { self.0.starts_with("gateway") } + + pub fn is_node(&self) -> bool { + self.0.starts_with("node") + } } impl std::fmt::Display for NodeLabel { @@ -158,6 +161,10 @@ impl SimNetwork { net } + pub fn with_start_backoff(&mut self, value: Duration) { + self.init_backoff = value; + } + #[allow(unused)] pub fn debug(&mut self) { self.debug = true; @@ -285,12 +292,8 @@ impl SimNetwork { for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { let node_spec = specs.remove(&label); self.initialize_peer(node, label, node_spec); - if gw_not_init != 0 { - gw_not_init -= 1; - tokio::time::sleep(self.init_backoff).await; - } else { - tokio::time::sleep(self.init_backoff).await; - } + gw_not_init = gw_not_init.saturating_sub(1); + tokio::time::sleep(self.init_backoff).await; } } @@ -411,6 +414,8 @@ impl SimNetwork { Ok(()) } + /// Checks that all peers in the network have acquired at least one connection to any + /// other peers. pub async fn check_connectivity(&self, time_out: Duration) -> Result<(), anyhow::Error> { let num_nodes = self.nodes.capacity(); let mut connected = HashSet::new(); @@ -452,27 +457,55 @@ impl SimNetwork { Ok(()) } - pub fn connections_per_peer(&self) -> Result<(), anyhow::Error> { + /// Recommended to calling after `check_connectivity` to ensure enough time + /// elapsed for all peers to become connected. + /// + /// Checks that there is a good connectivity over the simulated network, + /// meaning that: + /// + /// - at least 50% of the peers have more than the minimum connections + /// - + pub fn network_connectivity_quality(&self) -> Result<(), anyhow::Error> { + const HIGHER_THAN_MIN_THRESHOLD: f64 = 0.5; let num_nodes = self.nodes.capacity(); + let min_connections_threshold = (num_nodes as f64 * HIGHER_THAN_MIN_THRESHOLD) as usize; let node_connectivity = self.node_connectivity(); let mut connections_per_peer: Vec<_> = node_connectivity .iter() .map(|(k, v)| (k, v.1.len())) - .filter_map(|(k, v)| if !k.is_gateway() { Some(v) } else { None }) + .filter(|&(k, _)| !k.is_gateway()) + .map(|(_, v)| v) .collect(); - // ensure at least normal nodes have more than one connection + // ensure at least "most" normal nodes have more than one connection connections_per_peer.sort_unstable_by_key(|num_conn| *num_conn); - if *connections_per_peer.iter().last().unwrap() < 2 { - anyhow::bail!("low connectivy; some nodes didn't connect beyond the gateway"); + if connections_per_peer[min_connections_threshold] < self.min_connections { + tracing::error!( + "Low connectivity; more than {:.0}% of the nodes don't have more than minimum connections", + HIGHER_THAN_MIN_THRESHOLD * 100.0 + ); + anyhow::bail!("low connectivity"); + } else { + let idx = connections_per_peer[min_connections_threshold..] + .iter() + .position(|num_conn| *num_conn < self.min_connections) + .unwrap_or_else(|| connections_per_peer[min_connections_threshold..].len() - 1) + + (min_connections_threshold - 1); + let percentile = idx as f64 / connections_per_peer.len() as f64 * 100.0; + tracing::info!("{percentile:.0}% nodes have higher than required minimum connections"); } - // ensure the average number of connections per peer is above N + // ensure the average number of connections per peer is above the mean between max and min connections + let expected_avg_connections = + ((self.max_connections - self.min_connections) / 2) + self.min_connections; let avg_connections: usize = connections_per_peer.iter().sum::() / num_nodes; - tracing::info!("Average connections: {}", avg_connections); - if avg_connections < 1 { - anyhow::bail!("average number of connections is low"); + tracing::info!( + "Average connections: {avg_connections} (expected > {expected_avg_connections})" + ); + if avg_connections < expected_avg_connections { + tracing::error!("Average number of connections ({avg_connections}) is low (< {expected_avg_connections})"); + anyhow::bail!("low average number of connections"); } Ok(()) } diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index aa2731816..296716230 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -132,7 +132,7 @@ impl TryFrom for GetResult { fn try_from(value: GetOp) -> Result { match value.result { Some(r) => Ok(r), - _ => todo!(), + _ => Err(OpError::UnexpectedOpState), } } } diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/join_ring.rs index c8f70ba2e..0039e6b6f 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/join_ring.rs @@ -43,16 +43,37 @@ impl JoinRingOp { pub(super) fn record_transfer(&mut self) {} } +/// Not really used since client requests will never interact with this directly. pub(crate) struct JoinRingResult {} impl TryFrom for JoinRingResult { type Error = OpError; fn try_from(_value: JoinRingOp) -> Result { - todo!() + Ok(Self {}) } } +/* +Will need to do some changes for when we perform parallel joins. + +t0: (#1 join attempt) +joiner: + 1. dont knows location + 3. knows location + n. connected to the ring +gateway: + 2. assigned_location: None; assigned location to `joiner` based on IP and communicate to joiner + 4. forward to N peers + ... + +(2 join subsequently to acquire more/better connections) +join: + 1. knows location +gateway: + 2. assigned_location: Some(loc) +*/ + impl Operation for JoinRingOp { type Message = JoinRingMsg; type Result = JoinRingResult; @@ -110,8 +131,9 @@ impl Operation for JoinRingOp { msg: JoinRequest::StartReq { target: this_node_loc, - req_peer, + joiner, hops_to_live, + assigned_location, .. }, } => { @@ -119,24 +141,25 @@ impl Operation for JoinRingOp { tracing::debug!( tx = %id, "Initial join request received from {} with HTL {} @ {}", - req_peer, + joiner, hops_to_live, this_node_loc.peer ); - let new_location = Location::random(); + // todo: location should be based on your public IP + let new_location = assigned_location.unwrap_or_else(Location::random); // FIXME: don't try to forward to peers which have already been tried (add a rejected_by list) let accepted_by = if op_storage.ring.should_accept(&new_location) { - tracing::debug!(tx = %id, "Accepting connection from {}", req_peer,); + tracing::debug!(tx = %id, "Accepting connection from {}", joiner,); HashSet::from_iter([this_node_loc]) } else { - tracing::debug!(tx = %id, at_peer = %this_node_loc.peer, "Rejecting connection from peer {}", req_peer); + tracing::debug!(tx = %id, at_peer = %this_node_loc.peer, "Rejecting connection from peer {}", joiner); HashSet::new() }; let new_peer_loc = PeerKeyLocation { location: Some(new_location), - peer: req_peer, + peer: joiner, }; if let Some(mut updated_state) = forward_conn( id, @@ -164,7 +187,7 @@ impl Operation for JoinRingOp { tx = %id, "OC received at gateway {} from requesting peer {}", this_node_loc.peer, - req_peer + joiner ); new_state = Some(JRState::OCReceived); } else { @@ -176,10 +199,10 @@ impl Operation for JoinRingOp { msg: JoinResponse::AcceptedBy { peers: accepted_by, your_location: new_location, - your_peer_id: req_peer, + your_peer_id: joiner, }, target: PeerKeyLocation { - peer: req_peer, + peer: joiner, location: Some(new_location), }, }); @@ -681,7 +704,7 @@ enum JRState { Initializing, Connecting(ConnectionInfo), AwaitingProxyResponse { - /// Could be either the requester or nodes which have been previously forwarded to + /// Could be either the joiner or nodes which have been previously forwarded to target: PeerKeyLocation, accepted_by: HashSet, new_location: Location, @@ -786,11 +809,13 @@ where ); conn_manager.add_connection(gateway.peer).await?; + let assigned_location = op_storage.ring.own_location().location; let join_req = Message::from(messages::JoinRingMsg::Request { id: tx, msg: messages::JoinRequest::StartReq { target: gateway, - req_peer: this_peer, + joiner: this_peer, + assigned_location, hops_to_live: max_hops_to_live, max_hops_to_live, }, @@ -819,7 +844,7 @@ async fn forward_conn( ring: &Ring, conn_manager: &mut CM, req_peer: PeerKeyLocation, - new_peer_loc: PeerKeyLocation, + joiner: PeerKeyLocation, left_htl: usize, num_accepted: usize, ) -> Result, OpError> @@ -829,7 +854,7 @@ where if left_htl == 0 { tracing::debug!( tx = %id, - requester = %req_peer.peer, + joiner = %joiner.peer, "Couldn't forward join petition, no hops left or enough connections", ); return Ok(None); @@ -838,7 +863,7 @@ where if ring.num_connections() == 0 { tracing::warn!( tx = %id, - requester = %req_peer.peer, + joiner = %joiner.peer, "Couldn't forward join petition, not enough connections", ); return Ok(None); @@ -847,25 +872,26 @@ where let forward_to = if left_htl >= ring.rnd_if_htl_above { tracing::debug!( tx = %id, - requester = %req_peer.peer, + joiner = %joiner.peer, "Randomly selecting peer to forward JoinRequest", ); ring.random_peer(|p| p != &req_peer.peer) } else { tracing::debug!( tx = %id, - requester = %req_peer.peer, + joiner = %joiner.peer, "Selecting close peer to forward request", ); - ring.routing(&new_peer_loc.location.unwrap(), Some(&req_peer.peer), &[]) - .and_then(|pkl| (pkl.peer != new_peer_loc.peer).then_some(pkl)) + // FIXME: target the `desired_location` + ring.routing(&joiner.location.unwrap(), Some(&req_peer.peer), &[]) + .and_then(|pkl| (pkl.peer != joiner.peer).then_some(pkl)) }; if let Some(forward_to) = forward_to { let forwarded = Message::from(JoinRingMsg::Request { id, msg: JoinRequest::Proxy { - joiner: new_peer_loc, + joiner, hops_to_live: left_htl.min(ring.max_hops_to_live) - 1, sender: ring.own_location(), }, @@ -881,8 +907,8 @@ where let new_state = JRState::AwaitingProxyResponse { target: req_peer, accepted_by: HashSet::new(), - new_location: new_peer_loc.location.unwrap(), - new_peer_id: new_peer_loc.peer, + new_location: joiner.location.unwrap(), + new_peer_id: joiner.peer, }; Ok(Some(new_state)) } else { @@ -943,7 +969,10 @@ mod messages { Response { sender, .. } => Some(&sender.peer), Connected { sender, .. } => Some(&sender.peer), Request { - msg: JoinRequest::StartReq { req_peer, .. }, + msg: + JoinRequest::StartReq { + joiner: req_peer, .. + }, .. } => Some(req_peer), _ => None, @@ -1004,7 +1033,7 @@ mod messages { .. } => write!(f, "RouteValue(id: {id})"), Self::Connected { .. } => write!(f, "Connected(id: {id})"), - _ => todo!(), + _ => unimplemented!(), } } } @@ -1013,7 +1042,8 @@ mod messages { pub(crate) enum JoinRequest { StartReq { target: PeerKeyLocation, - req_peer: PeerKey, + joiner: PeerKey, + assigned_location: Option, hops_to_live: usize, max_hops_to_live: usize, }, @@ -1065,40 +1095,54 @@ mod test { /// Once a gateway is left without remaining open slots, ensure forwarding connects #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn forward_connection_to_node() -> Result<(), anyhow::Error> { - crate::config::set_logger(); - const NUM_NODES: usize = 10usize; + // crate::config::set_logger(); + const NUM_NODES: usize = 3usize; const NUM_GW: usize = 1usize; let mut sim_nw = SimNetwork::new( "join_forward_connection_to_node", NUM_GW, NUM_NODES, - 3, 2, - 5, - 3, + 1, + 2, + 1, ) .await; + // sim_nw.with_start_backoff(Duration::from_millis(100)); sim_nw.start().await; - sim_nw.check_connectivity(Duration::from_secs(10)).await + sim_nw.check_connectivity(Duration::from_secs(5)).await?; + let some_forwarded = sim_nw + .node_connectivity() + .into_iter() + .flat_map(|(_this, (_, conns))| conns.into_keys()) + .any(|c| c.is_node()); + assert!( + some_forwarded, + "didn't find any connection succesfully forwarded" + ); + Ok(()) } - /// Given a network of N peers all nodes should have connections. + /// Given a network of N peers all good connectivity #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn all_nodes_should_connect() -> Result<(), anyhow::Error> { + async fn network_should_achieve_good_connectivity() -> Result<(), anyhow::Error> { + crate::config::set_logger(); const NUM_NODES: usize = 10usize; - const NUM_GW: usize = 1usize; + const NUM_GW: usize = 2usize; let mut sim_nw = SimNetwork::new( "join_all_nodes_should_connect", NUM_GW, NUM_NODES, + 5, 3, - 2, - 1000, + 6, 2, ) .await; + sim_nw.with_start_backoff(Duration::from_millis(200)); sim_nw.start().await; - sim_nw.check_connectivity(Duration::from_secs(10)).await + sim_nw.check_connectivity(Duration::from_secs(10)).await?; + sim_nw.network_connectivity_quality() } } diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index f933b48d9..8b85e3811 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -264,6 +264,7 @@ impl Ring { tracing::debug!(peer = %self.peer_key, "max connections reached"); false } else { + // todo: in the future maybe use the `small worldness` metric to decide let median_distance = self .median_distance_to(my_location) .unwrap_or(Distance(0.5)); diff --git a/crates/fdev/src/build.rs b/crates/fdev/src/build.rs index d16a6ef90..58b797a07 100644 --- a/crates/fdev/src/build.rs +++ b/crates/fdev/src/build.rs @@ -220,7 +220,7 @@ mod contract { pub lang: Option, pub typescript: Option, #[serde(rename = "state-sources")] - pub state_sources: Option, + pub state_sources: Sources, pub metadata: Option, pub dependencies: Option, } @@ -228,7 +228,6 @@ mod contract { #[derive(Serialize, Deserialize, PartialEq)] #[serde(rename_all = "lowercase")] pub(crate) enum SupportedWebLangs { - Javascript, Typescript, } @@ -243,6 +242,11 @@ mod contract { embedded_deps: EmbeddedDeps, cwd: &Path, ) -> Result<(), DynError> { + let Some(web_config) = &config.webapp else { + println!("No webapp config found."); + return Ok(()); + }; + let metadata = if let Some(md) = config.webapp.as_ref().and_then(|a| a.metadata.as_ref()) { let mut buf = vec![]; File::open(md)?.read_to_end(&mut buf)?; @@ -252,73 +256,68 @@ mod contract { }; let mut archive: Builder>> = Builder::new(Cursor::new(Vec::new())); - if let Some(web_config) = &config.webapp { - println!("Bundling webapp contract state"); - match &web_config.lang { - Some(SupportedWebLangs::Typescript) => { - let child = Command::new("npm") - .args(["install"]) + + println!("Bundling webapp contract state"); + match &web_config.lang { + Some(SupportedWebLangs::Typescript) => { + let child = Command::new("npm") + .args(["install"]) + .current_dir(cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| { + eprintln!("Error while installing npm packages: {e}"); + Error::CommandFailed("npm") + })?; + pipe_std_streams(child)?; + let webpack = web_config + .typescript + .as_ref() + .map(|c| c.webpack) + .unwrap_or_default(); + use std::io::IsTerminal; + if webpack { + let cmd_args: &[&str] = + if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { + &["--color"] + } else { + &[] + }; + let child = Command::new("webpack") + .args(cmd_args) .current_dir(cwd) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .map_err(|e| { - eprintln!("Error while installing npm packages: {e}"); - Error::CommandFailed("npm") + eprintln!("Error while executing webpack command: {e}"); + Error::CommandFailed("tsc") })?; pipe_std_streams(child)?; - let webpack = web_config - .typescript - .as_ref() - .map(|c| c.webpack) - .unwrap_or_default(); - use std::io::IsTerminal; - if webpack { - let cmd_args: &[&str] = - if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { - &["--color"] - } else { - &[] - }; - let child = Command::new("webpack") - .args(cmd_args) - .current_dir(cwd) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| { - eprintln!("Error while executing webpack command: {e}"); - Error::CommandFailed("tsc") - })?; - pipe_std_streams(child)?; - println!("Compiled input using webpack"); - } else { - let cmd_args: &[&str] = - if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { - &["--pretty"] - } else { - &[] - }; - let child = Command::new("tsc") - .args(cmd_args) - .current_dir(cwd) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| { - eprintln!("Error while executing command tsc: {e}"); - Error::CommandFailed("tsc") - })?; - pipe_std_streams(child)?; - println!("Compiled input using tsc"); - } + println!("Compiled input using webpack"); + } else { + let cmd_args: &[&str] = + if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { + &["--pretty"] + } else { + &[] + }; + let child = Command::new("tsc") + .args(cmd_args) + .current_dir(cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| { + eprintln!("Error while executing command tsc: {e}"); + Error::CommandFailed("tsc") + })?; + pipe_std_streams(child)?; + println!("Compiled input using tsc"); } - Some(SupportedWebLangs::Javascript) => todo!(), - None => {} } - } else { - println!("No webapp config found."); - return Ok(()); + None => {} } let build_state = |sources: &Sources| -> Result<(), DynError> { @@ -393,15 +392,8 @@ mod contract { Ok(()) }; - if let Some(sources) = config - .webapp - .as_ref() - .and_then(|a| a.state_sources.as_ref()) - { - build_state(sources) - } else { - todo!() - } + let sources = &web_config.state_sources; + build_state(sources) } fn build_generic_state(config: &mut BuildToolConfig, cwd: &Path) -> Result<(), DynError> { @@ -603,10 +595,10 @@ mod contract { webapp: Some(WebAppContract { lang: Some(SupportedWebLangs::Typescript), typescript: Some(TypescriptConfig { webpack: true }), - state_sources: Some(Sources { + state_sources: Sources { source_dirs: Some(vec!["dist".into()]), files: None, - }), + }, metadata: None, dependencies: Some( toml::toml! { diff --git a/crates/fdev/src/inspect.rs b/crates/fdev/src/inspect.rs index 7da43cab1..623fe63b1 100644 --- a/crates/fdev/src/inspect.rs +++ b/crates/fdev/src/inspect.rs @@ -16,7 +16,6 @@ pub struct InspectCliConfig { enum FileType { Code(CodeInspection), Delegate, - Contract, } /// Inspect the packaged WASM code for Freenet. @@ -47,7 +46,6 @@ delegate API version: {version} "# ); } - FileType::Contract => todo!(), } Ok(()) diff --git a/crates/fdev/src/local_node/commands.rs b/crates/fdev/src/local_node/commands.rs index 00962506e..e479a98fe 100644 --- a/crates/fdev/src/local_node/commands.rs +++ b/crates/fdev/src/local_node/commands.rs @@ -75,7 +75,6 @@ async fn execute_command( })) => { println!("current state for {key}:"); app.printout_deser(state.as_ref())?; - todo!() } Err(err) => { println!("error: {err}"); @@ -85,15 +84,6 @@ async fn execute_command( } _ => unreachable!(), }, - ClientRequest::DelegateOp(op) => { - match node.handle_request(ClientId::FIRST, op.into(), None).await { - Ok(_res) => todo!(), - Err(err) => { - println!("error: {err}"); - } - } - } - ClientRequest::Disconnect { .. } => return Ok(true), _ => { tracing::error!("op not supported"); return Err("op not support".into()); diff --git a/crates/fdev/src/new_package.rs b/crates/fdev/src/new_package.rs index 000104933..44485f0d2 100644 --- a/crates/fdev/src/new_package.rs +++ b/crates/fdev/src/new_package.rs @@ -34,10 +34,10 @@ fn create_view_package(cwd: &Path) -> Result<(), DynError> { webapp: Some(WebAppContract { lang: Some(SupportedWebLangs::Typescript), typescript: Some(TypescriptConfig { webpack: true }), - state_sources: Some(Sources { + state_sources: Sources { source_dirs: Some(vec![PathBuf::from("dist")]), files: None, - }), + }, metadata: None, dependencies: None, }), From fa9f667b61b5cc86cc71043d9618499c9284cca7 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 19 Oct 2023 14:21:09 +0200 Subject: [PATCH 08/14] Add rand bytes func --- .../core/src/node/conn_manager/in_memory.rs | 13 +--- crates/core/src/runtime/native_api.rs | 73 +++++++++++++------ crates/core/src/runtime/wasm_runtime.rs | 3 +- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/crates/core/src/node/conn_manager/in_memory.rs b/crates/core/src/node/conn_manager/in_memory.rs index f1fbda01f..abcc136eb 100644 --- a/crates/core/src/node/conn_manager/in_memory.rs +++ b/crates/core/src/node/conn_manager/in_memory.rs @@ -8,7 +8,7 @@ use std::{ use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; -use rand::{prelude::StdRng, thread_rng, Rng, SeedableRng}; +use rand::{prelude::StdRng, seq::SliceRandom, thread_rng, Rng, SeedableRng}; use tokio::sync::Mutex; use super::{ConnectionBridge, ConnectionError, PeerKey}; @@ -180,7 +180,8 @@ impl InMemoryTransport { for (_, msgs) in delayed.drain() { queue.extend(msgs); } - Self::shuffle(&mut queue); + let queue = &mut queue; + queue.shuffle(&mut thread_rng()); } } tracing::error!("Stopped receiving messages in {}", interface_peer); @@ -204,12 +205,4 @@ impl InMemoryTransport { tracing::error!("Network shutdown") } } - - fn shuffle(iter: &mut [T]) { - let mut rng = thread_rng(); - for i in (1..(iter.len() - 1)).rev() { - let idx = rng.gen_range(0..=i); - iter.swap(idx, i); - } - } } diff --git a/crates/core/src/runtime/native_api.rs b/crates/core/src/runtime/native_api.rs index 475b6bd63..c061fd78d 100644 --- a/crates/core/src/runtime/native_api.rs +++ b/crates/core/src/runtime/native_api.rs @@ -19,56 +19,81 @@ fn compute_ptr(ptr: i64, start_ptr: i64) -> *mut T { (start_ptr + ptr) as _ } -pub(crate) mod time { +pub(crate) mod log { + use wasmer::Function; + use super::*; - use chrono::{DateTime, Utc as UtcOriginal}; pub(crate) fn prepare_export(store: &mut wasmer::Store, imports: &mut Imports) { - let utc_now = Function::new_typed(store, utc_now); + let utc_now = Function::new_typed(store, info); imports.register_namespace( - "freenet_time", - [("__frnt__time__utc_now".to_owned(), utc_now.into())], + "freenet_log", + [("__frnt__logger__info".to_owned(), utc_now.into())], ); } - fn utc_now(id: i64, ptr: i64) { + // TODO: this API right now is just a patch, ideally we want to impl a tracing subscriber + // that can be used in wasm and that under the hood will just pass data to the host via + // functions like this in a structured way + fn info(id: i64, ptr: i64, len: i32) { if id == -1 { panic!("unset module id"); } let info = MEM_ADDR.get(&id).expect("instance mem space not recorded"); - let now = UtcOriginal::now(); - let ptr = compute_ptr::>(ptr, info.start_ptr); - // eprintln!("{ptr:p} ({}) outside", ptr as i64); - unsafe { - ptr.write(now); - }; + let ptr = compute_ptr::(ptr, info.start_ptr); + let msg = + unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(ptr, len as _)) }; + tracing::info!(target: "contract", contract = %info.value().key(), "{msg}"); } } -pub(crate) mod log { - use wasmer::Function; +pub(crate) mod rand { + use ::rand::{thread_rng, RngCore}; use super::*; pub(crate) fn prepare_export(store: &mut wasmer::Store, imports: &mut Imports) { - let utc_now = Function::new_typed(store, info); + let rand_bytes = Function::new_typed(store, rand_bytes); imports.register_namespace( - "freenet_logger", - [("__frnt__logger__info".to_owned(), utc_now.into())], + "freenet_rand", + [("__frnt__rand__rand_bytes".to_owned(), rand_bytes.into())], ); } - // TODO: this API right now is just a patch, ideally we want to impl a tracing subscriber - // that can be used in wasm and that under the hood will just pass data to the host via - // functions like this in a structured way - fn info(id: i64, ptr: i64, len: i32) { + fn rand_bytes(id: i64, ptr: i64, len: u32) { if id == -1 { panic!("unset module id"); } let info = MEM_ADDR.get(&id).expect("instance mem space not recorded"); let ptr = compute_ptr::(ptr, info.start_ptr); - let msg = - unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(ptr, len as _)) }; - tracing::info!(target: "contract", contract = %info.value().key(), "{msg}"); + let slice = unsafe { &mut *std::ptr::slice_from_raw_parts_mut(ptr, len as usize) }; + let mut rng = thread_rng(); + rng.fill_bytes(slice); + } +} + +pub(crate) mod time { + use super::*; + use chrono::{DateTime, Utc as UtcOriginal}; + + pub(crate) fn prepare_export(store: &mut wasmer::Store, imports: &mut Imports) { + let utc_now = Function::new_typed(store, utc_now); + imports.register_namespace( + "freenet_time", + [("__frnt__time__utc_now".to_owned(), utc_now.into())], + ); + } + + fn utc_now(id: i64, ptr: i64) { + if id == -1 { + panic!("unset module id"); + } + let info = MEM_ADDR.get(&id).expect("instance mem space not recorded"); + let now = UtcOriginal::now(); + let ptr = compute_ptr::>(ptr, info.start_ptr); + // eprintln!("{ptr:p} ({}) outside", ptr as i64); + unsafe { + ptr.write(now); + }; } } diff --git a/crates/core/src/runtime/wasm_runtime.rs b/crates/core/src/runtime/wasm_runtime.rs index b7d2a6446..4c9d2db29 100644 --- a/crates/core/src/runtime/wasm_runtime.rs +++ b/crates/core/src/runtime/wasm_runtime.rs @@ -127,8 +127,9 @@ impl Runtime { } else { (None, imports! {}) }; - native_api::time::prepare_export(&mut store, &mut top_level_imports); native_api::log::prepare_export(&mut store, &mut top_level_imports); + native_api::rand::prepare_export(&mut store, &mut top_level_imports); + native_api::time::prepare_export(&mut store, &mut top_level_imports); Ok(Self { wasm_store: store, From ca5afc3cef85efabee676b50277ff9aa4e00a97d Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 19 Oct 2023 17:28:40 +0200 Subject: [PATCH 09/14] Rename join ring to connect --- crates/core/src/message.rs | 19 +- crates/core/src/node.rs | 15 +- crates/core/src/node/event_log.rs | 12 +- crates/core/src/node/op_state_manager.rs | 30 +-- crates/core/src/operations.rs | 6 +- .../operations/{join_ring.rs => connect.rs} | 181 +++++++++--------- crates/core/src/operations/get.rs | 22 +-- crates/core/src/operations/put.rs | 24 +-- crates/core/src/operations/subscribe.rs | 42 ++-- crates/core/src/operations/update.rs | 35 +++- stdlib | 2 +- 11 files changed, 213 insertions(+), 175 deletions(-) rename crates/core/src/operations/{join_ring.rs => connect.rs} (91%) diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index f284f7503..ef8046198 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -8,8 +8,7 @@ use ulid::Ulid; use crate::{ node::{ConnectionError, PeerKey}, operations::{ - get::GetMsg, join_ring::JoinRingMsg, put::PutMsg, subscribe::SubscribeMsg, - update::UpdateMsg, + connect::ConnectMsg, get::GetMsg, put::PutMsg, subscribe::SubscribeMsg, update::UpdateMsg, }, ring::{Location, PeerKeyLocation}, }; @@ -127,7 +126,7 @@ mod sealed_msg_type { } transaction_type_enumeration!(decl struct { - JoinRing -> JoinRingMsg, + JoinRing -> ConnectMsg, Put -> PutMsg, Get -> GetMsg, Subscribe -> SubscribeMsg, @@ -137,7 +136,7 @@ mod sealed_msg_type { #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) enum Message { - JoinRing(JoinRingMsg), + JoinRing(ConnectMsg), Put(PutMsg), Get(GetMsg), Subscribe(SubscribeMsg), @@ -148,6 +147,10 @@ pub(crate) enum Message { pub(crate) trait InnerMessage: Into { fn id(&self) -> &Transaction; + + fn target(&self) -> Option<&PeerKeyLocation>; + + fn terminal(&self) -> bool; } /// Internal node events emitted to the event loop. @@ -190,7 +193,7 @@ impl Message { Put(op) => op.id(), Get(op) => op.id(), Subscribe(op) => op.id(), - Update(_op) => todo!(), + Update(op) => op.id(), Aborted(tx) => tx, } } @@ -202,7 +205,7 @@ impl Message { Put(op) => op.target(), Get(op) => op.target(), Subscribe(op) => op.target(), - Update(_op) => todo!(), + Update(op) => op.target(), Aborted(_) => None, } } @@ -215,7 +218,7 @@ impl Message { Put(op) => op.terminal(), Get(op) => op.terminal(), Subscribe(op) => op.terminal(), - Update(_op) => todo!(), + Update(op) => op.terminal(), Aborted(_) => true, } } @@ -235,7 +238,7 @@ impl Display for Message { Put(msg) => msg.fmt(f)?, Get(msg) => msg.fmt(f)?, Subscribe(msg) => msg.fmt(f)?, - Update(_op) => todo!(), + Update(msg) => msg.fmt(f)?, Aborted(msg) => msg.fmt(f)?, }; write!(f, "}}") diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 510b44d91..39bd2c9e2 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -32,9 +32,8 @@ use crate::{ }, message::{InnerMessage, Message, Transaction, TransactionType}, operations::{ - get, - join_ring::{self, JoinRingMsg, JoinRingOp}, - put, subscribe, OpEnum, OpError, OpOutcome, + connect::{self, ConnectMsg, ConnectOp}, + get, put, subscribe, OpEnum, OpError, OpOutcome, }, ring::{Location, PeerKeyLocation}, router::{RouteEvent, RouteOutcome}, @@ -282,9 +281,9 @@ async fn join_ring_request( where CM: ConnectionBridge + Send + Sync, { - let tx_id = Transaction::new::(); + let tx_id = Transaction::new::(); let mut op = - join_ring::initial_request(peer_key, *gateway, op_storage.ring.max_hops_to_live, tx_id); + connect::initial_request(peer_key, *gateway, op_storage.ring.max_hops_to_live, tx_id); if let Some(mut backoff) = backoff { // backoff to retry later in case it failed tracing::warn!("Performing a new join, attempt {}", backoff.retries() + 1); @@ -294,7 +293,7 @@ where } op.backoff = Some(backoff); } - join_ring::join_ring_request(tx_id, op_storage, conn_manager, op).await?; + connect::join_ring_request(tx_id, op_storage, conn_manager, op).await?; Ok(()) } @@ -524,7 +523,7 @@ async fn process_message( match msg { Message::JoinRing(op) => { log_handling_msg!("join", op.id(), op_storage); - let op_result = handle_op_request::( + let op_result = handle_op_request::( &op_storage, &mut conn_manager, op, @@ -624,7 +623,7 @@ where // is useless without connecting to the network, we will retry with exponential backoff match op_storage.pop(&tx) { Some(OpEnum::JoinRing(op)) if op.has_backoff() => { - let JoinRingOp { + let ConnectOp { gateway, backoff, .. } = *op; let backoff = backoff.expect("infallible"); diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index 08f0ecab0..e2a99705c 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -15,7 +15,7 @@ use crate::{ config::GlobalExecutor, contract::StoreResponse, message::{Message, Transaction}, - operations::{get::GetMsg, join_ring, put::PutMsg}, + operations::{connect, get::GetMsg, put::PutMsg}, ring::PeerKeyLocation, router::RouteEvent, DynError, @@ -72,9 +72,9 @@ impl<'a> EventLog<'a> { op_storage: &'a OpManager, ) -> Either> { let kind = match msg { - Message::JoinRing(join_ring::JoinRingMsg::Response { + Message::JoinRing(connect::ConnectMsg::Response { msg: - join_ring::JoinResponse::AcceptedBy { + connect::ConnectResponse::AcceptedBy { peers, your_location, your_peer_id, @@ -108,9 +108,9 @@ impl<'a> EventLog<'a> { op_storage: &'a OpManager, ) -> Either> { let kind = match msg { - Message::JoinRing(join_ring::JoinRingMsg::Response { + Message::JoinRing(connect::ConnectMsg::Response { msg: - join_ring::JoinResponse::AcceptedBy { + connect::ConnectResponse::AcceptedBy { peers, your_location, your_peer_id, @@ -680,7 +680,7 @@ pub(super) mod test_utils { use crate::ring::Location; let peer_id = PeerKey::random(); let loc = Location::try_from(0.5)?; - let tx = Transaction::new::(); + let tx = Transaction::new::(); let locations = [ (PeerKey::random(), Location::try_from(0.5)?), (PeerKey::random(), Location::try_from(0.75)?), diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index addb61193..6aeafec07 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -13,7 +13,8 @@ use crate::{ dev_tool::ClientId, message::{Message, Transaction, TransactionType}, operations::{ - get::GetOp, join_ring::JoinRingOp, put::PutOp, subscribe::SubscribeOp, OpEnum, OpError, + connect::ConnectOp, get::GetOp, put::PutOp, subscribe::SubscribeOp, update::UpdateOp, + OpEnum, OpError, }, ring::Ring, }; @@ -23,10 +24,11 @@ use super::{conn_manager::EventLoopNotificationsSender, PeerKey}; /// Thread safe and friendly data structure to maintain state of the different operations /// and enable their execution. pub(crate) struct OpManager { - join_ring: DashMap, + join_ring: DashMap, put: DashMap, get: DashMap, subscribe: DashMap, + update: DashMap, to_event_listener: EventLoopNotificationsSender, // todo: remove the need for a mutex here ch_outbound: Mutex>, @@ -55,6 +57,7 @@ impl OpManager { put: DashMap::default(), get: DashMap::default(), subscribe: DashMap::default(), + update: DashMap::default(), ring, to_event_listener: notification_channel, ch_outbound: Mutex::new(contract_handler), @@ -108,25 +111,30 @@ impl OpManager { pub fn push(&self, id: Transaction, op: OpEnum) -> Result<(), OpError> { match op { - OpEnum::JoinRing(tx) => { + OpEnum::JoinRing(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::JoinRing); - self.join_ring.insert(id, *tx); + self.join_ring.insert(id, *op); } - OpEnum::Put(tx) => { + OpEnum::Put(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::Put); - self.put.insert(id, tx); + self.put.insert(id, op); } - OpEnum::Get(tx) => { + OpEnum::Get(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::Get); - self.get.insert(id, tx); + self.get.insert(id, op); } - OpEnum::Subscribe(tx) => { + OpEnum::Subscribe(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::Subscribe); - self.subscribe.insert(id, tx); + self.subscribe.insert(id, op); + } + OpEnum::Update(op) => { + #[cfg(debug_assertions)] + check_id_op!(id.tx_type(), TransactionType::Update); + self.update.insert(id, op); } } Ok(()) @@ -146,7 +154,7 @@ impl OpManager { .remove(id) .map(|(_k, v)| v) .map(OpEnum::Subscribe), - TransactionType::Update => todo!(), + TransactionType::Update => self.update.remove(id).map(|(_k, v)| v).map(OpEnum::Update), TransactionType::Canceled => unreachable!(), } } diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index 410bff877..6c5b766e9 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -12,8 +12,8 @@ use crate::{ DynError, }; +pub(crate) mod connect; pub(crate) mod get; -pub(crate) mod join_ring; pub(crate) mod op_trait; pub(crate) mod put; pub(crate) mod subscribe; @@ -126,10 +126,11 @@ where } pub(crate) enum OpEnum { - JoinRing(Box), + JoinRing(Box), Put(put::PutOp), Get(get::GetOp), Subscribe(subscribe::SubscribeOp), + Update(update::UpdateOp), } impl OpEnum { @@ -139,6 +140,7 @@ impl OpEnum { OpEnum::Put(op) => op, OpEnum::Get(op) => op, OpEnum::Subscribe(op) => op, + OpEnum::Update(op) => op, } { pub fn id(&self) -> &Transaction; pub fn outcome(&self) -> OpOutcome; diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/connect.rs similarity index 91% rename from crates/core/src/operations/join_ring.rs rename to crates/core/src/operations/connect.rs index 0039e6b6f..0f34f3290 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/connect.rs @@ -1,3 +1,4 @@ +//! Operation which seeks new connections in the ring. use futures::Future; use std::pin::Pin; use std::{collections::HashSet, time::Duration}; @@ -15,9 +16,9 @@ use crate::{ util::ExponentialBackoff, }; -pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; +pub(crate) use self::messages::{ConnectMsg, ConnectRequest, ConnectResponse}; -pub(crate) struct JoinRingOp { +pub(crate) struct ConnectOp { id: Transaction, state: Option, pub gateway: Box, @@ -27,7 +28,7 @@ pub(crate) struct JoinRingOp { _ttl: Duration, } -impl JoinRingOp { +impl ConnectOp { pub fn has_backoff(&self) -> bool { self.backoff.is_some() } @@ -44,12 +45,12 @@ impl JoinRingOp { } /// Not really used since client requests will never interact with this directly. -pub(crate) struct JoinRingResult {} +pub(crate) struct ConnectResult {} -impl TryFrom for JoinRingResult { +impl TryFrom for ConnectResult { type Error = OpError; - fn try_from(_value: JoinRingOp) -> Result { + fn try_from(_value: ConnectOp) -> Result { Ok(Self {}) } } @@ -74,9 +75,9 @@ gateway: 2. assigned_location: Some(loc) */ -impl Operation for JoinRingOp { - type Message = JoinRingMsg; - type Result = JoinRingResult; +impl Operation for ConnectOp { + type Message = ConnectMsg; + type Result = ConnectResult; fn load_or_init( op_storage: &OpManager, @@ -126,10 +127,10 @@ impl Operation for JoinRingOp { let mut new_state = None; match input { - JoinRingMsg::Request { + ConnectMsg::Request { id, msg: - JoinRequest::StartReq { + ConnectRequest::StartReq { target: this_node_loc, joiner, hops_to_live, @@ -193,10 +194,10 @@ impl Operation for JoinRingOp { } else { new_state = None } - return_msg = Some(JoinRingMsg::Response { + return_msg = Some(ConnectMsg::Response { id, sender: this_node_loc, - msg: JoinResponse::AcceptedBy { + msg: ConnectResponse::AcceptedBy { peers: accepted_by, your_location: new_location, your_peer_id: joiner, @@ -208,10 +209,10 @@ impl Operation for JoinRingOp { }); } } - JoinRingMsg::Request { + ConnectMsg::Request { id, msg: - JoinRequest::Proxy { + ConnectRequest::Proxy { sender, joiner, hops_to_live, @@ -292,11 +293,11 @@ impl Operation for JoinRingOp { sender.peer, target.peer ); - return_msg = Some(JoinRingMsg::Response { + return_msg = Some(ConnectMsg::Response { id, target, sender, - msg: JoinResponse::AcceptedBy { + msg: ConnectResponse::AcceptedBy { peers: accepted_by, your_location: new_location, your_peer_id: new_peer_id, @@ -316,11 +317,11 @@ impl Operation for JoinRingOp { sender.peer, own_loc.peer ); - return_msg = Some(JoinRingMsg::Response { + return_msg = Some(ConnectMsg::Response { id, target, sender, - msg: JoinResponse::Proxy { accepted_by }, + msg: ConnectResponse::Proxy { accepted_by }, }); } } @@ -335,11 +336,11 @@ impl Operation for JoinRingOp { }; } } - JoinRingMsg::Response { + ConnectMsg::Response { id, sender, msg: - JoinResponse::AcceptedBy { + ConnectResponse::AcceptedBy { peers: accepted_by, your_location, your_peer_id, @@ -369,9 +370,9 @@ impl Operation for JoinRingOp { gateway.peer ); new_state = Some(JRState::OCReceived); - return_msg = Some(JoinRingMsg::Response { + return_msg = Some(ConnectMsg::Response { id, - msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, + msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, sender: pk_loc, target: sender, }); @@ -389,11 +390,11 @@ impl Operation for JoinRingOp { op_storage, sender, &other_peer, - JoinRingMsg::Response { + ConnectMsg::Response { id, target: other_peer, sender: pk_loc, - msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, + msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, }, ) .await; @@ -405,7 +406,7 @@ impl Operation for JoinRingOp { peer = %your_peer_id, "Failed to establish any connections, aborting" ); - let op = JoinRingOp { + let op = ConnectOp { id, state: None, gateway: self.gateway, @@ -422,11 +423,11 @@ impl Operation for JoinRingOp { return Err(OpError::StatePushed); } } - JoinRingMsg::Response { + ConnectMsg::Response { id, sender, target, - msg: JoinResponse::Proxy { mut accepted_by }, + msg: ConnectResponse::Proxy { mut accepted_by }, } => { tracing::debug!(tx = %id, "Received proxy join at @ {}", target.peer); match self.state { @@ -445,8 +446,8 @@ impl Operation for JoinRingOp { tracing::debug!("Failed to connect at proxy {}", sender.peer); new_state = None; } - return_msg = Some(JoinRingMsg::Response { - msg: JoinResponse::Proxy { accepted_by }, + return_msg = Some(ConnectMsg::Response { + msg: ConnectResponse::Proxy { accepted_by }, sender, id, target, @@ -485,11 +486,11 @@ impl Operation for JoinRingOp { target.peer, original_target.peer ); - return_msg = Some(JoinRingMsg::Response { + return_msg = Some(ConnectMsg::Response { id, target: original_target, sender: target, - msg: JoinResponse::AcceptedBy { + msg: ConnectResponse::AcceptedBy { peers: previously_accepted, your_location: new_location, your_peer_id: new_peer_id, @@ -504,11 +505,11 @@ impl Operation for JoinRingOp { original_target.peer ); - return_msg = Some(JoinRingMsg::Response { + return_msg = Some(ConnectMsg::Response { id, target: original_target, sender: target, - msg: JoinResponse::Proxy { + msg: ConnectResponse::Proxy { accepted_by: previously_accepted, }, }); @@ -524,17 +525,17 @@ impl Operation for JoinRingOp { } }; } - JoinRingMsg::Response { + ConnectMsg::Response { id, sender, - msg: JoinResponse::ReceivedOC { by_peer }, + msg: ConnectResponse::ReceivedOC { by_peer }, target, } => { match self.state { Some(JRState::OCReceived) => { tracing::debug!(tx = %id, "Acknowledge connected at gateway"); new_state = Some(JRState::Connected); - return_msg = Some(JoinRingMsg::Connected { + return_msg = Some(ConnectMsg::Connected { id, sender: target, target: sender, @@ -556,7 +557,7 @@ impl Operation for JoinRingOp { } }; } - JoinRingMsg::Connected { target, sender, id } => { + ConnectMsg::Connected { target, sender, id } => { match self.state { Some(JRState::OCReceived) => { tracing::debug!(tx = %id, "Acknowledge connected at peer {}", target.peer); @@ -602,12 +603,12 @@ impl Operation for JoinRingOp { fn build_op_result( id: Transaction, state: Option, - msg: Option, + msg: Option, gateway: Box, backoff: Option, ttl: Duration, ) -> Result { - let output_op = Some(JoinRingOp { + let output_op = Some(ConnectOp { id, state, gateway, @@ -625,7 +626,7 @@ fn try_proxy_connection( sender: &PeerKeyLocation, own_loc: &PeerKeyLocation, accepted_by: HashSet, -) -> (Option, Option) { +) -> (Option, Option) { let new_state = if accepted_by.contains(own_loc) { tracing::debug!( tx = %id, @@ -638,8 +639,8 @@ fn try_proxy_connection( tracing::debug!(tx = %id, "Failed to connect at proxy {}", sender.peer); None }; - let return_msg = Some(JoinRingMsg::Response { - msg: JoinResponse::Proxy { accepted_by }, + let return_msg = Some(ConnectMsg::Response { + msg: ConnectResponse::Proxy { accepted_by }, sender: *own_loc, id: *id, target: *sender, @@ -652,7 +653,7 @@ async fn propagate_oc_to_accepted_peers( op_storage: &OpManager, sender: PeerKeyLocation, other_peer: &PeerKeyLocation, - msg: JoinRingMsg, + msg: ConnectMsg, ) -> Result<(), OpError> { let id = msg.id(); if op_storage.ring.should_accept( @@ -752,7 +753,7 @@ pub(crate) fn initial_request( gateway: PeerKeyLocation, max_hops_to_live: usize, id: Transaction, -) -> JoinRingOp { +) -> ConnectOp { const MAX_JOIN_RETRIES: usize = 3; tracing::debug!(tx = %id, "Connecting to gw {} from {}", gateway.peer, this_peer); let state = JRState::Connecting(ConnectionInfo { @@ -765,7 +766,7 @@ pub(crate) fn initial_request( } else { Duration::from_secs(120) }; - JoinRingOp { + ConnectOp { id, state: Some(state), gateway: Box::new(gateway), @@ -783,12 +784,12 @@ pub(crate) async fn join_ring_request( tx: Transaction, op_storage: &OpManager, conn_manager: &mut CB, - join_op: JoinRingOp, + join_op: ConnectOp, ) -> Result<(), OpError> where CB: ConnectionBridge, { - let JoinRingOp { + let ConnectOp { id, state, backoff, @@ -810,9 +811,9 @@ where conn_manager.add_connection(gateway.peer).await?; let assigned_location = op_storage.ring.own_location().location; - let join_req = Message::from(messages::JoinRingMsg::Request { + let join_req = Message::from(messages::ConnectMsg::Request { id: tx, - msg: messages::JoinRequest::StartReq { + msg: messages::ConnectRequest::StartReq { target: gateway, joiner: this_peer, assigned_location, @@ -823,7 +824,7 @@ where conn_manager.send(&gateway.peer, join_req).await?; op_storage.push( tx, - OpEnum::JoinRing(Box::new(JoinRingOp { + OpEnum::JoinRing(Box::new(ConnectOp { id, state: Some(JRState::Connecting(ConnectionInfo { gateway, @@ -888,9 +889,9 @@ where }; if let Some(forward_to) = forward_to { - let forwarded = Message::from(JoinRingMsg::Request { + let forwarded = Message::from(ConnectMsg::Request { id, - msg: JoinRequest::Proxy { + msg: ConnectRequest::Proxy { joiner, hops_to_live: left_htl.min(ring.max_hops_to_live) - 1, sender: ring.own_location(), @@ -934,16 +935,16 @@ mod messages { use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum JoinRingMsg { + pub(crate) enum ConnectMsg { Request { id: Transaction, - msg: JoinRequest, + msg: ConnectRequest, }, Response { id: Transaction, sender: PeerKeyLocation, target: PeerKeyLocation, - msg: JoinResponse, + msg: ConnectResponse, }, Connected { id: Transaction, @@ -952,7 +953,7 @@ mod messages { }, } - impl InnerMessage for JoinRingMsg { + impl InnerMessage for ConnectMsg { fn id(&self) -> &Transaction { match self { Self::Request { id, .. } => id, @@ -960,31 +961,13 @@ mod messages { Self::Connected { id, .. } => id, } } - } - - impl JoinRingMsg { - pub fn sender(&self) -> Option<&PeerKey> { - use JoinRingMsg::*; - match self { - Response { sender, .. } => Some(&sender.peer), - Connected { sender, .. } => Some(&sender.peer), - Request { - msg: - JoinRequest::StartReq { - joiner: req_peer, .. - }, - .. - } => Some(req_peer), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { - use JoinRingMsg::*; + fn target(&self) -> Option<&PeerKeyLocation> { + use ConnectMsg::*; match self { Response { target, .. } => Some(target), Request { - msg: JoinRequest::StartReq { target, .. }, + msg: ConnectRequest::StartReq { target, .. }, .. } => Some(target), Connected { target, .. } => Some(target), @@ -992,44 +975,62 @@ mod messages { } } - pub fn terminal(&self) -> bool { - use JoinRingMsg::*; + fn terminal(&self) -> bool { + use ConnectMsg::*; matches!( self, Response { - msg: JoinResponse::Proxy { .. }, + msg: ConnectResponse::Proxy { .. }, .. } | Connected { .. } ) } } - impl Display for JoinRingMsg { + impl ConnectMsg { + pub fn sender(&self) -> Option<&PeerKey> { + use ConnectMsg::*; + match self { + Response { sender, .. } => Some(&sender.peer), + Connected { sender, .. } => Some(&sender.peer), + Request { + msg: + ConnectRequest::StartReq { + joiner: req_peer, .. + }, + .. + } => Some(req_peer), + _ => None, + } + } + } + + impl Display for ConnectMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); match self { Self::Request { - msg: JoinRequest::StartReq { .. }, + msg: ConnectRequest::StartReq { .. }, .. } => write!(f, "StartRequest(id: {id})"), Self::Request { - msg: JoinRequest::Accepted { .. }, + msg: ConnectRequest::Accepted { .. }, .. } => write!(f, "RequestAccepted(id: {id})"), Self::Request { - msg: JoinRequest::Proxy { .. }, + msg: ConnectRequest::Proxy { .. }, .. } => write!(f, "ProxyRequest(id: {id})"), Self::Response { - msg: JoinResponse::AcceptedBy { .. }, + msg: ConnectResponse::AcceptedBy { .. }, .. } => write!(f, "RouteValue(id: {id})"), Self::Response { - msg: JoinResponse::ReceivedOC { .. }, + msg: ConnectResponse::ReceivedOC { .. }, .. } => write!(f, "RouteValue(id: {id})"), Self::Response { - msg: JoinResponse::Proxy { .. }, + msg: ConnectResponse::Proxy { .. }, .. } => write!(f, "RouteValue(id: {id})"), Self::Connected { .. } => write!(f, "Connected(id: {id})"), @@ -1039,7 +1040,7 @@ mod messages { } #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum JoinRequest { + pub(crate) enum ConnectRequest { StartReq { target: PeerKeyLocation, joiner: PeerKey, @@ -1062,7 +1063,7 @@ mod messages { } #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum JoinResponse { + pub(crate) enum ConnectResponse { AcceptedBy { peers: HashSet, your_location: Location, diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 296716230..9a5733420 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -772,17 +772,8 @@ mod messages { Self::ReturnGet { id, .. } => id, } } - } - - impl GetMsg { - pub fn sender(&self) -> Option<&PeerKeyLocation> { - match self { - Self::SeekNode { target, .. } => Some(target), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { + fn target(&self) -> Option<&PeerKeyLocation> { match self { Self::SeekNode { target, .. } => Some(target), Self::RequestGet { target, .. } => Some(target), @@ -790,12 +781,21 @@ mod messages { } } - pub fn terminal(&self) -> bool { + fn terminal(&self) -> bool { use GetMsg::*; matches!(self, ReturnGet { .. }) } } + impl GetMsg { + pub fn sender(&self) -> Option<&PeerKeyLocation> { + match self { + Self::SeekNode { target, .. } => Some(target), + _ => None, + } + } + } + impl Display for GetMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index b1c00fa67..84ab12e7b 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -869,18 +869,8 @@ mod messages { Self::BroadcastTo { id, .. } => id, } } - } - - impl PutMsg { - pub fn sender(&self) -> Option<&PeerKeyLocation> { - match self { - Self::SeekNode { sender, .. } => Some(sender), - Self::BroadcastTo { sender, .. } => Some(sender), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { + fn target(&self) -> Option<&PeerKeyLocation> { match self { Self::SeekNode { target, .. } => Some(target), Self::RequestPut { target, .. } => Some(target), @@ -888,7 +878,7 @@ mod messages { } } - pub fn terminal(&self) -> bool { + fn terminal(&self) -> bool { use PutMsg::*; matches!( self, @@ -897,6 +887,16 @@ mod messages { } } + impl PutMsg { + pub fn sender(&self) -> Option<&PeerKeyLocation> { + match self { + Self::SeekNode { sender, .. } => Some(sender), + Self::BroadcastTo { sender, .. } => Some(sender), + _ => None, + } + } + } + impl Display for PutMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 3f443ce3a..5dd708001 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -9,7 +9,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::ContractError, - message::{Message, Transaction}, + message::{InnerMessage, Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{PeerKeyLocation, RingError}, @@ -39,13 +39,16 @@ impl SubscribeOp { pub(super) fn record_transfer(&mut self) {} } -pub(crate) enum SubscribeResult {} +pub(crate) struct SubscribeResult {} impl TryFrom for SubscribeResult { type Error = OpError; - fn try_from(_value: SubscribeOp) -> Result { - todo!() + fn try_from(value: SubscribeOp) -> Result { + value + .finalized() + .then_some(SubscribeResult {}) + .ok_or(OpError::UnexpectedOpState) } } @@ -416,26 +419,8 @@ mod messages { Self::ReturnSub { id, .. } => id, } } - } - - impl SubscribeMsg { - pub(crate) fn id(&self) -> &Transaction { - match self { - Self::SeekNode { id, .. } => id, - Self::FetchRouting { id, .. } => id, - Self::RequestSub { id, .. } => id, - Self::ReturnSub { id, .. } => id, - } - } - - pub fn sender(&self) -> Option<&PeerKeyLocation> { - match self { - Self::ReturnSub { sender, .. } => Some(sender), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { + fn target(&self) -> Option<&PeerKeyLocation> { match self { Self::SeekNode { target, .. } => Some(target), Self::ReturnSub { target, .. } => Some(target), @@ -443,12 +428,21 @@ mod messages { } } - pub fn terminal(&self) -> bool { + fn terminal(&self) -> bool { use SubscribeMsg::*; matches!(self, ReturnSub { .. } | SeekNode { .. }) } } + impl SubscribeMsg { + pub fn sender(&self) -> Option<&PeerKeyLocation> { + match self { + Self::ReturnSub { sender, .. } => Some(sender), + _ => None, + } + } + } + impl Display for SubscribeMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index 42339191f..e85e495bb 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -3,10 +3,22 @@ pub(crate) use self::messages::UpdateMsg; use crate::{client_events::ClientId, node::ConnectionBridge}; -use super::{op_trait::Operation, OpError}; +use super::{op_trait::Operation, OpError, OpOutcome}; pub(crate) struct UpdateOp {} +impl UpdateOp { + pub fn outcome(&self) -> OpOutcome { + OpOutcome::Irrelevant + } + + pub fn finalized(&self) -> bool { + todo!() + } + + pub fn record_transfer(&mut self) {} +} + pub(crate) struct UpdateResult {} impl TryFrom for UpdateResult { @@ -46,9 +58,14 @@ impl Operation for UpdateOp { } mod messages { + use std::fmt::Display; + use serde::{Deserialize, Serialize}; - use crate::message::{InnerMessage, Transaction}; + use crate::{ + message::{InnerMessage, Transaction}, + ring::PeerKeyLocation, + }; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub(crate) enum UpdateMsg {} @@ -57,5 +74,19 @@ mod messages { fn id(&self) -> &Transaction { todo!() } + + fn target(&self) -> Option<&PeerKeyLocation> { + todo!() + } + + fn terminal(&self) -> bool { + todo!() + } + } + + impl Display for UpdateMsg { + fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } } } diff --git a/stdlib b/stdlib index cdb8ceaba..bf0da9a67 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit cdb8ceabaa6fcd2cd730b7be66da7fc2b57a50bf +Subproject commit bf0da9a67429e50439b41a4dcd7668b8f119acf1 From 5d46cd1af9ed59fdb0cff7b5e3d13110d4d28e48 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 20 Oct 2023 11:57:55 +0200 Subject: [PATCH 10/14] Fix some issues with put op + remove some todos --- crates/core/src/client_events.rs | 2 +- crates/core/src/config.rs | 11 ++- crates/core/src/contract.rs | 3 +- crates/core/src/contract/executor.rs | 3 + crates/core/src/contract/handler.rs | 1 + crates/core/src/node.rs | 63 +++++++----- .../core/src/node/conn_manager/in_memory.rs | 50 +++++----- .../core/src/node/conn_manager/p2p_protoc.rs | 21 ++-- crates/core/src/node/event_log.rs | 27 ++++- crates/core/src/node/in_memory_impl.rs | 10 +- crates/core/src/node/op_state_manager.rs | 4 +- crates/core/src/node/tests.rs | 35 +++++-- crates/core/src/operations.rs | 4 +- crates/core/src/operations/connect.rs | 98 +++++++++---------- crates/core/src/operations/get.rs | 44 +++++---- crates/core/src/operations/put.rs | 55 ++++++++--- crates/core/src/operations/subscribe.rs | 47 ++++----- .../core/src/{runtime/mod.rs => runtime.rs} | 0 crates/core/src/{server/mod.rs => server.rs} | 0 crates/core/src/server/errors.rs | 17 +++- crates/core/src/server/path_handlers.rs | 6 +- stdlib | 2 +- 22 files changed, 324 insertions(+), 179 deletions(-) rename crates/core/src/{runtime/mod.rs => runtime.rs} (100%) rename crates/core/src/{server/mod.rs => server.rs} (100%) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index 22137b264..af1085741 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -255,7 +255,7 @@ pub(crate) mod test { } else { // let contract_no = rng.gen_range(0..self.non_owned_contracts.len()); // self.non_owned_contracts[contract_no] - todo!("fixme") + todo!() }; break ContractRequest::Subscribe { key, summary: None }.into(); } diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index f12094421..07360c1a1 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -333,11 +333,18 @@ pub fn set_logger() { return; } + let filter = if cfg!(any(test, debug_assertions)) { + tracing_subscriber::filter::LevelFilter::DEBUG.into() + } else { + tracing_subscriber::filter::LevelFilter::INFO.into() + }; + let sub = tracing_subscriber::fmt().with_level(true).with_env_filter( tracing_subscriber::EnvFilter::builder() - .with_default_directive(tracing_subscriber::filter::LevelFilter::DEBUG.into()) + .with_default_directive(filter) .from_env_lossy() - .add_directive("stretto=off".parse().unwrap()), + .add_directive("stretto=off".parse().unwrap()) + .add_directive("sqlx=error".parse().unwrap()), ); if cfg!(any(test, debug_assertions)) { diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index b018490fd..1b621dfd8 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -91,11 +91,12 @@ where ContractHandlerEvent::PutQuery { key, state, + related_contracts, parameters, } => { let put_result = contract_handler .executor() - .upsert_contract_state(key, Either::Left(state), parameters) + .upsert_contract_state(key, Either::Left(state), related_contracts, parameters) .await .map_err(Into::into); contract_handler diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index 82526414d..864a6df18 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -233,6 +233,7 @@ pub(crate) trait ContractExecutor: Send + Sync + 'static { &mut self, key: ContractKey, state: Either>, + related_contracts: RelatedContracts<'static>, params: Option>, ) -> Result; } @@ -1247,6 +1248,7 @@ impl ContractExecutor for Executor { &mut self, _key: ContractKey, _state: Either>, + _related_contracts: RelatedContracts<'static>, _params: Option>, ) -> Result { todo!() @@ -1295,6 +1297,7 @@ impl ContractExecutor for Executor { &mut self, key: ContractKey, state: Either>, + _related_contracts: RelatedContracts<'static>, params: Option>, ) -> Result { // todo: instead allow to perform mutations per contract based on incoming value so we can track diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 20a90c702..e259404bb 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -346,6 +346,7 @@ pub(crate) enum ContractHandlerEvent { PutQuery { key: ContractKey, state: WrappedState, + related_contracts: RelatedContracts<'static>, parameters: Option>, }, /// The response to a push query. diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 39bd2c9e2..da34531b6 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -98,6 +98,10 @@ pub struct NodeBuilder { pub(crate) local_ip: Option, /// socket port to bind to the listener pub(crate) local_port: Option, + /// IP dialers should connect to + pub(crate) public_ip: Option, + /// socket port dialers should connect to + pub(crate) public_port: Option, /// At least an other running listener node is required for joining the network. /// Not necessary if this is an initial node. pub(crate) remote_nodes: Vec, @@ -122,6 +126,8 @@ impl NodeBuilder { remote_nodes: Vec::with_capacity(1), local_ip: None, local_port: None, + public_ip: None, + public_port: None, location: None, max_hops_to_live: None, rnd_if_htl_above: None, @@ -293,7 +299,7 @@ where } op.backoff = Some(backoff); } - connect::join_ring_request(tx_id, op_storage, conn_manager, op).await?; + connect::connect_request(tx_id, op_storage, conn_manager, op).await?; Ok(()) } @@ -343,9 +349,9 @@ async fn client_event_handling( #[inline] async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc) { // this will indirectly start actions on the local contract executor - let op_storage_cp = op_storage.clone(); let fut = async move { let client_id = request.client_id; + let mut missing_contract = false; match *request.request { ClientRequest::ContractOp(ops) => match ops { ContractRequest::Put { @@ -356,13 +362,17 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc, op_storage: Arc { // Initialize a subscribe op. loop { - // FIXME: this will block the loop until the subscribe op succeeds - // instead the op should be deferred for later execution let op = subscribe::start_op(key.clone()); - match subscribe::request_subscribe(&op_storage_cp, op, Some(client_id)) - .await - { - Err(OpError::ContractError(ContractError::ContractNotFound(key))) => { - tracing::warn!("Trying to subscribe to a contract not present: {}, requesting it first", key); + match subscribe::request_subscribe(&op_storage, op, Some(client_id)).await { + Err(OpError::ContractError(ContractError::ContractNotFound(key))) + if !missing_contract => + { + tracing::info!("Trying to subscribe to a contract not present: {key}, requesting it first"); + missing_contract = true; let get_op = get::start_op(key.clone(), true); if let Err(err) = - get::request_get(&op_storage_cp, get_op, Some(client_id)).await + get::request_get(&op_storage, get_op, Some(client_id)).await { - tracing::error!("Failed getting the contract `{}` while previously trying to subscribe; bailing: {}", key, err); - tokio::time::sleep(Duration::from_secs(5)).await; + tracing::error!("Failed getting the contract `{key}` while previously trying to subscribe; bailing: {err}"); + break; } } + Err(OpError::ContractError(ContractError::ContractNotFound(_))) => { + tokio::time::sleep(Duration::from_secs(1)).await + } Err(err) => { tracing::error!("{}", err); break; } - Ok(()) => break, + Ok(()) => { + if missing_contract { + tracing::debug!( + "Got back the missing contract ({key}) while subscribing" + ); + } + break; + } } } - todo!() } _ => { tracing::error!("op not supported"); @@ -423,7 +441,6 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { + Some(OpEnum::Connect(op)) if op.has_backoff() => { let ConnectOp { gateway, backoff, .. } = *op; @@ -631,7 +648,7 @@ where join_ring_request(Some(backoff), peer_key, &gateway, op_storage, conn_manager) .await?; } - Some(OpEnum::JoinRing(_)) => { + Some(OpEnum::Connect(_)) => { return Err(OpError::MaxRetriesExceeded(tx, tx.tx_type())); } _ => {} diff --git a/crates/core/src/node/conn_manager/in_memory.rs b/crates/core/src/node/conn_manager/in_memory.rs index abcc136eb..8953afc49 100644 --- a/crates/core/src/node/conn_manager/in_memory.rs +++ b/crates/core/src/node/conn_manager/in_memory.rs @@ -8,7 +8,7 @@ use std::{ use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; -use rand::{prelude::StdRng, seq::SliceRandom, thread_rng, Rng, SeedableRng}; +use rand::{prelude::StdRng, seq::SliceRandom, Rng, SeedableRng}; use tokio::sync::Mutex; use super::{ConnectionBridge, ConnectionError, PeerKey}; @@ -34,23 +34,21 @@ impl MemoryConnManager { peer: PeerKey, log_register: Box, op_manager: Arc, + add_noise: bool, ) -> Self { - let transport = InMemoryTransport::new(peer); + let transport = InMemoryTransport::new(peer, add_noise); let msg_queue = Arc::new(Mutex::new(Vec::new())); let msg_queue_cp = msg_queue.clone(); - let tr_cp = transport.clone(); + let transport_cp = transport.clone(); GlobalExecutor::spawn(async move { // evaluate the messages as they arrive loop { - let Some(msg) = tr_cp.msg_stack_queue.lock().await.pop() else { - tokio::time::sleep(Duration::from_millis(10)).await; + let Some(msg) = transport_cp.msg_stack_queue.lock().await.pop() else { continue; }; let msg_data: Message = bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); - if let Ok(mut queue) = msg_queue_cp.try_lock() { - queue.push(msg_data); - } + msg_queue_cp.lock().await.push(msg_data); } }); @@ -65,7 +63,9 @@ impl MemoryConnManager { pub async fn recv(&self) -> Result { loop { - let Some(msg) = self.msg_queue.try_lock().ok().and_then(|mut l| l.pop()) else { + let mut queue = self.msg_queue.lock().await; + let Some(msg) = queue.pop() else { + std::mem::drop(queue); tokio::time::sleep(Duration::from_millis(10)).await; continue; }; @@ -131,14 +131,13 @@ pub struct InMemoryTransport { } impl InMemoryTransport { - fn new(interface_peer: PeerKey) -> Self { + fn new(interface_peer: PeerKey, add_noise: bool) -> Self { let msg_stack_queue = Arc::new(Mutex::new(Vec::new())); - let (tx, rx) = NETWORK_WIRES.get_or_init(crossbeam::channel::unbounded); + let (network_tx, network_rx) = NETWORK_WIRES.get_or_init(crossbeam::channel::unbounded); // store messages incoming from the network in the msg stack - let rcv_msg_c = msg_stack_queue.clone(); - let rx = rx.clone(); - let tx_cp = tx.clone(); + let msg_stack_queue_cp = msg_stack_queue.clone(); + let network_tx_cp = network_tx.clone(); GlobalExecutor::spawn(async move { const MAX_DELAYED_MSG: usize = 10; let mut rng = StdRng::from_entropy(); @@ -146,25 +145,29 @@ impl InMemoryTransport { let mut delayed: HashMap<_, Vec<_>> = HashMap::with_capacity(MAX_DELAYED_MSG); let last_drain = Instant::now(); loop { - match rx.try_recv() { + match network_rx.try_recv() { Ok(msg) if msg.target == interface_peer => { tracing::trace!( "Inbound message received for peer {} from {}", interface_peer, msg.origin ); - if (rng.gen_bool(0.5) && delayed.len() < MAX_DELAYED_MSG) - || delayed.contains_key(&msg.target) - { + if rng.gen_bool(0.5) && delayed.len() < MAX_DELAYED_MSG && add_noise { delayed.entry(msg.target).or_default().push(msg); tokio::time::sleep(Duration::from_millis(10)).await; } else { - rcv_msg_c.lock().await.push(msg); + let mut queue = msg_stack_queue_cp.lock().await; + queue.push(msg); + if add_noise && rng.gen_bool(0.2) { + queue.shuffle(&mut rng); + } } } Ok(msg) => { // send back to the network since this msg belongs to other peer - tx_cp.send(msg).expect("failed to send msg back to network"); + network_tx_cp + .send(msg) + .expect("failed to send msg back to network"); tokio::time::sleep(Duration::from_nanos(1_000)).await } Err(channel::TryRecvError::Disconnected) => break, @@ -176,22 +179,21 @@ impl InMemoryTransport { && !delayed.is_empty()) || delayed.len() == MAX_DELAYED_MSG { - let mut queue = rcv_msg_c.lock().await; + let mut queue = msg_stack_queue_cp.lock().await; for (_, msgs) in delayed.drain() { queue.extend(msgs); } let queue = &mut queue; - queue.shuffle(&mut thread_rng()); + queue.shuffle(&mut rng); } } tracing::error!("Stopped receiving messages in {}", interface_peer); }); - let network = tx.clone(); Self { interface_peer, msg_stack_queue, - network, + network: network_tx.clone(), } } diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index d7c0c0b64..f8acf8202 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -64,7 +64,7 @@ const CURRENT_IDENTIFY_PROTOC_VER: &str = "/id/1.0.0"; fn config_behaviour( local_key: &Keypair, gateways: &[InitPeerNode], - _public_addr: &Option, + _private_addr: &Option, op_manager: Arc, ) -> NetBehaviour { let routing_table: HashMap<_, _> = gateways @@ -234,6 +234,7 @@ pub(in crate::node) struct P2pConnManager { conn_bridge_rx: Receiver, /// last valid observed public address public_addr: Option, + listening_addr: Option, event_listener: Box, } @@ -248,7 +249,14 @@ impl P2pConnManager { // to reuse it's thread pool and scheduler in order to drive futures. let global_executor = GlobalExecutor; - let public_addr = if let Some(conn) = config.local_ip.zip(config.local_port) { + let private_addr = if let Some(conn) = config.local_ip.zip(config.local_port) { + let public_addr = multiaddr_from_connection(conn); + Some(public_addr) + } else { + None + }; + + let public_addr = if let Some(conn) = config.public_ip.zip(config.public_port) { let public_addr = multiaddr_from_connection(conn); Some(public_addr) } else { @@ -258,7 +266,7 @@ impl P2pConnManager { let behaviour = config_behaviour( &config.local_key, &config.remote_nodes, - &public_addr, + &private_addr, op_manager.clone(), ); let mut swarm = Swarm::new( @@ -283,12 +291,13 @@ impl P2pConnManager { bridge, conn_bridge_rx: rx_bridge_cmd, public_addr, + listening_addr: private_addr, event_listener: Box::new(event_listener), }) } pub fn listen_on(&mut self) -> Result<(), anyhow::Error> { - if let Some(listening_addr) = &self.public_addr { + if let Some(listening_addr) = &self.listening_addr { self.swarm.listen_on(listening_addr.clone())?; } Ok(()) @@ -510,7 +519,7 @@ impl P2pConnManager { self.public_addr = Some(address); } Ok(Right(IsPrivatePeer(_peer))) => { - todo!("attempt hole punching") + todo!("this peer is private, attempt hole punching") } Ok(Right(ClosedChannel)) => { tracing::info!("Notification channel closed"); @@ -561,7 +570,7 @@ enum ConnMngrActions { }, /// Update self own public address, useful when communicating for first time UpdatePublicAddr(Multiaddr), - /// A peer which we attempted connection to is private, attempt hole-punching + /// This is private, so when establishing connections hole-punching should be performed IsPrivatePeer(PeerId), NodeAction(NodeEvent), ClosedChannel, diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index e2a99705c..b759ab921 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -15,7 +15,7 @@ use crate::{ config::GlobalExecutor, contract::StoreResponse, message::{Message, Transaction}, - operations::{connect, get::GetMsg, put::PutMsg}, + operations::{connect, get::GetMsg, put::PutMsg, subscribe::SubscribeMsg}, ring::PeerKeyLocation, router::RouteEvent, DynError, @@ -177,6 +177,15 @@ impl<'a> EventLog<'a> { value: StoreResponse { state: Some(_), .. }, .. }) => EventKind::Get { key: key.clone() }, + Message::Subscribe(SubscribeMsg::ReturnSub { + subscribed: true, + key, + sender, + .. + }) => EventKind::Subscribed { + key: key.clone(), + at: *sender, + }, _ => EventKind::Ignored, }; Either::Left(EventLog { @@ -451,6 +460,10 @@ enum EventKind { key: ContractKey, }, Route(RouteEvent), + Subscribed { + key: ContractKey, + at: PeerKeyLocation, + }, Ignored, } @@ -610,6 +623,18 @@ pub(super) mod test_utils { }) } + pub fn is_subscribed_to_contract( + &self, + peer: &PeerKey, + expected_key: &ContractKey, + ) -> bool { + let logs = self.logs.lock(); + logs.iter().any(|log| { + &log.peer_id == peer + && matches!(log.kind, EventKind::Subscribed { ref key, .. } if key == expected_key ) + }) + } + /// Unique connections for a given peer and their relative distance to other peers. pub fn connections(&self, peer: PeerKey) -> impl Iterator { let logs = self.logs.lock(); diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index 47246894d..d24652742 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -42,6 +42,7 @@ impl NodeInMemory { builder: NodeBuilder<1>, event_listener: EL, ch_builder: String, + add_noise: bool, ) -> Result { let event_listener = Box::new(event_listener); let peer_key = PeerKey::from(builder.local_key.public()); @@ -58,8 +59,12 @@ impl NodeInMemory { .await .map_err(|e| anyhow::anyhow!(e))?; - let conn_manager = - MemoryConnManager::new(peer_key, event_listener.trait_clone(), op_storage.clone()); + let conn_manager = MemoryConnManager::new( + peer_key, + event_listener.trait_clone(), + op_storage.clone(), + add_noise, + ); GlobalExecutor::spawn(contract::contract_handling(contract_handler)); @@ -118,6 +123,7 @@ impl NodeInMemory { ContractHandlerEvent::PutQuery { key: key.clone(), state, + related_contracts: RelatedContracts::default(), parameters: Some(parameters), }, None, diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 6aeafec07..528833d81 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -111,7 +111,7 @@ impl OpManager { pub fn push(&self, id: Transaction, op: OpEnum) -> Result<(), OpError> { match op { - OpEnum::JoinRing(op) => { + OpEnum::Connect(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::JoinRing); self.join_ring.insert(id, *op); @@ -146,7 +146,7 @@ impl OpManager { .join_ring .remove(id) .map(|(_k, v)| v) - .map(|op| OpEnum::JoinRing(Box::new(op))), + .map(|op| OpEnum::Connect(Box::new(op))), TransactionType::Put => self.put.remove(id).map(|(_k, v)| v).map(OpEnum::Put), TransactionType::Get => self.get.remove(id).map(|(_k, v)| v).map(OpEnum::Get), TransactionType::Subscribe => self diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index 4bccb574b..c1ad04b85 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -127,6 +127,7 @@ pub(crate) struct SimNetwork { max_connections: usize, min_connections: usize, init_backoff: Duration, + add_noise: bool, } impl SimNetwork { @@ -155,6 +156,7 @@ impl SimNetwork { max_connections, min_connections, init_backoff: Duration::from_millis(1), + add_noise: false, }; net.build_gateways(gateways).await; net.build_nodes(nodes).await; @@ -165,6 +167,12 @@ impl SimNetwork { self.init_backoff = value; } + /// Simulates network random behaviour, like messages arriving delayed or out of order, throttling etc. + #[allow(unused)] + pub fn with_noise(&mut self) { + self.add_noise = true; + } + #[allow(unused)] pub fn debug(&mut self) { self.debug = true; @@ -226,6 +234,7 @@ impl SimNetwork { this_node, self.event_listener.clone(), format!("{}-{label}", self.name, label = this_config.label), + self.add_noise, ) .await .unwrap(); @@ -275,6 +284,7 @@ impl SimNetwork { config, self.event_listener.clone(), format!("{}-{label}", self.name), + self.add_noise, ) .await .unwrap(); @@ -344,25 +354,33 @@ impl SimNetwork { pub fn has_put_contract( &self, - peer: &NodeLabel, + peer: impl Into, key: &ContractKey, value: &WrappedState, ) -> bool { - if let Some(pk) = self.labels.get(peer) { + if let Some(pk) = self.labels.get(&peer.into()) { self.event_listener.has_put_contract(pk, key, value) } else { panic!("peer not found"); } } - pub fn has_got_contract(&self, peer: &NodeLabel, key: &ContractKey) -> bool { - if let Some(pk) = self.labels.get(peer) { + pub fn has_got_contract(&self, peer: impl Into, key: &ContractKey) -> bool { + if let Some(pk) = self.labels.get(&peer.into()) { self.event_listener.has_got_contract(pk, key) } else { panic!("peer not found"); } } + pub fn is_subscribed_to_contract(&self, peer: impl Into, key: &ContractKey) -> bool { + if let Some(pk) = self.labels.get(&peer.into()) { + self.event_listener.is_subscribed_to_contract(pk, key) + } else { + panic!("peer not found"); + } + } + /// Builds an histogram of the distribution in the ring of each node relative to each other. pub fn ring_distribution(&self, scale: i32) -> Vec<(f64, usize)> { let mut all_dists = Vec::with_capacity(self.labels.len()); @@ -395,15 +413,20 @@ impl SimNetwork { peers_connections } + /// # Arguments + /// + /// - label: node for which to trigger the + /// - event_id: which event to trigger + /// - await_for: if set, wait for the duration before returning pub async fn trigger_event( &self, - label: &NodeLabel, + label: impl Into, event_id: EventId, await_for: Option, ) -> Result<(), anyhow::Error> { let peer = self .labels - .get(label) + .get(&label.into()) .ok_or_else(|| anyhow::anyhow!("node not found"))?; self.user_ev_controller .send((event_id, *peer)) diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index 6c5b766e9..cef8a097a 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -126,7 +126,7 @@ where } pub(crate) enum OpEnum { - JoinRing(Box), + Connect(Box), Put(put::PutOp), Get(get::GetOp), Subscribe(subscribe::SubscribeOp), @@ -136,7 +136,7 @@ pub(crate) enum OpEnum { impl OpEnum { delegate::delegate! { to match self { - OpEnum::JoinRing(op) => op, + OpEnum::Connect(op) => op, OpEnum::Put(op) => op, OpEnum::Get(op) => op, OpEnum::Subscribe(op) => op, diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index 0f34f3290..01962d34e 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -20,7 +20,7 @@ pub(crate) use self::messages::{ConnectMsg, ConnectRequest, ConnectResponse}; pub(crate) struct ConnectOp { id: Transaction, - state: Option, + state: Option, pub gateway: Box, /// keeps track of the number of retries and applies an exponential backoff cooldown period pub backoff: Option, @@ -38,7 +38,7 @@ impl ConnectOp { } pub(super) fn finalized(&self) -> bool { - matches!(self.state, Some(JRState::Connected)) + matches!(self.state, Some(ConnectState::Connected)) } pub(super) fn record_transfer(&mut self) {} @@ -86,11 +86,11 @@ impl Operation for ConnectOp { let sender; let tx = *msg.id(); match op_storage.pop(msg.id()) { - Some(OpEnum::JoinRing(join_op)) => { + Some(OpEnum::Connect(connect_op)) => { sender = msg.sender().cloned(); // was an existing operation, the other peer messaged back Ok(OpInitialization { - op: *join_op, + op: *connect_op, sender, }) } @@ -100,7 +100,7 @@ impl Operation for ConnectOp { Ok(OpInitialization { op: Self { id: tx, - state: Some(JRState::Initializing), + state: Some(ConnectState::Initializing), backoff: None, gateway: Box::new(op_storage.ring.own_location()), _ttl: PEER_TIMEOUT, @@ -141,7 +141,7 @@ impl Operation for ConnectOp { // likely a gateway which accepts connections tracing::debug!( tx = %id, - "Initial join request received from {} with HTL {} @ {}", + "Connection request received from {} with HTL {} @ {}", joiner, hops_to_live, this_node_loc.peer @@ -190,7 +190,7 @@ impl Operation for ConnectOp { this_node_loc.peer, joiner ); - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); } else { new_state = None } @@ -221,7 +221,7 @@ impl Operation for ConnectOp { let own_loc = op_storage.ring.own_location(); tracing::debug!( tx = %id, - "Proxy join request received from {} to join new peer {} with HTL {} @ {}", + "Proxy connect request received from {} to ing new peer {} with HTL {} @ {}", sender.peer, joiner.peer, hops_to_live, @@ -259,7 +259,7 @@ impl Operation for ConnectOp { return_msg = None; } else { match self.state { - Some(JRState::Initializing) => { + Some(ConnectState::Initializing) => { let (state, msg) = try_proxy_connection( &id, &sender, @@ -269,7 +269,7 @@ impl Operation for ConnectOp { new_state = state; return_msg = msg; } - Some(JRState::AwaitingProxyResponse { + Some(ConnectState::AwaitingProxyResponse { accepted_by: mut previously_accepted, new_peer_id, target, @@ -285,7 +285,7 @@ impl Operation for ConnectOp { } if match_target { - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); tracing::debug!( tx = %id, "Sending response to join request with all the peers that accepted \ @@ -309,10 +309,10 @@ impl Operation for ConnectOp { // is that we would end up with a dead connection; // this then must be dealed with by the normal mechanisms that keep // connections alive and prune any dead connections - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); tracing::debug!( tx = %id, - "Sending response to join request with all the peers that accepted \ + "Sending response to connect request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", sender.peer, own_loc.peer @@ -347,7 +347,7 @@ impl Operation for ConnectOp { }, .. } => { - tracing::debug!(tx = %id, "Join response received from {}", sender.peer); + tracing::debug!(tx = %id, "Connect response received from {}", sender.peer); // Set the given location let pk_loc = PeerKeyLocation { @@ -357,7 +357,7 @@ impl Operation for ConnectOp { // fixme: remove tracing::debug!("accepted by state: {:?} ", self.state,); - let Some(JRState::Connecting(ConnectionInfo { gateway, .. })) = self.state + let Some(ConnectState::Connecting(ConnectionInfo { gateway, .. })) = self.state else { return Err(OpError::InvalidStateTransition(self.id)); }; @@ -369,7 +369,7 @@ impl Operation for ConnectOp { your_peer_id, gateway.peer ); - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); return_msg = Some(ConnectMsg::Response { id, msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, @@ -416,7 +416,7 @@ impl Operation for ConnectOp { op_storage .notify_op_change( Message::Aborted(id), - OpEnum::JoinRing(op.into()), + OpEnum::Connect(op.into()), None, ) .await?; @@ -429,9 +429,9 @@ impl Operation for ConnectOp { target, msg: ConnectResponse::Proxy { mut accepted_by }, } => { - tracing::debug!(tx = %id, "Received proxy join at @ {}", target.peer); + tracing::debug!(tx = %id, "Received proxy connect at @ {}", target.peer); match self.state { - Some(JRState::Initializing) => { + Some(ConnectState::Initializing) => { // the sender of the response is the target of the request and // is only a completed tx if it accepted the connection if accepted_by.contains(&sender) { @@ -441,7 +441,7 @@ impl Operation for ConnectOp { target.peer, sender.peer, ); - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); } else { tracing::debug!("Failed to connect at proxy {}", sender.peer); new_state = None; @@ -453,7 +453,7 @@ impl Operation for ConnectOp { target, }); } - Some(JRState::AwaitingProxyResponse { + Some(ConnectState::AwaitingProxyResponse { accepted_by: mut previously_accepted, new_peer_id, target: original_target, @@ -467,21 +467,21 @@ impl Operation for ConnectOp { if is_accepted { previously_accepted.extend(accepted_by.drain()); if is_target_peer { - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); } else { // for proxies just consider the connection open directly // what would happen in case that the connection is not confirmed end-to-end // is that we would end up with a dead connection; // this then must be dealed with by the normal mechanisms that keep // connections alive and prune any dead connections - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); } } if is_target_peer { tracing::debug!( tx = %id, - "Sending response to join request with all the peers that accepted \ + "Sending response to connect request with all the peers that accepted \ connection from gateway {} to peer {}", target.peer, original_target.peer @@ -499,7 +499,7 @@ impl Operation for ConnectOp { } else { tracing::debug!( tx = %id, - "Sending response to join request with all the peers that accepted \ + "Sending response to connect request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", target.peer, original_target.peer @@ -532,9 +532,9 @@ impl Operation for ConnectOp { target, } => { match self.state { - Some(JRState::OCReceived) => { + Some(ConnectState::OCReceived) => { tracing::debug!(tx = %id, "Acknowledge connected at gateway"); - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); return_msg = Some(ConnectMsg::Connected { id, sender: target, @@ -559,9 +559,9 @@ impl Operation for ConnectOp { } ConnectMsg::Connected { target, sender, id } => { match self.state { - Some(JRState::OCReceived) => { + Some(ConnectState::OCReceived) => { tracing::debug!(tx = %id, "Acknowledge connected at peer {}", target.peer); - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); return_msg = None; } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -602,7 +602,7 @@ impl Operation for ConnectOp { fn build_op_result( id: Transaction, - state: Option, + state: Option, msg: Option, gateway: Box, backoff: Option, @@ -617,7 +617,7 @@ fn build_op_result( }); Ok(OperationResult { return_msg: msg.map(Message::from), - state: output_op.map(|op| OpEnum::JoinRing(Box::new(op))), + state: output_op.map(|op| OpEnum::Connect(Box::new(op))), }) } @@ -626,7 +626,7 @@ fn try_proxy_connection( sender: &PeerKeyLocation, own_loc: &PeerKeyLocation, accepted_by: HashSet, -) -> (Option, Option) { +) -> (Option, Option) { let new_state = if accepted_by.contains(own_loc) { tracing::debug!( tx = %id, @@ -634,7 +634,7 @@ fn try_proxy_connection( sender.peer, own_loc.peer, ); - Some(JRState::Connected) + Some(ConnectState::Connected) } else { tracing::debug!(tx = %id, "Failed to connect at proxy {}", sender.peer); None @@ -685,7 +685,7 @@ mod states { use super::*; use std::fmt::Display; - impl Display for JRState { + impl Display for ConnectState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Initializing => write!(f, "Initializing"), @@ -701,7 +701,7 @@ mod states { } #[derive(Debug)] -enum JRState { +enum ConnectState { Initializing, Connecting(ConnectionInfo), AwaitingProxyResponse { @@ -722,7 +722,7 @@ struct ConnectionInfo { max_hops_to_live: usize, } -impl JRState { +impl ConnectState { fn try_unwrap_connecting(self) -> Result { if let Self::Connecting(conn_info) = self { Ok(conn_info) @@ -732,7 +732,7 @@ impl JRState { } fn is_connected(&self) -> bool { - matches!(self, JRState::Connected { .. }) + matches!(self, ConnectState::Connected { .. }) } fn add_new_proxy( @@ -755,8 +755,8 @@ pub(crate) fn initial_request( id: Transaction, ) -> ConnectOp { const MAX_JOIN_RETRIES: usize = 3; - tracing::debug!(tx = %id, "Connecting to gw {} from {}", gateway.peer, this_peer); - let state = JRState::Connecting(ConnectionInfo { + tracing::debug!(tx = %id, "Connecting to gateway {} from {}", gateway.peer, this_peer); + let state = ConnectState::Connecting(ConnectionInfo { gateway, this_peer, max_hops_to_live, @@ -780,7 +780,7 @@ pub(crate) fn initial_request( } /// Join ring routine, called upon performing a join operation for this node. -pub(crate) async fn join_ring_request( +pub(crate) async fn connect_request( tx: Transaction, op_storage: &OpManager, conn_manager: &mut CB, @@ -804,7 +804,7 @@ where tracing::info!( tx = %id, - "Joining ring via {} (at {})", + "Connecting to peer {} (at {})", gateway.peer, gateway.location.ok_or(ConnectionError::LocationUnknown)?, ); @@ -824,9 +824,9 @@ where conn_manager.send(&gateway.peer, join_req).await?; op_storage.push( tx, - OpEnum::JoinRing(Box::new(ConnectOp { + OpEnum::Connect(Box::new(ConnectOp { id, - state: Some(JRState::Connecting(ConnectionInfo { + state: Some(ConnectState::Connecting(ConnectionInfo { gateway, this_peer, max_hops_to_live, @@ -848,7 +848,7 @@ async fn forward_conn( joiner: PeerKeyLocation, left_htl: usize, num_accepted: usize, -) -> Result, OpError> +) -> Result, OpError> where CM: ConnectionBridge, { @@ -856,7 +856,7 @@ where tracing::debug!( tx = %id, joiner = %joiner.peer, - "Couldn't forward join petition, no hops left or enough connections", + "Couldn't forward connect petition, no hops left or enough connections", ); return Ok(None); } @@ -865,7 +865,7 @@ where tracing::warn!( tx = %id, joiner = %joiner.peer, - "Couldn't forward join petition, not enough connections", + "Couldn't forward connect petition, not enough connections", ); return Ok(None); } @@ -874,7 +874,7 @@ where tracing::debug!( tx = %id, joiner = %joiner.peer, - "Randomly selecting peer to forward JoinRequest", + "Randomly selecting peer to forward connect request", ); ring.random_peer(|p| p != &req_peer.peer) } else { @@ -899,13 +899,13 @@ where }); tracing::debug!( tx = %id, - "Forwarding JoinRequest from sender {} to {}", + "Forwarding connect request from sender {} to {}", req_peer.peer, forward_to.peer ); conn_manager.send(&forward_to.peer, forwarded).await?; // awaiting for responses from forward nodes - let new_state = JRState::AwaitingProxyResponse { + let new_state = ConnectState::AwaitingProxyResponse { target: req_peer, accepted_by: HashSet::new(), new_location: joiner.location.unwrap(), diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 9a5733420..8c3cf4d70 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -236,6 +236,7 @@ impl Operation for GetOp { if !is_cached_contract { tracing::warn!( + tx = %id, "Contract `{}` not found while processing a get request at node @ {}", key, target.peer @@ -243,6 +244,7 @@ impl Operation for GetOp { if htl == 0 { tracing::warn!( + tx = %id, "The maximum HOPS number has been exceeded, sending the error \ back to the node @ {}", sender.peer @@ -271,7 +273,7 @@ impl Operation for GetOp { let Some(new_target) = op_storage.ring.closest_caching(&key, &[sender.peer]) else { - tracing::warn!("no peer found while trying getting contract {key}"); + tracing::warn!(tx = %id, "no peer found while trying getting contract {key}"); return Err(OpError::RingError(RingError::NoCachingPeers(key))); }; @@ -316,24 +318,25 @@ impl Operation for GetOp { Err(err) => return Err(err), } - tracing::debug!("Contract {returned_key} found @ peer {}", target.peer); + tracing::debug!(tx = %id, "Contract {returned_key} found @ peer {}", target.peer); match self.state { Some(GetState::AwaitingResponse { .. }) => { tracing::debug!( - "Completed operation, Get response received for contract {key}" + tx = %id, + "Completed operation, get response received for contract {key}" ); // Completed op new_state = None; return_msg = None; } Some(GetState::ReceivedRequest) => { - tracing::debug!("Returning contract {} to {}", key, sender.peer); + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; let value = match value { Ok(res) => res, Err(err) => { - tracing::error!("error: {err}"); + tracing::error!(tx = %id, "error: {err}"); return Err(OpError::ExecutorError(err)); } }; @@ -365,6 +368,7 @@ impl Operation for GetOp { } => { let this_loc = target; tracing::warn!( + tx = %id, "Neither contract or contract value for contract `{}` found at peer {}, \ retrying with other peers", key, @@ -406,6 +410,7 @@ impl Operation for GetOp { }); } else { tracing::error!( + tx = %id, "Failed getting a value for contract {}, reached max retries", key ); @@ -413,7 +418,7 @@ impl Operation for GetOp { } } Some(GetState::ReceivedRequest) => { - tracing::debug!("Returning contract {} to {}", key, sender.peer); + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; return_msg = Some(GetMsg::ReturnGet { id, @@ -459,10 +464,11 @@ impl Operation for GetOp { ) .await?; let key = contract.key(); - tracing::debug!("Contract `{}` successfully cached", key); + tracing::debug!(tx = %id, "Contract `{}` successfully cached", key); } else { // no contract, consider this like an error ignoring the incoming update value tracing::warn!( + tx = %id, "Contract not received from peer {} while requested", sender.peer ); @@ -501,6 +507,7 @@ impl Operation for GetOp { ContractHandlerEvent::PutQuery { key: key.clone(), state: value.clone(), + related_contracts: RelatedContracts::default(), parameters, }, client_id, @@ -511,6 +518,7 @@ impl Operation for GetOp { Some(GetState::AwaitingResponse { fetch_contract, .. }) => { if fetch_contract && contract.is_none() { tracing::error!( + tx = %id, "Get response received for contract {key}, but the contract wasn't returned" ); new_state = None; @@ -520,7 +528,7 @@ impl Operation for GetOp { contract, }); } else { - tracing::debug!("Get response received for contract {}", key); + tracing::debug!(tx = %id, "Get response received for contract {}", key); new_state = None; return_msg = None; result = Some(GetResult { @@ -530,7 +538,7 @@ impl Operation for GetOp { } } Some(GetState::ReceivedRequest) => { - tracing::debug!("Returning contract {} to {}", key, sender.peer); + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; return_msg = Some(GetMsg::ReturnGet { id, @@ -581,6 +589,7 @@ async fn continue_seeking( retry_msg: Message, ) -> Result<(), OpError> { tracing::info!( + tx = %retry_msg.id(), "Retrying to get the contract from node @ {}", new_target.peer ); @@ -600,6 +609,7 @@ fn check_contract_found( if returned_key != key { // shouldn't be a reachable path tracing::error!( + tx = %id, "contract retrieved ({}) and asked ({}) are not the same", returned_key, key @@ -622,9 +632,8 @@ fn check_contract_found( pub(crate) fn start_op(key: ContractKey, fetch_contract: bool) -> GetOp { let contract_location = Location::from(&key); - tracing::debug!("Requesting get contract {} @ loc({contract_location})", key,); - let id = Transaction::new::(); + tracing::debug!(tx = %id, "Requesting get contract {key} @ loc({contract_location})"); let state = Some(GetState::PrepareRequest { key, id, @@ -866,10 +875,9 @@ mod test { // trigger get @ node-0, which does not own the contract sim_nw - .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(100))) + .trigger_event("node-0", 1, Some(Duration::from_millis(50))) .await?; - tokio::time::sleep(Duration::from_millis(200)).await; - assert!(sim_nw.has_got_contract(&"node-0".into(), &key)); + assert!(sim_nw.has_got_contract("node-0", &key)); Ok(()) } @@ -905,9 +913,9 @@ mod test { // trigger get @ node-1, which does not own the contract sim_nw - .trigger_event(&"node-1".into(), 1, Some(Duration::from_millis(100))) + .trigger_event("node-1", 1, Some(Duration::from_millis(50))) .await?; - assert!(!sim_nw.has_got_contract(&"node-1".into(), &key)); + assert!(!sim_nw.has_got_contract("node-1", &key)); Ok(()) } @@ -973,9 +981,9 @@ mod test { sim_nw.check_connectivity(Duration::from_secs(3)).await?; sim_nw - .trigger_event(&"node-0".into(), 1, Some(Duration::from_millis(500))) + .trigger_event("node-0", 1, Some(Duration::from_millis(50))) .await?; - assert!(sim_nw.has_got_contract(&"node-0".into(), &key)); + assert!(sim_nw.has_got_contract("node-0", &key)); Ok(()) } } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 84ab12e7b..11bd6bbf9 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -178,6 +178,7 @@ impl Operation for PutOp { PutMsg::RequestPut { id, contract, + related_contracts, value, htl, target, @@ -198,6 +199,7 @@ impl Operation for PutOp { target, value, contract, + related_contracts, htl, skip_list: vec![sender.peer], }); @@ -210,6 +212,7 @@ impl Operation for PutOp { sender, value, contract, + related_contracts, htl, target, mut skip_list, @@ -246,8 +249,15 @@ impl Operation for PutOp { // after the contract has been cached, push the update query tracing::debug!("Attempting contract value update"); let parameters = contract.params(); - let new_value = - put_contract(op_storage, key.clone(), value, parameters, client_id).await?; + let new_value = put_contract( + op_storage, + key.clone(), + value, + related_contracts, + parameters, + client_id, + ) + .await?; tracing::debug!("Contract successfully updated"); // if the change was successful, communicate this back to the requestor and broadcast the change conn_manager @@ -320,6 +330,7 @@ impl Operation for PutOp { op_storage, key.clone(), new_value, + RelatedContracts::default(), parameters.clone(), client_id, ) @@ -468,9 +479,15 @@ impl Operation for PutOp { }); } // after the contract has been cached, push the update query - let new_value = - put_contract(op_storage, key, new_value, contract.params(), client_id) - .await?; + let new_value = put_contract( + op_storage, + key, + new_value, + RelatedContracts::default(), + contract.params(), + client_id, + ) + .await?; //update skip list skip_list.push(peer_loc.peer); @@ -597,7 +614,12 @@ async fn try_to_broadcast( Ok((new_state, return_msg)) } -pub(crate) fn start_op(contract: ContractContainer, value: WrappedState, htl: usize) -> PutOp { +pub(crate) fn start_op( + contract: ContractContainer, + related_contracts: RelatedContracts<'static>, + value: WrappedState, + htl: usize, +) -> PutOp { let key = contract.key(); let contract_location = Location::from(&key); tracing::debug!( @@ -609,6 +631,7 @@ pub(crate) fn start_op(contract: ContractContainer, value: WrappedState, htl: us // let payload_size = contract.data().len(); let state = Some(PutState::PrepareRequest { contract, + related_contracts, value, htl, }); @@ -632,6 +655,7 @@ enum PutState { ReceivedRequest, PrepareRequest { contract: ContractContainer, + related_contracts: RelatedContracts<'static>, value: WrappedState, htl: usize, }, @@ -675,13 +699,14 @@ pub(crate) async fn request_put( contract, value, htl, - .. + related_contracts, }) => { let key = contract.key(); let new_state = Some(PutState::AwaitingResponse { contract: key }); let msg = PutMsg::RequestPut { id, contract, + related_contracts, value, htl, target, @@ -708,6 +733,7 @@ async fn put_contract( op_storage: &OpManager, key: ContractKey, state: WrappedState, + related_contracts: RelatedContracts<'static>, parameters: Parameters<'static>, client_id: Option, ) -> Result { @@ -717,6 +743,7 @@ async fn put_contract( ContractHandlerEvent::PutQuery { key, state, + related_contracts, parameters: Some(parameters), }, client_id, @@ -800,6 +827,8 @@ mod messages { RequestPut { id: Transaction, contract: ContractContainer, + #[serde(deserialize_with = "RelatedContracts::deser_related_contracts")] + related_contracts: RelatedContracts<'static>, value: WrappedState, /// max hops to live htl: usize, @@ -828,6 +857,8 @@ mod messages { target: PeerKeyLocation, value: WrappedState, contract: ContractContainer, + #[serde(deserialize_with = "RelatedContracts::deser_related_contracts")] + related_contracts: RelatedContracts<'static>, /// max hops to live htl: usize, // FIXME: remove skip list once we deduplicate at top msg handling level @@ -926,7 +957,6 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_put_op_between_nodes() -> Result<(), anyhow::Error> { - crate::config::set_logger(); const NUM_NODES: usize = 2usize; const NUM_GW: usize = 1usize; @@ -941,9 +971,9 @@ mod test { "successful_put_op_between_nodes", NUM_GW, NUM_NODES, - 3, 2, - 4, + 1, + 3, 2, ) .await; @@ -997,14 +1027,13 @@ mod test { ]); sim_nw.start_with_spec(put_specs).await; - tokio::time::sleep(Duration::from_secs(5)).await; sim_nw.check_connectivity(Duration::from_secs(3)).await?; // trigger the put op @ gw-0 sim_nw - .trigger_event(&"gateway-0".into(), 1, Some(Duration::from_secs(3))) + .trigger_event("gateway-0", 1, Some(Duration::from_millis(100))) .await?; - assert!(sim_nw.has_put_contract(&"gateway-0".into(), &key, &new_value)); + assert!(sim_nw.has_put_contract("gateway-0", &key, &new_value)); assert!(sim_nw.event_listener.contract_broadcasted(&key)); Ok(()) } diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 5dd708001..c7dc67cf6 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -146,13 +146,13 @@ impl Operation for SubscribeOp { }; if !op_storage.ring.is_contract_cached(&key) { - tracing::info!("Contract {} not found while processing info", key); - tracing::info!("Trying to found the contract from another node"); + tracing::info!(tx = %id, "Contract {} not found while processing info", key); + tracing::info!(tx = %id, "Trying to found the contract from another node"); let Some(new_target) = op_storage.ring.closest_caching(&key, &[sender.peer]) else { - tracing::warn!("no peer found while trying getting contract {key}"); + tracing::warn!(tx = %id, "No peer found while trying getting contract {key}"); return Err(OpError::RingError(RingError::NoCachingPeers(key))); }; let new_htl = htl + 1; @@ -187,9 +187,9 @@ impl Operation for SubscribeOp { match self.state { Some(SubscribeState::ReceivedRequest) => { tracing::info!( - "Peer {} successfully subscribed to contract {}", + tx = %id, + "Peer {} successfully subscribed to contract {key}", subscriber.peer, - key ); new_state = Some(SubscribeState::Completed); // TODO review behaviour, if the contract is not cached should return subscribed false? @@ -212,8 +212,8 @@ impl Operation for SubscribeOp { id, } => { tracing::warn!( - "Contract `{}` not found at potential subscription provider {}", - key, + tx = %id, + "Contract `{key}` not found at potential subscription provider {}", sender.peer ); // will error out in case it has reached max number of retries @@ -261,14 +261,10 @@ impl Operation for SubscribeOp { target: _, id: _, } => { - tracing::warn!( - "Subscribed to `{}` not found at potential subscription provider {}", - key, - sender.peer - ); + tracing::warn!("Subscribed to `{key}` at provider {}", sender.peer); op_storage.ring.add_subscription(key); - let _ = client_id; // todo: should inform back to the network event loop? + let _ = client_id; match self.state { Some(SubscribeState::AwaitingResponse { .. }) => { @@ -465,9 +461,9 @@ mod test { use super::*; use crate::node::tests::{NodeSpecification, SimNetwork}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn successful_subscribe_op_between_nodes() -> Result<(), anyhow::Error> { + crate::config::set_logger(); const NUM_NODES: usize = 4usize; const NUM_GW: usize = 1usize; @@ -483,13 +479,6 @@ mod test { } .into(); let first_node = NodeSpecification { - owned_contracts: Vec::new(), - non_owned_contracts: vec![contract_key], - events_to_generate: HashMap::from_iter([(1, event)]), - contract_subscribers: HashMap::new(), - }; - - let second_node = NodeSpecification { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, @@ -498,6 +487,12 @@ mod test { events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), }; + let second_node = NodeSpecification { + owned_contracts: Vec::new(), + non_owned_contracts: vec![contract_key.clone()], + events_to_generate: HashMap::from_iter([(1, event)]), + contract_subscribers: HashMap::new(), + }; let subscribe_specs = HashMap::from_iter([ ("node-0".into(), first_node), @@ -507,15 +502,21 @@ mod test { "successful_subscribe_op_between_nodes", NUM_GW, NUM_NODES, + 4, 3, - 2, 4, 2, ) .await; + sim_nw.with_start_backoff(Duration::from_millis(150)); sim_nw.start_with_spec(subscribe_specs).await; sim_nw.check_connectivity(Duration::from_secs(3)).await?; - + sim_nw + .trigger_event("node-1", 1, Some(Duration::from_millis(50))) + .await?; + assert!(sim_nw.has_got_contract("node-1", &contract_key)); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(sim_nw.is_subscribed_to_contract("node-1", &contract_key)); Ok(()) } } diff --git a/crates/core/src/runtime/mod.rs b/crates/core/src/runtime.rs similarity index 100% rename from crates/core/src/runtime/mod.rs rename to crates/core/src/runtime.rs diff --git a/crates/core/src/server/mod.rs b/crates/core/src/server.rs similarity index 100% rename from crates/core/src/server/mod.rs rename to crates/core/src/server.rs diff --git a/crates/core/src/server/errors.rs b/crates/core/src/server/errors.rs index c63cfc2f4..f62c79467 100644 --- a/crates/core/src/server/errors.rs +++ b/crates/core/src/server/errors.rs @@ -1,6 +1,7 @@ use axum::http::StatusCode; use axum::response::{Html, IntoResponse, Response}; use freenet_stdlib::client_api::ErrorKind; +use freenet_stdlib::prelude::ContractKey; use std::fmt::{Display, Formatter}; #[derive(Debug)] @@ -15,14 +16,18 @@ pub(super) enum WebSocketApiError { AxumError { error: ErrorKind, }, + MissingContract { + key: ContractKey, + }, } impl WebSocketApiError { pub fn status_code(&self) -> StatusCode { match self { WebSocketApiError::InvalidParam { .. } => StatusCode::BAD_REQUEST, - WebSocketApiError::NodeError { .. } => StatusCode::BAD_GATEWAY, + WebSocketApiError::NodeError { .. } => StatusCode::INTERNAL_SERVER_ERROR, WebSocketApiError::AxumError { .. } => StatusCode::INTERNAL_SERVER_ERROR, + WebSocketApiError::MissingContract { .. } => StatusCode::NOT_FOUND, } } @@ -32,7 +37,8 @@ impl WebSocketApiError { format!("Invalid request params: {}", error_cause) } WebSocketApiError::NodeError { error_cause } => format!("Node error: {}", error_cause), - WebSocketApiError::AxumError { error } => format!("Axum error: {}", error), + WebSocketApiError::AxumError { error } => format!("Server error: {}", error), + WebSocketApiError::MissingContract { key } => format!("Missing contract {key}"), } } } @@ -61,7 +67,12 @@ impl IntoResponse for WebSocketApiError { { (StatusCode::NOT_FOUND, error_cause) } - WebSocketApiError::NodeError { error_cause } => (StatusCode::BAD_GATEWAY, error_cause), + WebSocketApiError::NodeError { error_cause } => { + (StatusCode::INTERNAL_SERVER_ERROR, error_cause) + } + err @ WebSocketApiError::MissingContract { .. } => { + (StatusCode::NOT_FOUND, err.error_message()) + } WebSocketApiError::AxumError { error } => { (StatusCode::INTERNAL_SERVER_ERROR, format!("{error}")) } diff --git a/crates/core/src/server/path_handlers.rs b/crates/core/src/server/path_handlers.rs index 02b341d7c..a7184b392 100644 --- a/crates/core/src/server/path_handlers.rs +++ b/crates/core/src/server/path_handlers.rs @@ -45,7 +45,9 @@ pub(super) async fn contract_home( let client_id = if let Some(HostCallbackResult::NewId { id }) = response_recv.recv().await { id } else { - todo!("this is an error"); + return Err(WebSocketApiError::NodeError { + error_cause: "Couldn't register new client in the node".into(), + }); }; request_sender .send(ClientConnection::Request { @@ -120,7 +122,7 @@ pub(super) async fn contract_home( web_body } None => { - todo!("error indicating the contract is not present"); + return Err(WebSocketApiError::MissingContract { key }); } }, Some(HostCallbackResult::Result { diff --git a/stdlib b/stdlib index bf0da9a67..e6b4b32d0 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit bf0da9a67429e50439b41a4dcd7668b8f119acf1 +Subproject commit e6b4b32d0db22a803a1b0f46ea9aec48829443b7 From 75da931ee419dd475b614193825a7248a910dec7 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 20 Oct 2023 13:55:31 +0200 Subject: [PATCH 11/14] Fix issue of not caching contract after get --- crates/core/src/contract.rs | 6 ++- crates/core/src/node.rs | 4 +- crates/core/src/node/tests.rs | 2 +- crates/core/src/operations/get.rs | 57 +++++++++++++++++++------ crates/core/src/operations/put.rs | 2 +- crates/core/src/operations/subscribe.rs | 7 ++- 6 files changed, 55 insertions(+), 23 deletions(-) diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index 1b621dfd8..a8815f433 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -30,7 +30,7 @@ where { loop { let (id, event) = contract_handler.channel().recv_from_event_loop().await?; - tracing::debug!(%event, "got contract handling event"); + tracing::debug!(%event, "Got contract handling event"); match event { ContractHandlerEvent::GetQuery { key, @@ -42,6 +42,7 @@ where .await { Ok((state, contract)) => { + tracing::debug!("Fetched contract {key}"); contract_handler .channel() .send_to_event_loop( @@ -57,7 +58,7 @@ where .await?; } Err(err) => { - tracing::warn!("error while executing get contract query: {err}"); + tracing::warn!("Error while executing get contract query: {err}"); contract_handler .channel() .send_to_event_loop( @@ -126,4 +127,5 @@ pub(crate) enum ContractError { IOError(#[from] std::io::Error), #[error("no response received from handler")] NoEvHandlerResponse, + } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index da34531b6..cafa2f7a6 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -413,7 +413,8 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { - tokio::time::sleep(Duration::from_secs(1)).await + tracing::warn!("Still waiting for {key} contract"); + tokio::time::sleep(Duration::from_secs(2)).await } Err(err) => { tracing::error!("{}", err); @@ -425,6 +426,7 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc = expected .difference(&connected) diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 8c3cf4d70..de16f53e3 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -29,7 +29,7 @@ const MAX_GET_RETRY_HOPS: usize = 1; pub(crate) struct GetOp { id: Transaction, state: Option, - result: Option, + pub(super) result: Option, stats: Option, _ttl: Duration, } @@ -86,10 +86,7 @@ impl GetOp { } pub(super) fn finalized(&self) -> bool { - self.stats - .as_ref() - .map(|s| s.transfer_time.is_some()) - .unwrap_or(false) + self.result.is_some() } pub(super) fn record_transfer(&mut self) { @@ -212,11 +209,13 @@ impl Operation for GetOp { first_response_time: None, step: Default::default(), }); + let own_loc = op_storage.ring.own_location(); return_msg = Some(GetMsg::SeekNode { key, id, target, - sender: op_storage.ring.own_location(), + skip_list: vec![own_loc.peer], + sender: own_loc, fetch_contract, htl: MAX_GET_RETRY_HOPS, }); @@ -227,12 +226,14 @@ impl Operation for GetOp { fetch_contract, sender, target, + mut skip_list, htl, } => { let is_cached_contract = op_storage.ring.is_contract_cached(&key); if let Some(s) = stats.as_mut() { s.caching_peer = Some(target); } + skip_list.push(target.peer); if !is_cached_contract { tracing::warn!( @@ -261,6 +262,7 @@ impl Operation for GetOp { contract: None, }, sender: op_storage.ring.own_location(), + updated_skip_list: skip_list, target: sender, // return to requester }), None, @@ -270,13 +272,11 @@ impl Operation for GetOp { } let new_htl = htl - 1; - let Some(new_target) = - op_storage.ring.closest_caching(&key, &[sender.peer]) + let Some(new_target) = op_storage.ring.closest_caching(&key, &skip_list) else { - tracing::warn!(tx = %id, "no peer found while trying getting contract {key}"); + tracing::warn!(tx = %id, "No other peers found while trying getting contract {key} @ {}", target.peer); return Err(OpError::RingError(RingError::NoCachingPeers(key))); }; - continue_seeking( conn_manager, &new_target, @@ -286,6 +286,7 @@ impl Operation for GetOp { fetch_contract, sender, target: new_target, + skip_list, htl: new_htl, }) .into(), @@ -344,6 +345,7 @@ impl Operation for GetOp { id, key, value, + updated_skip_list: vec![], sender: target, target: sender, }); @@ -364,7 +366,7 @@ impl Operation for GetOp { }, sender, target, - .. + updated_skip_list, } => { let this_loc = target; tracing::warn!( @@ -399,6 +401,7 @@ impl Operation for GetOp { sender: this_loc, fetch_contract, htl: MAX_GET_RETRY_HOPS, + skip_list: updated_skip_list, }); } else { return Err(RingError::NoCachingPeers(key).into()); @@ -427,6 +430,7 @@ impl Operation for GetOp { state: None, contract: None, }, + updated_skip_list, sender, target, }); @@ -444,7 +448,9 @@ impl Operation for GetOp { }, sender, target, + mut updated_skip_list, } => { + updated_skip_list.push(sender.peer); let require_contract = matches!( self.state, Some(GetState::AwaitingResponse { @@ -457,19 +463,28 @@ impl Operation for GetOp { if require_contract { if let Some(contract) = &contract { // store contract first - op_storage + let res = op_storage .notify_contract_handler( ContractHandlerEvent::Cache(contract.clone()), client_id, ) .await?; + match res { + ContractHandlerEvent::CacheResult(Ok(_)) => { + op_storage.ring.contract_cached(&key); + } + ContractHandlerEvent::CacheResult(Err(err)) => { + return Err(OpError::ContractError(err)); + } + _ => unreachable!(), + } let key = contract.key(); tracing::debug!(tx = %id, "Contract `{}` successfully cached", key); } else { // no contract, consider this like an error ignoring the incoming update value tracing::warn!( tx = %id, - "Contract not received from peer {} while requested", + "Contract not received from peer {} while required", sender.peer ); @@ -492,6 +507,7 @@ impl Operation for GetOp { }, sender, target, + updated_skip_list, }), OpEnum::Get(op), None, @@ -502,7 +518,7 @@ impl Operation for GetOp { } let parameters = contract.as_ref().map(|c| c.params()); - op_storage + let res = op_storage .notify_contract_handler( ContractHandlerEvent::PutQuery { key: key.clone(), @@ -513,6 +529,15 @@ impl Operation for GetOp { client_id, ) .await?; + match res { + ContractHandlerEvent::PutResponse { new_value: Ok(_) } => {} + ContractHandlerEvent::PutResponse { + new_value: Err(err), + } => { + return Err(OpError::ExecutorError(err)); + } + _ => unreachable!(), + } match self.state { Some(GetState::AwaitingResponse { fetch_contract, .. }) => { @@ -549,6 +574,7 @@ impl Operation for GetOp { }, sender, target, + updated_skip_list, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -763,6 +789,8 @@ mod messages { target: PeerKeyLocation, sender: PeerKeyLocation, htl: usize, + // FIXME: remove skip list once we deduplicate at top msg handling level + skip_list: Vec, }, ReturnGet { id: Transaction, @@ -770,6 +798,7 @@ mod messages { value: StoreResponse, sender: PeerKeyLocation, target: PeerKeyLocation, + updated_skip_list: Vec, }, } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 11bd6bbf9..d3e216453 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -535,7 +535,7 @@ fn build_op_result( }) } -async fn try_to_cache_contract<'a>( +pub(super) async fn try_to_cache_contract<'a>( op_storage: &'a OpManager, contract: &ContractContainer, key: &ContractKey, diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index c7dc67cf6..114d64fbf 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -504,18 +504,17 @@ mod test { NUM_NODES, 4, 3, - 4, + 5, 2, ) .await; - sim_nw.with_start_backoff(Duration::from_millis(150)); sim_nw.start_with_spec(subscribe_specs).await; sim_nw.check_connectivity(Duration::from_secs(3)).await?; sim_nw - .trigger_event("node-1", 1, Some(Duration::from_millis(50))) + .trigger_event("node-1", 1, Some(Duration::from_secs(2))) .await?; assert!(sim_nw.has_got_contract("node-1", &contract_key)); - tokio::time::sleep(Duration::from_secs(2)).await; + tokio::time::sleep(Duration::from_secs(3)).await; assert!(sim_nw.is_subscribed_to_contract("node-1", &contract_key)); Ok(()) } From 9a9b17172e9660af2069d48fa4cf28e128fdb636 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Mon, 23 Oct 2023 09:14:20 +0200 Subject: [PATCH 12/14] Pass subscribe op test --- crates/core/src/contract.rs | 6 +-- crates/core/src/contract/executor.rs | 1 + crates/core/src/contract/storages/sqlite.rs | 2 +- crates/core/src/node.rs | 24 ++++++++---- crates/core/src/operations/connect.rs | 2 +- crates/core/src/operations/get.rs | 9 ++--- crates/core/src/operations/subscribe.rs | 41 ++++++++++++++------- crates/core/src/runtime/error.rs | 6 +-- crates/core/src/runtime/secrets_store.rs | 2 +- crates/core/src/server/app_packaging.rs | 2 +- 10 files changed, 58 insertions(+), 37 deletions(-) diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index a8815f433..3a69540b4 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -81,6 +81,7 @@ where .await?; } Err(err) => { + tracing::error!("Error while caching: {err}"); let err = ContractError::ContractRuntimeError(err); contract_handler .channel() @@ -121,11 +122,10 @@ pub(crate) enum ContractError { ChannelDropped(Box), #[error("contract {0} not found in storage")] ContractNotFound(ContractKey), - #[error("")] - ContractRuntimeError(ContractRtError), #[error(transparent)] + ContractRuntimeError(ContractRtError), + #[error("{0}")] IOError(#[from] std::io::Error), #[error("no response received from handler")] NoEvHandlerResponse, - } diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index 864a6df18..58319f760 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -1191,6 +1191,7 @@ impl Executor { let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{data_dir}")); let contracts_data_dir = tmp_path.join("contracts"); + std::fs::create_dir_all(&contracts_data_dir).expect("directory created"); let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; // uses inmemory SQLite diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index e70f3bc71..9d25f6c31 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -129,7 +129,7 @@ pub enum SqlDbError { SqliteError(#[from] sqlx::Error), #[error(transparent)] RuntimeError(#[from] ContractError), - #[error(transparent)] + #[error("{0}")] IOError(#[from] std::io::Error), #[error(transparent)] StateStore(#[from] StateStoreError), diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index cafa2f7a6..558b57839 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -18,6 +18,7 @@ use std::{ use either::Either; use freenet_stdlib::client_api::{ClientRequest, ContractRequest}; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; +use tracing::Instrument; #[cfg(test)] use self::in_memory_impl::NodeInMemory; @@ -30,7 +31,7 @@ use crate::{ ClientResponses, ClientResponsesSender, ContractError, ExecutorToEventLoopChannel, NetworkContractHandler, NetworkEventListenerHalve, OperationMode, }, - message::{InnerMessage, Message, Transaction, TransactionType}, + message::{Message, Transaction, TransactionType}, operations::{ connect::{self, ConnectMsg, ConnectOp}, get, put, subscribe, OpEnum, OpError, OpOutcome, @@ -426,7 +427,7 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc, op_storage: Arc { tracing::debug!( @@ -515,8 +520,11 @@ async fn report_result( } } Ok(None) => {} + Err(OpError::OpNotPresent(tx)) => { + tracing::trace!(%tx, "Late message for finished transaction"); + } Err(err) => { - tracing::debug!("Finished transaction with error: {}", err) + tracing::debug!("Finished transaction with error: {err}") } } } @@ -541,7 +549,7 @@ async fn process_message( .await; match msg { Message::JoinRing(op) => { - log_handling_msg!("join", op.id(), op_storage); + // log_handling_msg!("join", op.id(), op_storage); let op_result = handle_op_request::( &op_storage, &mut conn_manager, @@ -559,7 +567,7 @@ async fn process_message( .await; } Message::Put(op) => { - log_handling_msg!("put", *op.id(), op_storage); + // log_handling_msg!("put", *op.id(), op_storage); let op_result = handle_op_request::( &op_storage, &mut conn_manager, @@ -577,7 +585,7 @@ async fn process_message( .await; } Message::Get(op) => { - log_handling_msg!("get", op.id(), op_storage); + // log_handling_msg!("get", op.id(), op_storage); let op_result = handle_op_request::( &op_storage, &mut conn_manager, @@ -595,7 +603,7 @@ async fn process_message( .await; } Message::Subscribe(op) => { - log_handling_msg!("subscribe", op.id(), op_storage); + // log_handling_msg!("subscribe", op.id(), op_storage); let op_result = handle_op_request::( &op_storage, &mut conn_manager, diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index 01962d34e..8f5c0543f 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -1111,7 +1111,7 @@ mod test { .await; // sim_nw.with_start_backoff(Duration::from_millis(100)); sim_nw.start().await; - sim_nw.check_connectivity(Duration::from_secs(5)).await?; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; let some_forwarded = sim_nw .node_connectivity() .into_iter() diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index de16f53e3..39ec3c90e 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -534,6 +534,7 @@ impl Operation for GetOp { ContractHandlerEvent::PutResponse { new_value: Err(err), } => { + tracing::debug!(tx = %id, error = %err, "Failed put at executor"); return Err(OpError::ExecutorError(err)); } _ => unreachable!(), @@ -614,14 +615,12 @@ async fn continue_seeking( new_target: &PeerKeyLocation, retry_msg: Message, ) -> Result<(), OpError> { - tracing::info!( + tracing::debug!( tx = %retry_msg.id(), - "Retrying to get the contract from node @ {}", + "Forwarding get request to {}", new_target.peer ); - conn_manager.send(&new_target.peer, retry_msg).await?; - Ok(()) } @@ -1003,7 +1002,7 @@ mod test { 3, 2, 4, - 3, + 2, ) .await; sim_nw.start_with_spec(get_specs).await; diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 114d64fbf..040c454c1 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -146,8 +146,7 @@ impl Operation for SubscribeOp { }; if !op_storage.ring.is_contract_cached(&key) { - tracing::info!(tx = %id, "Contract {} not found while processing info", key); - tracing::info!(tx = %id, "Trying to found the contract from another node"); + tracing::debug!(tx = %id, "Contract {} not found at {}, trying other peer", key, target.peer); let Some(new_target) = op_storage.ring.closest_caching(&key, &[sender.peer]) @@ -164,6 +163,7 @@ impl Operation for SubscribeOp { let mut new_skip_list = skip_list.clone(); new_skip_list.push(target.peer); + tracing::debug!(tx = %id, "Forward request to peer: {}", new_target.peer); // Retry seek node when the contract to subscribe has not been found in this node conn_manager .send( @@ -186,7 +186,7 @@ impl Operation for SubscribeOp { match self.state { Some(SubscribeState::ReceivedRequest) => { - tracing::info!( + tracing::debug!( tx = %id, "Peer {} successfully subscribed to contract {key}", subscriber.peer, @@ -258,20 +258,34 @@ impl Operation for SubscribeOp { subscribed: true, key, sender, - target: _, - id: _, + id, + target, + .. } => { - tracing::warn!("Subscribed to `{key}` at provider {}", sender.peer); - op_storage.ring.add_subscription(key); - // todo: should inform back to the network event loop? - let _ = client_id; - match self.state { Some(SubscribeState::AwaitingResponse { .. }) => { + tracing::debug!( + tx = %id, + target = ?target.peer, + this = ?op_storage.ring.own_location().peer, + "Subscribed to `{key}` at provider {}", sender.peer + ); + op_storage.ring.add_subscription(key); + // todo: should inform back to the network event loop? + let _ = client_id; new_state = None; return_msg = None; } - _ => return Err(OpError::InvalidStateTransition(self.id)), + state => { + tracing::error!( + tx = %id, + ?state, + target = ?target.peer, + this = ?op_storage.ring.own_location().peer, + "wrong state" + ); + return Err(OpError::InvalidStateTransition(self.id)); + } } } _ => return Err(OpError::UnexpectedOpState), @@ -288,9 +302,9 @@ fn build_op_result( msg: Option, ttl: Duration, ) -> Result { - let output_op = Some(SubscribeOp { + let output_op = state.map(|state| SubscribeOp { id, - state, + state: Some(state), _ttl: ttl, }); Ok(OperationResult { @@ -309,6 +323,7 @@ pub(crate) fn start_op(key: ContractKey) -> SubscribeOp { } } +#[derive(Debug)] enum SubscribeState { /// Prepare the request to subscribe. PrepareRequest { diff --git a/crates/core/src/runtime/error.rs b/crates/core/src/runtime/error.rs index a18b84064..7e855304b 100644 --- a/crates/core/src/runtime/error.rs +++ b/crates/core/src/runtime/error.rs @@ -8,7 +8,7 @@ use super::{delegate, secrets_store, wasm_runtime, DelegateExecError}; pub type RuntimeResult = std::result::Result; -#[derive(Debug)] +#[derive(thiserror::Error, Debug)] pub struct ContractError(Box); impl ContractError { @@ -65,8 +65,6 @@ impl Display for ContractError { } } -impl std::error::Error for ContractError {} - impl From for ContractError { fn from(err: RuntimeInnerError) -> Self { Self(Box::new(err)) @@ -104,7 +102,7 @@ pub(crate) enum RuntimeInnerError { #[error(transparent)] BufferError(#[from] freenet_stdlib::memory::buf::Error), - #[error(transparent)] + #[error("{0}")] IOError(#[from] std::io::Error), #[error(transparent)] diff --git a/crates/core/src/runtime/secrets_store.rs b/crates/core/src/runtime/secrets_store.rs index 486312892..092a7057d 100644 --- a/crates/core/src/runtime/secrets_store.rs +++ b/crates/core/src/runtime/secrets_store.rs @@ -54,7 +54,7 @@ impl StoreEntriesContainer for KeyToEncryptionMap { pub enum SecretStoreError { #[error("encryption error: {0}")] Encryption(EncryptionError), - #[error(transparent)] + #[error("{0}")] IO(#[from] std::io::Error), #[error("missing cipher")] MissingCipher, diff --git a/crates/core/src/server/app_packaging.rs b/crates/core/src/server/app_packaging.rs index ce241ca4b..5fa5744a0 100644 --- a/crates/core/src/server/app_packaging.rs +++ b/crates/core/src/server/app_packaging.rs @@ -12,7 +12,7 @@ use xz2::read::{XzDecoder, XzEncoder}; pub enum WebContractError { #[error("unpacking error: {0}")] UnpackingError(Box), - #[error(transparent)] + #[error("{0}")] StoringError(std::io::Error), #[error("file not found: {0}")] FileNotFound(String), From 25e0cd8a7eed7fd265ba29249a637e14b99aa634 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Tue, 24 Oct 2023 10:39:45 +0200 Subject: [PATCH 13/14] Handle concurrent same tx messages gracefully --- crates/core/src/message.rs | 20 +- crates/core/src/node.rs | 188 ++++++++++-------- .../core/src/node/conn_manager/p2p_protoc.rs | 8 +- crates/core/src/node/event_log.rs | 4 +- crates/core/src/node/in_memory_impl.rs | 2 +- crates/core/src/node/op_state_manager.rs | 46 +++-- crates/core/src/operations.rs | 8 +- crates/core/src/operations/connect.rs | 116 ++++++----- crates/core/src/operations/get.rs | 89 +++++---- crates/core/src/operations/op_trait.rs | 2 +- crates/core/src/operations/put.rs | 129 ++++++------ crates/core/src/operations/subscribe.rs | 81 ++++---- crates/core/src/operations/update.rs | 4 +- 13 files changed, 389 insertions(+), 308 deletions(-) diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index ef8046198..b35c821f3 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -86,7 +86,7 @@ mod sealed_msg_type { #[repr(u8)] #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] pub(crate) enum TransactionType { - JoinRing, + Connect, Put, Get, Subscribe, @@ -97,7 +97,7 @@ mod sealed_msg_type { impl Display for TransactionType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TransactionType::JoinRing => write!(f, "join ring"), + TransactionType::Connect => write!(f, "join ring"), TransactionType::Put => write!(f, "put"), TransactionType::Get => write!(f, "get"), TransactionType::Subscribe => write!(f, "subscribe"), @@ -126,7 +126,7 @@ mod sealed_msg_type { } transaction_type_enumeration!(decl struct { - JoinRing -> ConnectMsg, + Connect -> ConnectMsg, Put -> PutMsg, Get -> GetMsg, Subscribe -> SubscribeMsg, @@ -134,9 +134,9 @@ mod sealed_msg_type { }); } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize)] pub(crate) enum Message { - JoinRing(ConnectMsg), + Connect(ConnectMsg), Put(PutMsg), Get(GetMsg), Subscribe(SubscribeMsg), @@ -189,7 +189,7 @@ impl Message { pub fn id(&self) -> &Transaction { use Message::*; match self { - JoinRing(op) => op.id(), + Connect(op) => op.id(), Put(op) => op.id(), Get(op) => op.id(), Subscribe(op) => op.id(), @@ -201,7 +201,7 @@ impl Message { pub fn target(&self) -> Option<&PeerKeyLocation> { use Message::*; match self { - JoinRing(op) => op.target(), + Connect(op) => op.target(), Put(op) => op.target(), Get(op) => op.target(), Subscribe(op) => op.target(), @@ -214,7 +214,7 @@ impl Message { pub fn terminal(&self) -> bool { use Message::*; match self { - JoinRing(op) => op.terminal(), + Connect(op) => op.terminal(), Put(op) => op.terminal(), Get(op) => op.terminal(), Subscribe(op) => op.terminal(), @@ -225,7 +225,7 @@ impl Message { pub fn track_stats(&self) -> bool { use Message::*; - !matches!(self, JoinRing(_) | Subscribe(_) | Aborted(_)) + !matches!(self, Connect(_) | Subscribe(_) | Aborted(_)) } } @@ -234,7 +234,7 @@ impl Display for Message { use Message::*; write!(f, "Message {{")?; match self { - JoinRing(msg) => msg.fmt(f)?, + Connect(msg) => msg.fmt(f)?, Put(msg) => msg.fmt(f)?, Get(msg) => msg.fmt(f)?, Subscribe(msg) => msg.fmt(f)?, diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 558b57839..6dd9b3cb5 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -44,7 +44,7 @@ use crate::{ use crate::operations::handle_op_request; pub(crate) use conn_manager::{ConnectionBridge, ConnectionError}; pub(crate) use event_log::{EventLogRegister, EventRegister}; -pub(crate) use op_state_manager::OpManager; +pub(crate) use op_state_manager::{OpManager, OpNotAvailable}; mod conn_manager; mod event_log; @@ -462,6 +462,7 @@ macro_rules! log_handling_msg { } async fn report_result( + tx: Option, op_result: Result, OpError>, op_storage: &OpManager, executor_callback: Option>, @@ -520,15 +521,30 @@ async fn report_result( } } Ok(None) => {} - Err(OpError::OpNotPresent(tx)) => { - tracing::trace!(%tx, "Late message for finished transaction"); - } Err(err) => { + // just mark the operation as completed so no redundant messages are processed for this transaction anymore + if let Some(tx) = tx { + op_storage.completed(tx); + } tracing::debug!("Finished transaction with error: {err}") } } } +macro_rules! handle_op_not_available { + ($op_result:ident) => { + if let Err(OpError::OpNotAvailable(state)) = &$op_result { + match state { + OpNotAvailable::Running => { + tokio::time::sleep(Duration::from_micros(1_000)).await; + continue; + } + OpNotAvailable::Completed => return, + } + } + }; +} + #[tracing::instrument(name = "process_network_message", skip_all)] async fn process_message( msg: Result, @@ -544,87 +560,99 @@ async fn process_message( let cli_req = client_id.zip(client_req_handler_callback); match msg { Ok(msg) => { + let tx = Some(*msg.id()); event_listener .register_events(EventLog::from_inbound_msg(&msg, &op_storage)) .await; - match msg { - Message::JoinRing(op) => { - // log_handling_msg!("join", op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; - } - Message::Put(op) => { - // log_handling_msg!("put", *op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; - } - Message::Get(op) => { - // log_handling_msg!("get", op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; - } - Message::Subscribe(op) => { - // log_handling_msg!("subscribe", op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; + loop { + match &msg { + Message::Connect(op) => { + // log_handling_msg!("join", op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + Message::Put(op) => { + // log_handling_msg!("put", *op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + Message::Get(op) => { + // log_handling_msg!("get", op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + Message::Subscribe(op) => { + // log_handling_msg!("subscribe", op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + _ => break, } - _ => {} } } Err(err) => { report_result( + None, Err(err.into()), &op_storage, executor_callback, @@ -645,11 +673,11 @@ async fn handle_cancelled_op( where CM: ConnectionBridge + Send + Sync, { - if let TransactionType::JoinRing = tx.tx_type() { + if let TransactionType::Connect = tx.tx_type() { // the attempt to join the network failed, this could be a fatal error since the node // is useless without connecting to the network, we will retry with exponential backoff match op_storage.pop(&tx) { - Some(OpEnum::Connect(op)) if op.has_backoff() => { + Ok(Some(OpEnum::Connect(op))) if op.has_backoff() => { let ConnectOp { gateway, backoff, .. } = *op; @@ -658,7 +686,7 @@ where join_ring_request(Some(backoff), peer_key, &gateway, op_storage, conn_manager) .await?; } - Some(OpEnum::Connect(_)) => { + Ok(Some(OpEnum::Connect(_))) => { return Err(OpError::MaxRetriesExceeded(tx, tx.tx_type())); } _ => {} diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index f8acf8202..d2c3f49ad 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -435,7 +435,7 @@ impl P2pConnManager { .await; match res { Err(OpError::MaxRetriesExceeded(_, _)) - if tx_type == TransactionType::JoinRing + if tx_type == TransactionType::Connect && self.public_addr.is_none() /* FIXME: this should be not a gateway instead */ => { tracing::warn!("Retrying joining the ring with an other peer"); @@ -907,7 +907,7 @@ impl ConnectionHandler for Handler { Left(msg) => { let op_id = msg.id(); if msg.track_stats() { - if let Some(mut op) = self.op_manager.pop(op_id) { + if let Ok(Some(mut op)) = self.op_manager.pop(op_id) { op.record_transfer(); let _ = self.op_manager.push(*op_id, op); } @@ -952,7 +952,7 @@ impl ConnectionHandler for Handler { } => match Sink::poll_flush(Pin::new(&mut substream), cx) { Poll::Ready(Ok(())) => { if let Some(op_id) = op_id { - if let Some(mut op) = self.op_manager.pop(&op_id) { + if let Ok(Some(mut op)) = self.op_manager.pop(&op_id) { op.record_transfer(); let _ = self.op_manager.push(op_id, op); } @@ -981,7 +981,7 @@ impl ConnectionHandler for Handler { } => match Stream::poll_next(Pin::new(&mut substream), cx) { Poll::Ready(Some(Ok(msg))) => { let op_id = msg.id(); - if let Some(mut op) = self.op_manager.pop(op_id) { + if let Ok(Some(mut op)) = self.op_manager.pop(op_id) { op.record_transfer(); let _ = self.op_manager.push(*op_id, op); } diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index b759ab921..b2bbd9b96 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -72,7 +72,7 @@ impl<'a> EventLog<'a> { op_storage: &'a OpManager, ) -> Either> { let kind = match msg { - Message::JoinRing(connect::ConnectMsg::Response { + Message::Connect(connect::ConnectMsg::Response { msg: connect::ConnectResponse::AcceptedBy { peers, @@ -108,7 +108,7 @@ impl<'a> EventLog<'a> { op_storage: &'a OpManager, ) -> Either> { let kind = match msg { - Message::JoinRing(connect::ConnectMsg::Response { + Message::Connect(connect::ConnectMsg::Response { msg: connect::ConnectResponse::AcceptedBy { peers, diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index d24652742..af7303320 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -180,7 +180,7 @@ impl NodeInMemory { .await; match res { Err(OpError::MaxRetriesExceeded(_, _)) - if tx_type == TransactionType::JoinRing && !self.is_gateway => + if tx_type == TransactionType::Connect && !self.is_gateway => { tracing::warn!("Retrying joining the ring with an other peer"); if let Some(gateway) = self.gateways.iter().shuffle().next() { diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 528833d81..24750589c 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, time::Instant}; -use dashmap::DashMap; +use dashmap::{DashMap, DashSet}; use either::Either; use parking_lot::RwLock; use tokio::sync::{mpsc::error::SendError, Mutex}; @@ -21,6 +21,14 @@ use crate::{ use super::{conn_manager::EventLoopNotificationsSender, PeerKey}; +#[derive(Debug, thiserror::Error)] +pub(crate) enum OpNotAvailable { + #[error("operation running")] + Running, + #[error("operation completed")] + Completed, +} + /// Thread safe and friendly data structure to maintain state of the different operations /// and enable their execution. pub(crate) struct OpManager { @@ -34,6 +42,9 @@ pub(crate) struct OpManager { ch_outbound: Mutex>, // FIXME: think of an optimal strategy to check for timeouts and clean up garbage _ops_ttl: RwLock>>, + // todo: improve this when the anti-write amplification functionality is added + completed: DashSet, + in_progress: DashSet, pub ring: Ring, } @@ -61,6 +72,8 @@ impl OpManager { ring, to_event_listener: notification_channel, ch_outbound: Mutex::new(contract_handler), + completed: DashSet::new(), + in_progress: DashSet::new(), _ops_ttl: RwLock::new(BTreeMap::new()), } } @@ -84,14 +97,6 @@ impl OpManager { .map_err(|err| SendError(err.0.unwrap_left())) } - // /// Send an internal message to this node event loop. - // pub async fn notify_internal_op(&self, msg: NodeEvent) -> Result<(), SendError> { - // self.to_event_listener - // .send(Either::Right(msg)) - // .await - // .map_err(|err| SendError(err.0.unwrap_right())) - // } - /// Send an event to the contract handler and await a response event from it if successful. pub async fn notify_contract_handler( &self, @@ -110,10 +115,11 @@ impl OpManager { } pub fn push(&self, id: Transaction, op: OpEnum) -> Result<(), OpError> { + self.in_progress.remove(&id); match op { OpEnum::Connect(op) => { #[cfg(debug_assertions)] - check_id_op!(id.tx_type(), TransactionType::JoinRing); + check_id_op!(id.tx_type(), TransactionType::Connect); self.join_ring.insert(id, *op); } OpEnum::Put(op) => { @@ -140,9 +146,15 @@ impl OpManager { Ok(()) } - pub fn pop(&self, id: &Transaction) -> Option { - match id.tx_type() { - TransactionType::JoinRing => self + pub fn pop(&self, id: &Transaction) -> Result, OpNotAvailable> { + if self.completed.contains(id) { + return Err(OpNotAvailable::Completed); + } + if self.in_progress.contains(id) { + return Err(OpNotAvailable::Running); + } + let op = match id.tx_type() { + TransactionType::Connect => self .join_ring .remove(id) .map(|(_k, v)| v) @@ -156,7 +168,13 @@ impl OpManager { .map(OpEnum::Subscribe), TransactionType::Update => self.update.remove(id).map(|(_k, v)| v).map(OpEnum::Update), TransactionType::Canceled => unreachable!(), - } + }; + self.in_progress.insert(*id); + Ok(op) + } + + pub fn completed(&self, id: Transaction) { + self.completed.insert(id); } pub fn prune_connection(&self, peer: PeerKey) { diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index cef8a097a..786af2f0e 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -7,7 +7,7 @@ use crate::{ client_events::{ClientId, HostResult}, contract::ContractError, message::{InnerMessage, Message, Transaction, TransactionType}, - node::{ConnectionBridge, ConnectionError, OpManager, PeerKey}, + node::{ConnectionBridge, ConnectionError, OpManager, OpNotAvailable, PeerKey}, ring::{Location, PeerKeyLocation, RingError}, DynError, }; @@ -34,7 +34,7 @@ pub(crate) struct OpInitialization { pub(crate) async fn handle_op_request( op_storage: &OpManager, conn_manager: &mut CB, - msg: Op::Message, + msg: &Op::Message, client_id: Option, ) -> Result, OpError> where @@ -44,7 +44,7 @@ where let sender; let tx = *msg.id(); let result = { - let OpInitialization { sender: s, op } = Op::load_or_init(op_storage, &msg)?; + let OpInitialization { sender: s, op } = Op::load_or_init(op_storage, msg)?; sender = s; op.process_message(conn_manager, op_storage, msg, client_id) .await @@ -201,6 +201,8 @@ pub(crate) enum OpError { OpNotPresent(Transaction), #[error("max number of retries for tx {0} of op type `{1}` reached")] MaxRetriesExceeded(Transaction, TransactionType), + #[error("op not available")] + OpNotAvailable(#[from] OpNotAvailable), // user for control flow /// This is used as an early interrumpt of an op update when an op diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index 8f5c0543f..68607460b 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -86,7 +86,7 @@ impl Operation for ConnectOp { let sender; let tx = *msg.id(); match op_storage.pop(msg.id()) { - Some(OpEnum::Connect(connect_op)) => { + Ok(Some(OpEnum::Connect(connect_op))) => { sender = msg.sender().cloned(); // was an existing operation, the other peer messaged back Ok(OpInitialization { @@ -94,8 +94,11 @@ impl Operation for ConnectOp { sender, }) } - Some(_) => Err(OpError::OpNotPresent(tx)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(tx, op); + Err(OpError::OpNotPresent(tx)) + } + Ok(None) => { // new request to join this node, initialize the machine Ok(OpInitialization { op: Self { @@ -108,6 +111,7 @@ impl Operation for ConnectOp { sender: None, }) } + Err(err) => Err(err.into()), } } @@ -119,7 +123,7 @@ impl Operation for ConnectOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, _client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -152,7 +156,7 @@ impl Operation for ConnectOp { // FIXME: don't try to forward to peers which have already been tried (add a rejected_by list) let accepted_by = if op_storage.ring.should_accept(&new_location) { tracing::debug!(tx = %id, "Accepting connection from {}", joiner,); - HashSet::from_iter([this_node_loc]) + HashSet::from_iter([*this_node_loc]) } else { tracing::debug!(tx = %id, at_peer = %this_node_loc.peer, "Rejecting connection from peer {}", joiner); HashSet::new() @@ -160,15 +164,15 @@ impl Operation for ConnectOp { let new_peer_loc = PeerKeyLocation { location: Some(new_location), - peer: joiner, + peer: *joiner, }; if let Some(mut updated_state) = forward_conn( - id, + *id, &op_storage.ring, conn_manager, new_peer_loc, new_peer_loc, - hops_to_live, + *hops_to_live, accepted_by.len(), ) .await? @@ -192,18 +196,19 @@ impl Operation for ConnectOp { ); new_state = Some(ConnectState::OCReceived); } else { + op_storage.completed(*id); new_state = None } return_msg = Some(ConnectMsg::Response { - id, - sender: this_node_loc, + id: *id, + sender: *this_node_loc, msg: ConnectResponse::AcceptedBy { peers: accepted_by, your_location: new_location, - your_peer_id: joiner, + your_peer_id: *joiner, }, target: PeerKeyLocation { - peer: joiner, + peer: *joiner, location: Some(new_location), }, }); @@ -243,12 +248,12 @@ impl Operation for ConnectOp { }; if let Some(mut updated_state) = forward_conn( - id, + *id, &op_storage.ring, conn_manager, - sender, - joiner, - hops_to_live, + *sender, + *joiner, + *hops_to_live, accepted_by.len(), ) .await? @@ -260,12 +265,8 @@ impl Operation for ConnectOp { } else { match self.state { Some(ConnectState::Initializing) => { - let (state, msg) = try_proxy_connection( - &id, - &sender, - &own_loc, - accepted_by.clone(), - ); + let (state, msg) = + try_proxy_connection(id, sender, &own_loc, accepted_by.clone()); new_state = state; return_msg = msg; } @@ -294,9 +295,9 @@ impl Operation for ConnectOp { target.peer ); return_msg = Some(ConnectMsg::Response { - id, + id: *id, target, - sender, + sender: *sender, msg: ConnectResponse::AcceptedBy { peers: accepted_by, your_location: new_location, @@ -318,9 +319,9 @@ impl Operation for ConnectOp { own_loc.peer ); return_msg = Some(ConnectMsg::Response { - id, + id: *id, target, - sender, + sender: *sender, msg: ConnectResponse::Proxy { accepted_by }, }); } @@ -330,6 +331,7 @@ impl Operation for ConnectOp { if let Some(state) = new_state { if state.is_connected() { new_state = None; + op_storage.completed(*id); } else { new_state = Some(state); } @@ -351,8 +353,8 @@ impl Operation for ConnectOp { // Set the given location let pk_loc = PeerKeyLocation { - location: Some(your_location), - peer: your_peer_id, + location: Some(*your_location), + peer: *your_peer_id, }; // fixme: remove @@ -371,10 +373,10 @@ impl Operation for ConnectOp { ); new_state = Some(ConnectState::OCReceived); return_msg = Some(ConnectMsg::Response { - id, + id: *id, msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, sender: pk_loc, - target: sender, + target: *sender, }); tracing::debug!( tx = %id, @@ -382,17 +384,17 @@ impl Operation for ConnectOp { location = %your_location, "Updating assigned location" ); - op_storage.ring.update_location(Some(your_location)); + op_storage.ring.update_location(Some(*your_location)); for other_peer in accepted_by { let _ = propagate_oc_to_accepted_peers( conn_manager, op_storage, - sender, - &other_peer, + *sender, + other_peer, ConnectMsg::Response { - id, - target: other_peer, + id: *id, + target: *other_peer, sender: pk_loc, msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, }, @@ -407,7 +409,7 @@ impl Operation for ConnectOp { "Failed to establish any connections, aborting" ); let op = ConnectOp { - id, + id: *id, state: None, gateway: self.gateway, backoff: self.backoff, @@ -415,7 +417,7 @@ impl Operation for ConnectOp { }; op_storage .notify_op_change( - Message::Aborted(id), + Message::Aborted(*id), OpEnum::Connect(op.into()), None, ) @@ -427,14 +429,14 @@ impl Operation for ConnectOp { id, sender, target, - msg: ConnectResponse::Proxy { mut accepted_by }, + msg: ConnectResponse::Proxy { accepted_by }, } => { tracing::debug!(tx = %id, "Received proxy connect at @ {}", target.peer); match self.state { Some(ConnectState::Initializing) => { // the sender of the response is the target of the request and // is only a completed tx if it accepted the connection - if accepted_by.contains(&sender) { + if accepted_by.contains(sender) { tracing::debug!( tx = %id, "Return to {}, connected at proxy {}", @@ -445,12 +447,15 @@ impl Operation for ConnectOp { } else { tracing::debug!("Failed to connect at proxy {}", sender.peer); new_state = None; + op_storage.completed(*id); } return_msg = Some(ConnectMsg::Response { - msg: ConnectResponse::Proxy { accepted_by }, - sender, - id, - target, + msg: ConnectResponse::Proxy { + accepted_by: accepted_by.clone(), + }, + sender: *sender, + id: *id, + target: *target, }); } Some(ConnectState::AwaitingProxyResponse { @@ -465,7 +470,7 @@ impl Operation for ConnectOp { let is_target_peer = new_peer_id == original_target.peer; if is_accepted { - previously_accepted.extend(accepted_by.drain()); + previously_accepted.extend(accepted_by.iter().copied()); if is_target_peer { new_state = Some(ConnectState::OCReceived); } else { @@ -487,9 +492,9 @@ impl Operation for ConnectOp { original_target.peer ); return_msg = Some(ConnectMsg::Response { - id, + id: *id, target: original_target, - sender: target, + sender: *target, msg: ConnectResponse::AcceptedBy { peers: previously_accepted, your_location: new_location, @@ -506,9 +511,9 @@ impl Operation for ConnectOp { ); return_msg = Some(ConnectMsg::Response { - id, + id: *id, target: original_target, - sender: target, + sender: *target, msg: ConnectResponse::Proxy { accepted_by: previously_accepted, }, @@ -520,6 +525,7 @@ impl Operation for ConnectOp { if let Some(state) = new_state { if state.is_connected() { new_state = None; + op_storage.completed(*id); } else { new_state = Some(state) } @@ -536,16 +542,16 @@ impl Operation for ConnectOp { tracing::debug!(tx = %id, "Acknowledge connected at gateway"); new_state = Some(ConnectState::Connected); return_msg = Some(ConnectMsg::Connected { - id, - sender: target, - target: sender, + id: *id, + sender: *target, + target: *sender, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), } if let Some(state) = new_state { if !state.is_connected() { - return Err(OpError::InvalidStateTransition(id)); + return Err(OpError::InvalidStateTransition(*id)); } else { conn_manager.add_connection(sender.peer).await?; op_storage.ring.add_connection( @@ -554,6 +560,7 @@ impl Operation for ConnectOp { ); tracing::debug!(tx = %id, "Opened connection with peer {}", by_peer.peer); new_state = None; + op_storage.completed(*id); } }; } @@ -568,7 +575,7 @@ impl Operation for ConnectOp { }; if let Some(state) = new_state { if !state.is_connected() { - return Err(OpError::InvalidStateTransition(id)); + return Err(OpError::InvalidStateTransition(*id)); } else { tracing::info!( tx = %id, @@ -582,6 +589,7 @@ impl Operation for ConnectOp { sender.peer, ); new_state = None; + op_storage.completed(*id); } }; } @@ -934,7 +942,7 @@ mod messages { use crate::message::InnerMessage; use serde::{Deserialize, Serialize}; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum ConnectMsg { Request { id: Transaction, diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 39ec3c90e..2306a4ab7 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -147,13 +147,16 @@ impl Operation for GetOp { sender = Some(peer_key_loc.peer); }; let tx = *msg.id(); - let result = match op_storage.pop(msg.id()) { - Some(OpEnum::Get(get_op)) => { + match op_storage.pop(msg.id()) { + Ok(Some(OpEnum::Get(get_op))) => { Ok(OpInitialization { op: get_op, sender }) // was an existing operation, other peer messaged back } - Some(_) => return Err(OpError::OpNotPresent(tx)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(tx, op); + Err(OpError::OpNotPresent(tx)) + } + Ok(None) => { // new request to get a value for a contract, initialize the machine Ok(OpInitialization { op: Self { @@ -166,9 +169,8 @@ impl Operation for GetOp { sender, }) } - }; - - result + Err(err) => Err(err.into()), + } } fn id(&self) -> &Transaction { @@ -179,7 +181,7 @@ impl Operation for GetOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -203,7 +205,7 @@ impl Operation for GetOp { tracing::debug!(tx = %id, "Seek contract {} @ {}", key, target.peer); new_state = self.state; stats = Some(GetStats { - contract_location: Location::from(&key), + contract_location: Location::from(key), caching_peer: None, transfer_time: None, first_response_time: None, @@ -211,12 +213,12 @@ impl Operation for GetOp { }); let own_loc = op_storage.ring.own_location(); return_msg = Some(GetMsg::SeekNode { - key, - id, - target, + key: key.clone(), + id: *id, + target: *target, skip_list: vec![own_loc.peer], sender: own_loc, - fetch_contract, + fetch_contract: *fetch_contract, htl: MAX_GET_RETRY_HOPS, }); } @@ -226,13 +228,19 @@ impl Operation for GetOp { fetch_contract, sender, target, - mut skip_list, + skip_list, htl, } => { + let htl = *htl; + let id = *id; + let key: ContractKey = key.clone(); + let fetch_contract = *fetch_contract; + let is_cached_contract = op_storage.ring.is_contract_cached(&key); if let Some(s) = stats.as_mut() { - s.caching_peer = Some(target); + s.caching_peer = Some(*target); } + let mut skip_list = skip_list.clone(); skip_list.push(target.peer); if !is_cached_contract { @@ -263,7 +271,7 @@ impl Operation for GetOp { }, sender: op_storage.ring.own_location(), updated_skip_list: skip_list, - target: sender, // return to requester + target: *sender, // return to requester }), None, stats, @@ -284,7 +292,7 @@ impl Operation for GetOp { id, key, fetch_contract, - sender, + sender: *sender, target: new_target, skip_list, htl: new_htl, @@ -346,8 +354,8 @@ impl Operation for GetOp { key, value, updated_skip_list: vec![], - sender: target, - target: sender, + sender: *target, + target: *sender, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -390,21 +398,21 @@ impl Operation for GetOp { skip_list.push(target.peer); if let Some(target) = op_storage .ring - .closest_caching(&key, skip_list.as_slice()) + .closest_caching(key, skip_list.as_slice()) .into_iter() .next() { return_msg = Some(GetMsg::SeekNode { - id, - key, + id: *id, + key: key.clone(), target, - sender: this_loc, + sender: *this_loc, fetch_contract, htl: MAX_GET_RETRY_HOPS, - skip_list: updated_skip_list, + skip_list: updated_skip_list.clone(), }); } else { - return Err(RingError::NoCachingPeers(key).into()); + return Err(RingError::NoCachingPeers(key.clone()).into()); } new_state = Some(GetState::AwaitingResponse { skip_list, @@ -417,22 +425,22 @@ impl Operation for GetOp { "Failed getting a value for contract {}, reached max retries", key ); - return Err(OpError::MaxRetriesExceeded(id, id.tx_type())); + return Err(OpError::MaxRetriesExceeded(*id, id.tx_type())); } } Some(GetState::ReceivedRequest) => { tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; return_msg = Some(GetMsg::ReturnGet { - id, - key, + id: *id, + key: key.clone(), value: StoreResponse { state: None, contract: None, }, - updated_skip_list, - sender, - target, + updated_skip_list: updated_skip_list.clone(), + sender: *sender, + target: *target, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -448,8 +456,11 @@ impl Operation for GetOp { }, sender, target, - mut updated_skip_list, + updated_skip_list, } => { + let id = *id; + let key = key.clone(); + let mut updated_skip_list = updated_skip_list.clone(); updated_skip_list.push(sender.peer); let require_contract = matches!( self.state, @@ -505,8 +516,8 @@ impl Operation for GetOp { state: None, contract: None, }, - sender, - target, + sender: *sender, + target: *target, updated_skip_list, }), OpEnum::Get(op), @@ -551,7 +562,7 @@ impl Operation for GetOp { return_msg = None; result = Some(GetResult { state: value.clone(), - contract, + contract: contract.clone(), }); } else { tracing::debug!(tx = %id, "Get response received for contract {}", key); @@ -559,7 +570,7 @@ impl Operation for GetOp { return_msg = None; result = Some(GetResult { state: value.clone(), - contract, + contract: contract.clone(), }); } } @@ -573,8 +584,8 @@ impl Operation for GetOp { state: None, contract: None, }, - sender, - target, + sender: *sender, + target: *target, updated_skip_list, }); } @@ -773,7 +784,7 @@ mod messages { use super::*; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum GetMsg { RequestGet { id: Transaction, diff --git a/crates/core/src/operations/op_trait.rs b/crates/core/src/operations/op_trait.rs index 646fec42b..3c906106f 100644 --- a/crates/core/src/operations/op_trait.rs +++ b/crates/core/src/operations/op_trait.rs @@ -31,7 +31,7 @@ where self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>>; } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index d3e216453..314dac994 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -136,13 +136,16 @@ impl Operation for PutOp { }; let tx = *msg.id(); - let result = match op_storage.pop(msg.id()) { - Some(OpEnum::Put(put_op)) => { + match op_storage.pop(msg.id()) { + Ok(Some(OpEnum::Put(put_op))) => { // was an existing operation, the other peer messaged back Ok(OpInitialization { op: put_op, sender }) } - Some(_) => return Err(OpError::OpNotPresent(tx)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(tx, op); + Err(OpError::OpNotPresent(tx)) + } + Ok(None) => { // new request to put a new value for a contract, initialize the machine Ok(OpInitialization { op: Self { @@ -154,8 +157,8 @@ impl Operation for PutOp { sender, }) } - }; - result + Err(err) => Err(err.into()), + } } fn id(&self) -> &Transaction { @@ -166,7 +169,7 @@ impl Operation for PutOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -187,20 +190,20 @@ impl Operation for PutOp { let key = contract.key(); tracing::debug!( - "Performing a RequestPut for contract {} from {} to {}", + "Rquesting put for contract {} from {} to {}", key, sender.peer, target.peer ); return_msg = Some(PutMsg::SeekNode { - id, + id: *id, sender, - target, - value, - contract, - related_contracts, - htl, + target: *target, + value: value.clone(), + contract: contract.clone(), + related_contracts: related_contracts.clone(), + htl: *htl, skip_list: vec![sender.peer], }); @@ -215,15 +218,16 @@ impl Operation for PutOp { related_contracts, htl, target, - mut skip_list, + skip_list, } => { let key = contract.key(); let is_cached_contract = op_storage.ring.is_contract_cached(&key); tracing::debug!( - "Performing a SeekNode at {}, trying put the contract {}", + tx = %id, + "Puttting contract {} at target peer {}", + key, target.peer, - key ); if !is_cached_contract @@ -231,45 +235,48 @@ impl Operation for PutOp { .ring .within_caching_distance(&Location::from(&key)) { - tracing::debug!("Contract `{}` not cached @ peer {}", key, target.peer); - match try_to_cache_contract(op_storage, &contract, &key, client_id).await { + tracing::debug!(tx = %id, "Contract `{}` not cached @ peer {}", key, target.peer); + match try_to_cache_contract(op_storage, contract, &key, client_id).await { Ok(_) => {} Err(err) => return Err(err), } } else if !is_cached_contract { + // FIXME // in this case forward to a closer node to the target location and just wait for a response // to give back to requesting peer - // FIXME tracing::warn!( + tx = %id, "Contract {} not found while processing info, forwarding", key ); } // after the contract has been cached, push the update query - tracing::debug!("Attempting contract value update"); + tracing::debug!(tx = %id, "Attempting contract value update"); let parameters = contract.params(); let new_value = put_contract( op_storage, key.clone(), - value, - related_contracts, + value.clone(), + related_contracts.clone(), parameters, client_id, ) .await?; - tracing::debug!("Contract successfully updated"); + tracing::debug!(tx = %id, "Contract successfully updated"); // if the change was successful, communicate this back to the requestor and broadcast the change conn_manager .send( &sender.peer, (PutMsg::SuccessfulUpdate { - id, + id: *id, new_value: new_value.clone(), }) .into(), ) .await?; + + let mut skip_list = skip_list.clone(); skip_list.push(target.peer); if let Some(new_htl) = htl.checked_sub(1) { @@ -277,9 +284,9 @@ impl Operation for PutOp { forward_changes( op_storage, conn_manager, - &contract, + contract, new_value.clone(), - id, + *id, new_htl, skip_list.as_slice(), ) @@ -292,13 +299,14 @@ impl Operation for PutOp { .map(|i| i.value().to_vec()) .unwrap_or_default(); tracing::debug!( + tx = %id, "Successfully updated a value for contract {} @ {:?}", key, target.location ); match try_to_broadcast( - (id, client_id), + (*id, client_id), op_storage, self.state, broadcast_to, @@ -329,7 +337,7 @@ impl Operation for PutOp { let new_value = put_contract( op_storage, key.clone(), - new_value, + new_value.clone(), RelatedContracts::default(), parameters.clone(), client_id, @@ -339,12 +347,12 @@ impl Operation for PutOp { let broadcast_to = op_storage .ring - .subscribers_of(&key) + .subscribers_of(key) .map(|i| { // Avoid already broadcast nodes and sender from broadcasting let mut subscribers: Vec = i.value().to_vec(); let mut avoid_list: HashSet = - sender_subscribers.into_iter().map(|pl| pl.peer).collect(); + sender_subscribers.iter().map(|pl| pl.peer).collect(); avoid_list.insert(sender.peer); subscribers.retain(|s| !avoid_list.contains(&s.peer)); subscribers @@ -357,12 +365,12 @@ impl Operation for PutOp { ); match try_to_broadcast( - (id, client_id), + (*id, client_id), op_storage, self.state, broadcast_to, - key, - (parameters, new_value), + key.clone(), + (parameters.clone(), new_value), self._ttl, ) .await @@ -376,25 +384,30 @@ impl Operation for PutOp { } PutMsg::Broadcasting { id, - mut broadcast_to, - mut broadcasted_to, + broadcast_to, + broadcasted_to, key, new_value, parameters, } => { let sender = op_storage.ring.own_location(); - let msg = PutMsg::BroadcastTo { - id, - key: key.clone(), - new_value: new_value.clone(), - sender, - sender_subscribers: broadcast_to.clone(), - parameters, - }; + let mut broadcasted_to = *broadcasted_to; let mut broadcasting = Vec::with_capacity(broadcast_to.len()); - for peer in &broadcast_to { - let f = conn_manager.send(&peer.peer, msg.clone().into()); + let mut filtered_broadcast = broadcast_to + .iter() + .filter(|pk| pk.peer != sender.peer) + .collect::>(); + for peer in filtered_broadcast.iter() { + let msg = PutMsg::BroadcastTo { + id: *id, + key: key.clone(), + new_value: new_value.clone(), + sender, + sender_subscribers: broadcast_to.clone(), + parameters: parameters.clone(), + }; + let f = conn_manager.send(&peer.peer, msg.into()); broadcasting.push(f); } let error_futures = futures::future::join_all(broadcasting) @@ -413,7 +426,7 @@ impl Operation for PutOp { let mut incorrect_results = 0; for (peer_num, err) in error_futures { // remove the failed peers in reverse order - let peer = broadcast_to.remove(peer_num); + let peer = filtered_broadcast.remove(peer_num); tracing::warn!( "failed broadcasting put change to {} with error {}; dropping connection", peer.peer, @@ -429,13 +442,15 @@ impl Operation for PutOp { ); // Subscriber nodes have been notified of the change, the operation is completed + op_storage.completed(*id); return_msg = None; new_state = None; } - PutMsg::SuccessfulUpdate { .. } => { + PutMsg::SuccessfulUpdate { id, .. } => { match self.state { Some(PutState::AwaitingResponse { contract, .. }) => { tracing::debug!("Successfully updated value for {}", contract,); + op_storage.completed(*id); new_state = None; return_msg = None; } @@ -451,7 +466,7 @@ impl Operation for PutOp { contract, new_value, htl, - mut skip_list, + skip_list, } => { let key = contract.key(); let peer_loc = op_storage.ring.own_location(); @@ -467,7 +482,7 @@ impl Operation for PutOp { .ring .within_caching_distance(&Location::from(&key)); if !cached_contract && within_caching_dist { - match try_to_cache_contract(op_storage, &contract, &key, client_id).await { + match try_to_cache_contract(op_storage, contract, &key, client_id).await { Ok(_) => {} Err(err) => return Err(err), } @@ -482,14 +497,15 @@ impl Operation for PutOp { let new_value = put_contract( op_storage, key, - new_value, + new_value.clone(), RelatedContracts::default(), contract.params(), client_id, ) .await?; - //update skip list + // update skip list + let mut skip_list = skip_list.clone(); skip_list.push(peer_loc.peer); // if successful, forward to the next closest peers (if any) @@ -497,14 +513,15 @@ impl Operation for PutOp { forward_changes( op_storage, conn_manager, - &contract, + contract, new_value, - id, + *id, new_htl, skip_list.as_slice(), ) .await; } + op_storage.completed(*id); return_msg = None; new_state = None; } @@ -815,7 +832,7 @@ mod messages { use crate::message::InnerMessage; use serde::{Deserialize, Serialize}; - #[derive(Debug, Serialize, Deserialize, Clone)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum PutMsg { /// Initialize the put operation by routing the value RouteValue { @@ -1031,7 +1048,7 @@ mod test { // trigger the put op @ gw-0 sim_nw - .trigger_event("gateway-0", 1, Some(Duration::from_millis(100))) + .trigger_event("gateway-0", 1, Some(Duration::from_millis(150))) .await?; assert!(sim_nw.has_put_contract("gateway-0", &key, &new_value)); assert!(sim_nw.event_listener.contract_broadcasted(&key)); diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 040c454c1..38d6b08e5 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -66,16 +66,19 @@ impl Operation for SubscribeOp { }; let id = *msg.id(); - let result = match op_storage.pop(msg.id()) { - Some(OpEnum::Subscribe(subscribe_op)) => { + match op_storage.pop(msg.id()) { + Ok(Some(OpEnum::Subscribe(subscribe_op))) => { // was an existing operation, the other peer messaged back Ok(OpInitialization { op: subscribe_op, sender, }) } - Some(_) => return Err(OpError::OpNotPresent(id)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(id, op); + Err(OpError::OpNotPresent(id)) + } + Ok(None) => { // new request to subcribe to a contract, initialize the machine Ok(OpInitialization { op: Self { @@ -86,8 +89,8 @@ impl Operation for SubscribeOp { sender, }) } - }; - result + Err(err) => Err(err.into()), + } } fn id(&self) -> &Transaction { @@ -98,7 +101,7 @@ impl Operation for SubscribeOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -115,9 +118,9 @@ impl Operation for SubscribeOp { let sender = op_storage.ring.own_location(); new_state = self.state; return_msg = Some(SubscribeMsg::SeekNode { - id, - key, - target, + id: *id, + key: key.clone(), + target: *target, subscriber: sender, skip_list: vec![sender.peer], htl: 0, @@ -136,23 +139,22 @@ impl Operation for SubscribeOp { OperationResult { return_msg: Some(Message::from(SubscribeMsg::ReturnSub { key: key.clone(), - id, + id: *id, subscribed: false, sender, - target: subscriber, + target: *subscriber, })), state: None, } }; - if !op_storage.ring.is_contract_cached(&key) { + if !op_storage.ring.is_contract_cached(key) { tracing::debug!(tx = %id, "Contract {} not found at {}, trying other peer", key, target.peer); - let Some(new_target) = - op_storage.ring.closest_caching(&key, &[sender.peer]) + let Some(new_target) = op_storage.ring.closest_caching(key, &[sender.peer]) else { tracing::warn!(tx = %id, "No peer found while trying getting contract {key}"); - return Err(OpError::RingError(RingError::NoCachingPeers(key))); + return Err(OpError::RingError(RingError::NoCachingPeers(key.clone()))); }; let new_htl = htl + 1; @@ -169,9 +171,9 @@ impl Operation for SubscribeOp { .send( &new_target.peer, (SubscribeMsg::SeekNode { - id, + id: *id, key: key.clone(), - subscriber, + subscriber: *subscriber, target: new_target, skip_list: new_skip_list.clone(), htl: new_htl, @@ -179,7 +181,7 @@ impl Operation for SubscribeOp { .into(), ) .await?; - } else if op_storage.ring.add_subscriber(&key, subscriber).is_err() { + } else if op_storage.ring.add_subscriber(key, *subscriber).is_err() { // max number of subscribers for this contract reached return Ok(return_err()); } @@ -191,15 +193,16 @@ impl Operation for SubscribeOp { "Peer {} successfully subscribed to contract {key}", subscriber.peer, ); - new_state = Some(SubscribeState::Completed); + new_state = None; // TODO review behaviour, if the contract is not cached should return subscribed false? return_msg = Some(SubscribeMsg::ReturnSub { - sender: target, - target: subscriber, - id, - key, + sender: *target, + target: *subscriber, + id: *id, + key: key.clone(), subscribed: true, }); + op_storage.completed(*id); } _ => return Err(OpError::InvalidStateTransition(self.id)), } @@ -227,28 +230,28 @@ impl Operation for SubscribeOp { skip_list.push(sender.peer); if let Some(target) = op_storage .ring - .closest_caching(&key, skip_list.as_slice()) + .closest_caching(key, skip_list.as_slice()) .into_iter() .next() { let subscriber = op_storage.ring.own_location(); return_msg = Some(SubscribeMsg::SeekNode { - id, - key, + id: *id, + key: key.clone(), subscriber, target, skip_list: vec![target.peer], htl: 0, }); } else { - return Err(RingError::NoCachingPeers(key).into()); + return Err(RingError::NoCachingPeers(key.clone()).into()); } new_state = Some(SubscribeState::AwaitingResponse { skip_list, retries: retries + 1, }); } else { - return Err(OpError::MaxRetriesExceeded(id, id.tx_type())); + return Err(OpError::MaxRetriesExceeded(*id, id.tx_type())); } } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -270,20 +273,15 @@ impl Operation for SubscribeOp { this = ?op_storage.ring.own_location().peer, "Subscribed to `{key}` at provider {}", sender.peer ); - op_storage.ring.add_subscription(key); - // todo: should inform back to the network event loop? + op_storage.ring.add_subscription(key.clone()); + // todo: should inform back to the network event loop in case a client + // is waiting for response let _ = client_id; - new_state = None; + new_state = Some(SubscribeState::Completed); return_msg = None; + op_storage.completed(*id); } - state => { - tracing::error!( - tx = %id, - ?state, - target = ?target.peer, - this = ?op_storage.ring.own_location().peer, - "wrong state" - ); + _other => { return Err(OpError::InvalidStateTransition(self.id)); } } @@ -393,7 +391,7 @@ mod messages { use super::*; - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum SubscribeMsg { FetchRouting { id: Transaction, @@ -478,7 +476,6 @@ mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn successful_subscribe_op_between_nodes() -> Result<(), anyhow::Error> { - crate::config::set_logger(); const NUM_NODES: usize = 4usize; const NUM_GW: usize = 1usize; diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index e85e495bb..bfdae46a6 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -48,7 +48,7 @@ impl Operation for UpdateOp { self, _conn_manager: &'a mut CB, _op_storage: &'a crate::node::OpManager, - _input: Self::Message, + _input: &Self::Message, _client_id: Option, ) -> std::pin::Pin< Box> + Send + 'a>, @@ -67,7 +67,7 @@ mod messages { ring::PeerKeyLocation, }; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum UpdateMsg {} impl InnerMessage for UpdateMsg { From 368d0030699624e964467722a6a761132ceadf5e Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Tue, 24 Oct 2023 11:46:10 +0200 Subject: [PATCH 14/14] Fix CI issues --- .github/workflows/ci.yml | 4 +- crates/core/Cargo.toml | 6 +-- crates/core/src/config.rs | 64 ++++++++++++++------------- crates/core/src/operations/connect.rs | 1 - crates/core/src/operations/get.rs | 2 +- crates/core/src/operations/put.rs | 2 +- stdlib | 2 +- 7 files changed, 42 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae257aaf9..4c9589d5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,14 +10,14 @@ on: jobs: test_all: - name: Test features + name: Test runs-on: ubuntu-latest strategy: max-parallel: 1 matrix: - args: ["--no-default-features", "--features default"] + args: ["--no-default-features --features trace,websocket,sqlite"] env: FREENET_LOG: error CARGO_TARGET_DIR: ${{ github.workspace }}/target diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 4f4126651..52474ae34 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -74,7 +74,7 @@ tar = { version = "0.4.38" } xz2 = { version = "0.1" } # Tracing deps -tracing = { version = "0.1", optional = true } +tracing = { version = "0.1" } opentelemetry = { version = "0.20.0", default-features = false, features = ["rt-tokio", "trace"], optional = true } opentelemetry-jaeger = { version = "0.19.0", features = ["rt-tokio","collector_client", "isahc"], optional = true } tracing-opentelemetry = { version = "0.21.0", optional = true } @@ -91,11 +91,11 @@ pico-args = "0.5" freenet-stdlib = { workspace = true, features = ["testing", "net"] } [features] -default = ["websocket", "rocks_db", "trace"] +default = ["trace", "websocket", "sqlite"] testing = ["arbitrary"] rocks_db = ["rocksdb"] sqlite = ["sqlx"] websocket = ["axum/ws"] -trace = ["tracing", "opentelemetry", "opentelemetry-jaeger", "tracing-opentelemetry", "tracing-subscriber"] +trace = ["opentelemetry", "opentelemetry-jaeger", "tracing-opentelemetry", "tracing-subscriber"] local-mode = [] network-mode = [] diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 07360c1a1..b503edef2 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -52,6 +52,7 @@ pub(crate) struct WebSocketApiConfig { port: u16, } +#[cfg(feature = "websocket")] impl From for SocketAddr { fn from(val: WebSocketApiConfig) -> Self { (val.ip, val.port).into() @@ -320,44 +321,47 @@ impl libp2p::swarm::Executor for GlobalExecutor { } pub fn set_logger() { - static LOGGER_SET: AtomicBool = AtomicBool::new(false); - if LOGGER_SET - .compare_exchange( - false, - true, - std::sync::atomic::Ordering::Acquire, - std::sync::atomic::Ordering::SeqCst, - ) - .is_err() + #[cfg(feature = "trace")] { - return; - } + static LOGGER_SET: AtomicBool = AtomicBool::new(false); + if LOGGER_SET + .compare_exchange( + false, + true, + std::sync::atomic::Ordering::Acquire, + std::sync::atomic::Ordering::SeqCst, + ) + .is_err() + { + return; + } - let filter = if cfg!(any(test, debug_assertions)) { - tracing_subscriber::filter::LevelFilter::DEBUG.into() - } else { - tracing_subscriber::filter::LevelFilter::INFO.into() - }; - - let sub = tracing_subscriber::fmt().with_level(true).with_env_filter( - tracing_subscriber::EnvFilter::builder() - .with_default_directive(filter) - .from_env_lossy() - .add_directive("stretto=off".parse().unwrap()) - .add_directive("sqlx=error".parse().unwrap()), - ); - - if cfg!(any(test, debug_assertions)) { - sub.with_file(true).with_line_number(true).init(); - } else { - sub.init(); + let filter = if cfg!(any(test, debug_assertions)) { + tracing_subscriber::filter::LevelFilter::DEBUG.into() + } else { + tracing_subscriber::filter::LevelFilter::INFO.into() + }; + + let sub = tracing_subscriber::fmt().with_level(true).with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(filter) + .from_env_lossy() + .add_directive("stretto=off".parse().unwrap()) + .add_directive("sqlx=error".parse().unwrap()), + ); + + if cfg!(any(test, debug_assertions)) { + sub.with_file(true).with_line_number(true).init(); + } else { + sub.init(); + } } } +#[cfg(feature = "trace")] pub(super) mod tracer { use super::*; - #[cfg(feature = "trace")] pub fn init_tracer() -> Result<(), opentelemetry::trace::TraceError> { use opentelemetry::{global, sdk::propagation::TraceContextPropagator}; use tracing_subscriber::layer::SubscriberExt; diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index 68607460b..5eb8eb7f2 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -1104,7 +1104,6 @@ mod test { /// Once a gateway is left without remaining open slots, ensure forwarding connects #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn forward_connection_to_node() -> Result<(), anyhow::Error> { - // crate::config::set_logger(); const NUM_NODES: usize = 3usize; const NUM_GW: usize = 1usize; let mut sim_nw = SimNetwork::new( diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 2306a4ab7..97dff1e17 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -1020,7 +1020,7 @@ mod test { sim_nw.check_connectivity(Duration::from_secs(3)).await?; sim_nw - .trigger_event("node-0", 1, Some(Duration::from_millis(50))) + .trigger_event("node-0", 1, Some(Duration::from_millis(200))) .await?; assert!(sim_nw.has_got_contract("node-0", &key)); Ok(()) diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 314dac994..c4480836d 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -1048,7 +1048,7 @@ mod test { // trigger the put op @ gw-0 sim_nw - .trigger_event("gateway-0", 1, Some(Duration::from_millis(150))) + .trigger_event("gateway-0", 1, Some(Duration::from_millis(200))) .await?; assert!(sim_nw.has_put_contract("gateway-0", &key, &new_value)); assert!(sim_nw.event_listener.contract_broadcasted(&key)); diff --git a/stdlib b/stdlib index e6b4b32d0..f36475537 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit e6b4b32d0db22a803a1b0f46ea9aec48829443b7 +Subproject commit f36475537a1fa2e1f19721bbfac452ce508f02b9