diff --git a/crates/core/src/bin/freenet.rs b/crates/core/src/bin/freenet.rs index b74c27614..0b7416bbb 100644 --- a/crates/core/src/bin/freenet.rs +++ b/crates/core/src/bin/freenet.rs @@ -15,7 +15,7 @@ async fn run_local(config: PeerCliConfig) -> Result<(), DynError> { let port = config.port; let ip = config.address; freenet::config::Config::set_op_mode(OperationMode::Local); - let executor = Executor::from_config(config).await?; + let executor = Executor::from_config(config, None).await?; let socket: SocketAddr = (ip, port).into(); freenet::server::local_node::run_local_node(executor, socket).await } diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index 92ec15d09..66313f275 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -1,3 +1,5 @@ +//! Clients events related logic and type definitions. For example, receival of client events from applications throught the HTTP gateway. + use freenet_stdlib::client_api::ClientRequest; use freenet_stdlib::client_api::{ClientError, HostResponse}; use futures::future::BoxFuture; diff --git a/crates/core/src/client_events/websocket.rs b/crates/core/src/client_events/websocket.rs index ce393e4cf..117dac7a8 100644 --- a/crates/core/src/client_events/websocket.rs +++ b/crates/core/src/client_events/websocket.rs @@ -380,7 +380,7 @@ async fn process_client_request( Err(err) => { let result_error = bincode::serialize(&Err::( ErrorKind::DeserializationError { - cause: format!("{err}"), + cause: format!("{err}").into(), } .into(), )) diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index b0a606ba0..5255d1d4c 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -1,21 +1,23 @@ -use crate::runtime::ContractError as ContractRtError; +//! Handling of contracts and delegates, including storage, execution, caching, etc. +//! +//! Internally uses the wasm_runtime module to execute contract and/or delegate instructions. + use either::Either; use freenet_stdlib::prelude::*; mod executor; mod handler; -mod in_memory; pub mod storages; pub(crate) use executor::{ - executor_channel, ExecutorToEventLoopChannel, NetworkEventListenerHalve, + executor_channel, mock_runtime::MockRuntime, Callback, ExecutorToEventLoopChannel, + NetworkEventListenerHalve, }; pub(crate) use handler::{ - contract_handler_channel, ClientResponses, ClientResponsesSender, ContractHandler, - ContractHandlerChannel, ContractHandlerEvent, EventId, NetworkContractHandler, SenderHalve, - StoreResponse, + client_responses_channel, contract_handler_channel, in_memory::MemoryContractHandler, + ClientResponsesReceiver, ClientResponsesSender, ContractHandler, ContractHandlerChannel, + ContractHandlerEvent, NetworkContractHandler, SenderHalve, StoreResponse, WaitingResolution, }; -pub(crate) use in_memory::{MemoryContractHandler, MockRuntime}; pub use executor::{Executor, ExecutorError, OperationMode}; @@ -68,7 +70,7 @@ where id, ContractHandlerEvent::GetResponse { key, - response: Err(err.into()), + response: Err(err), }, ) .await @@ -79,43 +81,11 @@ where } } } - ContractHandlerEvent::Cache(contract) => { - let key = contract.key(); - match contract_handler - .executor() - .store_contract(contract) - .instrument(tracing::info_span!("store_contract", %key)) - .await - { - Ok(_) => { - contract_handler - .channel() - .send_to_sender(id, ContractHandlerEvent::CacheResult(Ok(()))) - .await - .map_err(|error| { - tracing::debug!(%error, "shutting down contract handler"); - error - })?; - } - Err(err) => { - tracing::error!("Error while caching: {err}"); - let err = ContractError::ContractRuntimeError(err); - contract_handler - .channel() - .send_to_sender(id, ContractHandlerEvent::CacheResult(Err(err))) - .await - .map_err(|error| { - tracing::debug!(%error, "shutting down contract handler"); - error - })?; - } - } - } ContractHandlerEvent::PutQuery { key, state, related_contracts, - parameters, + contract, } => { let put_result = contract_handler .executor() @@ -123,17 +93,16 @@ where key.clone(), Either::Left(state), related_contracts, - parameters, + contract, ) .instrument(tracing::info_span!("upsert_contract_state", %key)) - .await - .map_err(Into::into); + .await; contract_handler .channel() .send_to_sender( id, ContractHandlerEvent::PutResponse { - new_value: put_result, + new_value: put_result.map_err(Into::into), }, ) .await @@ -142,6 +111,24 @@ where error })?; } + ContractHandlerEvent::Subscribe { key } => { + let response = contract_handler + .executor() + .subscribe_to_contract(key.clone()) + .await; + contract_handler + .channel() + .send_to_sender( + id, + ContractHandlerEvent::SubscribeResponse { key, response }, + ) + .await + .map_err(|error| { + tracing::debug!(%error, "shutting down contract handler"); + error + })?; + todo!() + } _ => unreachable!(), } } @@ -153,8 +140,6 @@ pub(crate) enum ContractError { ChannelDropped(Box), #[error("contract {0} not found in storage")] ContractNotFound(ContractKey), - #[error(transparent)] - ContractRuntimeError(ContractRtError), #[error("{0}")] IOError(#[from] std::io::Error), #[error("no response received from handler")] diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index 00ee03ad0..6f28a856f 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -8,8 +8,8 @@ use std::time::{Duration, Instant}; use blake3::traits::digest::generic_array::GenericArray; use either::Either; use freenet_stdlib::client_api::{ - ClientError, ClientRequest, ContractError as CoreContractError, ContractRequest, - ContractResponse, DelegateError as CoreDelegateError, DelegateRequest, + ClientError as WsClientError, ClientRequest, ContractError as StdContractError, + ContractRequest, ContractResponse, DelegateError as StdDelegateError, DelegateRequest, HostResponse::{self, DelegateResponse}, RequestError, }; @@ -25,7 +25,8 @@ use crate::node::OpManager; ))] use crate::operations::get::GetResult; use crate::operations::{OpEnum, OpError}; -use crate::runtime::{ +use crate::ring::PeerKeyLocation; +use crate::wasm_runtime::{ ContractRuntimeInterface, ContractStore, DelegateRuntimeInterface, DelegateStore, Runtime, SecretsStore, StateStore, StateStoreError, }; @@ -38,15 +39,90 @@ use crate::{ use super::storages::Storage; +pub(super) mod mock_runtime; +pub(super) mod runtime; + +#[derive(Debug)] pub struct ExecutorError(Either, DynError>); +enum InnerOpError { + Upsert(ContractKey), + Delegate(DelegateKey), +} + +impl std::error::Error for ExecutorError {} + impl ExecutorError { - pub fn other(value: impl Into) -> Self { - Self(Either::Right(value.into())) + pub fn other(error: impl Into) -> Self { + Self(Either::Right(error.into())) + } + + /// Call this when an unreachable path is reached but need to avoid panics. + fn internal_error() -> Self { + ExecutorError(Either::Right("internal error".into())) } - fn request(value: impl Into) -> Self { - Self(Either::Left(Box::new(value.into()))) + fn request(error: impl Into) -> Self { + Self(Either::Left(Box::new(error.into()))) + } + + fn execution( + outer_error: crate::wasm_runtime::ContractError, + op: Option, + ) -> Self { + use crate::wasm_runtime::RuntimeInnerError; + let error = outer_error.deref(); + + if let RuntimeInnerError::ContractExecError(e) = error { + if let Some(InnerOpError::Upsert(key)) = &op { + return ExecutorError::request(StdContractError::update_exec_error(key.clone(), e)); + } + } + + if let RuntimeInnerError::DelegateNotFound(key) = error { + return ExecutorError::request(StdDelegateError::Missing(key.clone())); + } + + if let RuntimeInnerError::DelegateExecError(e) = error { + return ExecutorError::request(StdDelegateError::ExecutionError(format!("{e}").into())); + } + + if let ( + RuntimeInnerError::SecretStoreError( + crate::wasm_runtime::SecretStoreError::MissingSecret(secret), + ), + Some(InnerOpError::Delegate(key)), + ) = (error, &op) + { + return ExecutorError::request(StdDelegateError::MissingSecret { + key: key.clone(), + secret: secret.clone(), + }); + } + + match error { + RuntimeInnerError::WasmCompileError(e) => match op { + Some(InnerOpError::Upsert(key)) => { + return ExecutorError::request(StdContractError::update_exec_error(key, e)) + } + _ => return ExecutorError::other(format!("execution error: {e}")), + }, + RuntimeInnerError::WasmExportError(e) => match op { + Some(InnerOpError::Upsert(key)) => { + return ExecutorError::request(StdContractError::update_exec_error(key, e)) + } + _ => return ExecutorError::other(format!("execution error: {e}")), + }, + RuntimeInnerError::WasmInstantiationError(e) => match op { + Some(InnerOpError::Upsert(key)) => { + return ExecutorError::request(StdContractError::update_exec_error(key, e)) + } + _ => return ExecutorError::other(format!("execution error: {e}")), + }, + _ => {} + } + + ExecutorError::other(outer_error) } pub fn is_request(&self) -> bool { @@ -82,24 +158,6 @@ impl From> for ExecutorError { } } -impl From for DynError { - fn from(value: ExecutorError) -> Self { - match value.0 { - Either::Left(l) => l as DynError, - Either::Right(r) => r, - } - } -} - -impl From for anyhow::Error { - fn from(value: ExecutorError) -> Self { - match value.0 { - Either::Left(l) => anyhow::Error::new(*l), - Either::Right(r) => anyhow::Error::msg(r), - } - } -} - type Response = Result; #[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] @@ -110,7 +168,7 @@ pub enum OperationMode { Network, } -pub(crate) struct ExecutorToEventLoopChannel { +pub struct ExecutorToEventLoopChannel { op_manager: Arc, end: End, } @@ -121,73 +179,142 @@ pub(crate) fn executor_channel( ExecutorToEventLoopChannel, ExecutorToEventLoopChannel, ) { - let (sender, _b) = mpsc::channel(1); + // todo: use sensible values for channel buf sizes based on number concurrent tasks running + // when we are able to suspend execution of a request while waiting for a callback + let (waiting_for_op_tx, waiting_for_op_rx) = mpsc::channel(1); + let (response_for_tx, response_for_rx) = mpsc::channel(1); + let listener_halve = ExecutorToEventLoopChannel { op_manager: op_manager.clone(), - end: NetworkEventListenerHalve, + end: NetworkEventListenerHalve { + waiting_for_op_rx, + response_for_tx, + }, }; let sender_halve = ExecutorToEventLoopChannel { op_manager: op_manager.clone(), - end: ExecutorHalve { sender }, + end: ExecutorHalve { + waiting_for_op_tx, + response_for_rx, + completed: HashMap::default(), + }, }; (listener_halve, sender_halve) } +#[derive(thiserror::Error, Debug)] +enum CallbackError { + #[error(transparent)] + Err(#[from] ExecutorError), + #[error(transparent)] + Conversion(#[from] OpError), + #[error("missing result")] + MissingResult, +} + impl ExecutorToEventLoopChannel { - async fn send_to_event_loop(&mut self, message: T) -> Result<(), DynError> + async fn send_to_event_loop(&mut self, message: T) -> Result where T: ComposeNetworkMessage, Op: Operation + Send + 'static, { let op = message.initiate_op(&self.op_manager); - self.end.sender.send(*op.id()).await?; + let tx = *op.id(); + self.end.waiting_for_op_tx.send(tx).await?; >::resume_op(op, &self.op_manager).await?; - Ok(()) + Ok(tx) + } + + async fn receive_op_result(&mut self, transaction: Transaction) -> Result + where + Op: Operation + TryFrom, + { + if let Some(result) = self.end.completed.remove(&transaction) { + return result.try_into().map_err(CallbackError::Conversion); + } + let op_result = self + .end + .response_for_rx + .recv() + .await + .ok_or_else(|| ExecutorError::other("channel closed"))?; + if op_result.id() != &transaction { + self.end.completed.insert(*op_result.id(), op_result); + return Err(CallbackError::MissingResult); + } + op_result.try_into().map_err(CallbackError::Conversion) } } impl ExecutorToEventLoopChannel { - pub async fn transaction_from_executor(&mut self) -> Transaction { - todo!() + pub async fn transaction_from_executor(&mut self) -> Result { + let tx = self + .end + .waiting_for_op_rx + .recv() + .await + .ok_or("channel closed")?; + Ok(tx) } - pub async fn response(&mut self, _result: OpEnum) { - todo!() + pub(crate) fn callback(&self) -> ExecutorToEventLoopChannel { + ExecutorToEventLoopChannel { + op_manager: self.op_manager.clone(), + end: Callback { + response_for_tx: self.end.response_for_tx.clone(), + }, + } } } -impl Clone for ExecutorToEventLoopChannel { - fn clone(&self) -> Self { - todo!() +impl ExecutorToEventLoopChannel { + pub async fn response(&mut self, result: OpEnum) { + if self.end.response_for_tx.send(result).await.is_err() { + tracing::debug!("failed to send response to executor, channel closed"); + } } } -pub(crate) struct NetworkEventListenerHalve; -pub(crate) struct ExecutorHalve { - sender: mpsc::Sender, +pub(crate) struct Callback { + /// sends the callback response to the executor + response_for_tx: mpsc::Sender, +} + +pub(crate) struct NetworkEventListenerHalve { + /// this is the receiver end of the Executor halve, which will be sent from the executor + /// when a callback is expected for a given transaction + waiting_for_op_rx: mpsc::Receiver, + /// this is the sender end of the Executor halve receiver, which will communicate + /// back responses to the executor, it's cloned each tiome a new callback halve is created + response_for_tx: mpsc::Sender, +} + +pub struct ExecutorHalve { + /// communicates the executor is waiting for a callback for a given transaction + waiting_for_op_tx: mpsc::Sender, + /// receives the callback response from the `process_message` task after completion + response_for_rx: mpsc::Receiver, + /// stores the completed operations if they haven't been asked for yet in the executor + completed: HashMap, } mod sealed { - use super::{ExecutorHalve, NetworkEventListenerHalve}; - pub(crate) trait ChannelHalve {} + use super::{Callback, ExecutorHalve, NetworkEventListenerHalve}; + pub trait ChannelHalve {} impl ChannelHalve for NetworkEventListenerHalve {} impl ChannelHalve for ExecutorHalve {} + impl ChannelHalve for Callback {} } -#[allow(unused)] #[async_trait::async_trait] trait ComposeNetworkMessage where Self: Sized, Op: Operation + Send + 'static, { - fn initiate_op(self, op_manager: &OpManager) -> Op { - todo!() - } + fn initiate_op(self, op_manager: &OpManager) -> Op; - async fn resume_op(op: Op, op_manager: &OpManager) -> Result { - todo!() - } + async fn resume_op(op: Op, op_manager: &OpManager) -> Result<(), OpError>; } #[allow(unused)] @@ -202,13 +329,8 @@ impl ComposeNetworkMessage for GetContract { operations::get::start_op(self.key, self.fetch_contract) } - async fn resume_op( - op: operations::get::GetOp, - op_manager: &OpManager, - ) -> Result { - let id = *op.id(); - operations::get::request_get(op_manager, op, None).await?; - Ok(id) + async fn resume_op(op: operations::get::GetOp, op_manager: &OpManager) -> Result<(), OpError> { + operations::get::request_get(op_manager, op).await } } @@ -217,7 +339,47 @@ struct SubscribeContract { key: ContractKey, } -impl ComposeNetworkMessage for SubscribeContract {} +#[async_trait::async_trait] +impl ComposeNetworkMessage for SubscribeContract { + fn initiate_op(self, _op_manager: &OpManager) -> operations::subscribe::SubscribeOp { + operations::subscribe::start_op(self.key) + } + + async fn resume_op( + op: operations::subscribe::SubscribeOp, + op_manager: &OpManager, + ) -> Result<(), OpError> { + operations::subscribe::request_subscribe(op_manager, op).await + } +} + +#[allow(unused)] +struct PutContract { + contract: ContractContainer, + state: WrappedState, + related_contracts: RelatedContracts<'static>, +} + +#[async_trait::async_trait] +impl ComposeNetworkMessage for PutContract { + fn initiate_op(self, op_manager: &OpManager) -> operations::put::PutOp { + let PutContract { + contract, + state, + related_contracts, + } = self; + operations::put::start_op( + contract, + related_contracts, + state, + op_manager.ring.max_hops_to_live, + ) + } + + async fn resume_op(op: operations::put::PutOp, op_manager: &OpManager) -> Result<(), OpError> { + operations::put::request_put(op_manager, op).await + } +} #[allow(unused)] struct UpdateContract { @@ -225,7 +387,20 @@ struct UpdateContract { new_state: WrappedState, } -impl ComposeNetworkMessage for UpdateContract {} +#[async_trait::async_trait] +impl ComposeNetworkMessage for UpdateContract { + fn initiate_op(self, op_manager: &OpManager) -> operations::update::UpdateOp { + let UpdateContract { key, new_state } = self; + operations::update::start_op(key, new_state, op_manager.ring.max_hops_to_live) + } + + async fn resume_op( + op: operations::update::UpdateOp, + op_manager: &OpManager, + ) -> Result<(), OpError> { + operations::update::request_update(op_manager, op, None).await + } +} #[async_trait::async_trait] pub(crate) trait ContractExecutor: Send + 'static { @@ -234,18 +409,20 @@ pub(crate) trait ContractExecutor: Send + 'static { key: ContractKey, fetch_contract: bool, ) -> Result<(WrappedState, Option), ExecutorError>; - async fn store_contract( - &mut self, - contract: ContractContainer, - ) -> Result<(), crate::runtime::ContractError>; + async fn store_contract(&mut self, contract: ContractContainer) -> Result<(), ExecutorError>; async fn upsert_contract_state( &mut self, key: ContractKey, - state: Either>, + update: Either>, related_contracts: RelatedContracts<'static>, - params: Option>, - ) -> Result; + code: Option, + ) -> Result; + + async fn subscribe_to_contract( + &mut self, + key: ContractKey, + ) -> Result; } /// A WASM executor which will run any contracts, delegates, etc. registered. @@ -261,90 +438,14 @@ pub struct Executor { mode: OperationMode, runtime: R, pub state_store: StateStore, + /// Notification channels for any clients subscribed to updates for a given contract. update_notifications: HashMap)>>, + /// Summaries of the state of all clients subscribed to a given contract. subscriber_summaries: HashMap>>>, + /// Attested contract instances for a given delegate. delegate_attested_ids: HashMap>, - #[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - event_loop_channel: Option>, -} - -#[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) -))] -impl Executor { - pub(crate) fn event_loop_channel( - &mut self, - channel: ExecutorToEventLoopChannel, - ) { - self.event_loop_channel = Some(channel); - } - - async fn subscribe(&mut self, key: ContractKey) -> Result<(), ExecutorError> { - #[cfg(any( - all(not(feature = "local-mode"), not(feature = "network-mode")), - all(feature = "local-mode", feature = "network-mode") - ))] - { - if self.mode == OperationMode::Local { - return Ok(()); - } - } - let request = SubscribeContract { key }; - let op: operations::subscribe::SubscribeOp = self - .op_request(request) - .await - .map_err(ExecutorError::other)?; - let _sub: operations::subscribe::SubscribeResult = - op.try_into().map_err(ExecutorError::other)?; - Ok(()) - } - #[inline] - async fn local_state_or_from_network( - &mut self, - id: &ContractInstanceId, - ) -> Result, ExecutorError> { - if let Ok(contract) = self.state_store.get(&(*id).into()).await { - return Ok(Either::Left(contract)); - }; - let request: GetContract = GetContract { - key: (*id).into(), - fetch_contract: true, - }; - let op: operations::get::GetOp = self - .op_request(request) - .await - .map_err(ExecutorError::other)?; - let get_result: operations::get::GetResult = op.try_into().map_err(ExecutorError::other)?; - Ok(Either::Right(get_result)) - } - - // FIXME: must add suspension and resuming when doing this, - // otherwise it may be possible to end up in a deadlock waiting for a tree of contract - // dependencies to be resolved - async fn op_request(&mut self, request: M) -> Result - where - Op: Operation + Send + 'static, - M: ComposeNetworkMessage, - { - debug_assert!(self.event_loop_channel.is_some()); - let channel = match self.event_loop_channel.as_mut() { - Some(ch) => ch, - None => { - // Safety: this should be always set if network mode is ambiguous - // or using network mode unequivocally - unsafe { std::hint::unreachable_unchecked() } - } - }; - channel.send_to_event_loop(request).await?; - todo!() - } + event_loop_channel: Option>, } impl Executor { @@ -353,6 +454,7 @@ impl Executor { ctrl_handler: impl FnOnce() -> Result<(), DynError>, mode: OperationMode, runtime: R, + event_loop_channel: Option>, ) -> Result { ctrl_handler()?; @@ -367,12 +469,7 @@ impl Executor { update_notifications: HashMap::default(), subscriber_summaries: HashMap::default(), delegate_attested_ids: HashMap::default(), - #[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - event_loop_channel: None, + event_loop_channel, }) } @@ -418,947 +515,42 @@ impl Executor { Ok((contract_store, delegate_store, secret_store, state_store)) } -} - -impl Executor { - pub async fn from_config(config: PeerCliConfig) -> Result { - let (contract_store, delegate_store, secret_store, state_store) = - Self::get_stores(&config).await?; - let rt = Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(); - Executor::new( - state_store, - || { - crate::util::set_cleanup_on_exit()?; - Ok(()) - }, - OperationMode::Local, - rt, - ) - .await - } - - pub fn register_contract_notifier( - &mut self, - key: ContractKey, - cli_id: ClientId, - notification_ch: tokio::sync::mpsc::UnboundedSender, - summary: Option>, - ) -> Result<(), Box> { - let channels = self.update_notifications.entry(key.clone()).or_default(); - if let Ok(i) = channels.binary_search_by_key(&&cli_id, |(p, _)| p) { - let (_, existing_ch) = &channels[i]; - if !existing_ch.same_channel(¬ification_ch) { - return Err(RequestError::from(CoreContractError::Subscribe { - key, - cause: format!("Peer {cli_id} already subscribed"), - }) - .into()); - } - } else { - channels.push((cli_id, notification_ch)); - } - - if self - .subscriber_summaries - .entry(key.clone()) - .or_default() - .insert(cli_id, summary.map(StateSummary::into_owned)) - .is_some() - { - tracing::warn!( - "contract {key} already was registered for peer {cli_id}; replaced summary" - ); - } - Ok(()) - } - - pub async fn preload( - &mut self, - cli_id: ClientId, - contract: ContractContainer, - state: WrappedState, - related_contracts: RelatedContracts<'static>, - ) { - if let Err(err) = self - .contract_requests( - ContractRequest::Put { - contract, - state, - related_contracts, - }, - cli_id, - None, - ) - .await - { - match err.0 { - Either::Left(err) => tracing::error!("req error: {err}"), - Either::Right(err) => tracing::error!("other error: {err}"), - } - } - } - - pub async fn handle_request<'a>( - &mut self, - id: ClientId, - req: ClientRequest<'a>, - updates: Option>>, - ) -> Response { - match req { - ClientRequest::ContractOp(op) => self.contract_requests(op, id, updates).await, - ClientRequest::DelegateOp(op) => self.delegate_request(op, None), - ClientRequest::Disconnect { cause } => { - if let Some(cause) = cause { - tracing::info!("disconnecting cause: {cause}"); - } - Err(RequestError::Disconnect.into()) - } - _ => Err(ExecutorError::other("not supported")), - } - } - - /// Responde to requests made through any API's from client applications locally. - pub async fn contract_requests( - &mut self, - req: ContractRequest<'_>, - cli_id: ClientId, - updates: Option>>, - ) -> Response { - match req { - ContractRequest::Put { - contract, - state, - related_contracts, - } => { - self.perform_contract_put(contract, state, related_contracts) - .await - } - ContractRequest::Update { key, data } => self.perform_contract_update(key, data).await, - ContractRequest::Get { - key, - fetch_contract: contract, - } => self.perform_contract_get(contract, key).await, - ContractRequest::Subscribe { key, summary } => { - let updates = - updates.ok_or_else(|| ExecutorError::other("missing update channel"))?; - self.register_contract_notifier(key.clone(), cli_id, updates, summary)?; - let res = self.perform_contract_get(false, key.clone()).await?; - #[cfg(any( - all(not(feature = "local-mode"), not(feature = "network-mode")), - all(feature = "local-mode", feature = "network-mode"), - all(feature = "network-mode", not(feature = "network-mode")) - ))] - { - self.subscribe(key).await?; - } - // by default a subscribe op has an implicit get - Ok(res) - } - _ => Err(ExecutorError::other("not supported")), - } - } - - pub fn delegate_request( - &mut self, - req: DelegateRequest<'_>, - attestaded_contract: Option<&ContractInstanceId>, - ) -> Response { - match req { - DelegateRequest::RegisterDelegate { - delegate, - cipher, - nonce, - } => { - use chacha20poly1305::{KeyInit, XChaCha20Poly1305}; - let key = delegate.key().clone(); - let arr = GenericArray::from_slice(&cipher); - let cipher = XChaCha20Poly1305::new(arr); - let nonce = GenericArray::from_slice(&nonce).to_owned(); - tracing::debug!("registering delegate `{key}"); - if let Some(contract) = attestaded_contract { - self.delegate_attested_ids - .entry(key.clone()) - .or_default() - .push(*contract); - } - match self.runtime.register_delegate(delegate, cipher, nonce) { - Ok(_) => Ok(DelegateResponse { - key, - values: Vec::new(), - }), - Err(err) => { - tracing::error!("failed registering delegate `{key}`: {err}"); - Err(ExecutorError::other(CoreDelegateError::RegisterError(key))) - } - } - } - DelegateRequest::UnregisterDelegate(key) => { - self.delegate_attested_ids.remove(&key); - match self.runtime.unregister_delegate(&key) { - Ok(_) => Ok(HostResponse::Ok), - Err(err) => { - tracing::error!("failed unregistering delegate `{key}`: {err}"); - Ok(HostResponse::Ok) - } - } - } - DelegateRequest::GetSecretRequest { - key, - params, - get_request, - } => { - let attested = attestaded_contract.and_then(|contract| { - self.delegate_attested_ids - .get(&key) - .and_then(|contracts| contracts.iter().find(|c| *c == contract)) - }); - match self.runtime.inbound_app_message( - &key, - ¶ms, - attested.map(|c| c.as_bytes()), - vec![InboundDelegateMsg::GetSecretRequest(get_request)], - ) { - Ok(values) => Ok(HostResponse::DelegateResponse { key, values }), - Err(err) if err.delegate_is_missing() => { - tracing::error!("delegate not found `{key}`"); - Err(ExecutorError::request(CoreDelegateError::Missing(key))) - } - Err(err) => Err(ExecutorError::other(format!( - "uncontrolled error while getting secret for `{key}`: {err}" - ))), - } - } - DelegateRequest::ApplicationMessages { - key, - inbound, - params, - } => { - let attested = attestaded_contract.and_then(|contract| { - self.delegate_attested_ids - .get(&key) - .and_then(|contracts| contracts.iter().find(|c| *c == contract)) - }); - match self.runtime.inbound_app_message( - &key, - ¶ms, - attested.map(|c| c.as_bytes()), - inbound - .into_iter() - .map(InboundDelegateMsg::into_owned) - .collect(), - ) { - Ok(values) => Ok(HostResponse::DelegateResponse { key, values }), - Err(err) if err.is_delegate_exec_error() | err.is_execution_error() => { - tracing::error!("failed processing messages for delegate `{key}`: {err}"); - Err(ExecutorError::request(CoreDelegateError::ExecutionError( - format!("{err}"), - ))) - } - Err(err) if err.delegate_is_missing() => { - tracing::error!("delegate not found `{key}`"); - Err(ExecutorError::request(CoreDelegateError::Missing(key))) - } - Err(err) if err.secret_is_missing() => { - tracing::error!("secret not found `{key}`"); - Err(ExecutorError::request(CoreDelegateError::MissingSecret { - key, - secret: err.get_secret_id(), - })) - } - Err(err) => { - tracing::error!("failed executing delegate `{key}`: {err}"); - Err(ExecutorError::other(format!( - "uncontrolled error while executing `{key}`" - ))) - } - } - } - _ => Err(ExecutorError::other("not supported")), - } - } - async fn perform_contract_put( - &mut self, - contract: ContractContainer, - state: WrappedState, - related_contracts: RelatedContracts<'_>, - ) -> Response { - let key = contract.key(); - let params = contract.params(); - - if self.get_local_contract(key.id()).await.is_ok() { - // already existing contract, just try to merge states - return self - .perform_contract_update(key, UpdateData::State(state.into())) - .await; - } - - self.verify_and_store_contract(state.clone(), contract, related_contracts) - .await?; - - self.send_update_notification(&key, ¶ms, &state) - .await - .map_err(|_| { - ExecutorError::request(CoreContractError::Put { - key: key.clone(), - cause: "failed while sending notifications".to_owned(), - }) - })?; - Ok(ContractResponse::PutResponse { key }.into()) - } - - async fn perform_contract_update( - &mut self, - key: ContractKey, - data: UpdateData<'_>, - ) -> Response { - let parameters = { - self.state_store - .get_params(&key) - .await - .map_err(ExecutorError::other)? - .ok_or_else(|| { - RequestError::ContractError(CoreContractError::Update { - cause: format!("missing contract parameters: {key}"), - key: key.clone(), - }) - })? - }; - let new_state = { - let state = self - .state_store - .get(&key) - .await - .map_err(ExecutorError::other)? - .clone(); - let mut retrieved_contracts = Vec::new(); - retrieved_contracts.push(data); - - let start = Instant::now(); - loop { - let update_modification = - match self - .runtime - .update_state(&key, ¶meters, &state, &retrieved_contracts) - { - Err(err) if err.is_contract_exec_error() => { - return Err(ExecutorError::request(CoreContractError::Update { - key: key.clone(), - cause: format!("{err}"), - })); - } - Err(err) => return Err(ExecutorError::other(err)), - Ok(result) => result, - }; - let UpdateModification { - new_state, related, .. - } = update_modification; - if let Some(new_state) = new_state { - let new_state = WrappedState::new(new_state.into_bytes()); - self.state_store - .update(&key, new_state.clone()) - .await - .map_err(ExecutorError::other)?; - break new_state; - } else if !related.is_empty() { - // some required contracts are missing - let required_contracts = related.len() + 1; - for RelatedContract { - contract_instance_id: id, - mode, - } in related - { - match self.state_store.get(&id.into()).await { - Ok(state) => { - // in this case we are already subscribed to and are updating this contract, - // we can try first with the existing value - retrieved_contracts.push(UpdateData::RelatedState { - related_to: id, - state: state.into(), - }); - } - #[cfg(any( - all(not(feature = "local-mode"), not(feature = "network-mode")), - all(feature = "local-mode", feature = "network-mode") - ))] - Err(StateStoreError::MissingContract(_)) - if self.mode == OperationMode::Network => - { - let state = match self.local_state_or_from_network(&id).await? { - Either::Left(state) => state, - Either::Right(GetResult { state, contract }) => { - let contract = contract.unwrap(); // fixme: deal with unwrap - self.verify_and_store_contract( - state.clone(), - contract, - RelatedContracts::default(), - ) - .await?; - state - } - }; - retrieved_contracts.push(UpdateData::State(state.into())); - match mode { - RelatedMode::StateOnce => {} - RelatedMode::StateThenSubscribe => { - self.subscribe(id.into()).await?; - } - } - } - #[cfg(all(not(feature = "local-mode"), feature = "network-mode"))] - Err(StateStoreError::MissingContract(_)) => { - let state = match self.local_state_or_from_network(&id).await? { - Either::Left(state) => state, - Either::Right(GetResult { state, contract }) => { - self.verify_and_store_contract( - state.clone(), - contract, - RelatedContracts::default(), - ) - .await?; - state - } - }; - retrieved_contracts.push(UpdateData::State(state.into())); - match mode { - RelatedMode::StateOnce => {} - RelatedMode::StateThenSubscribe => { - self.subscribe(id.into()).await?; - } - } - } - Err(other_err) => { - let _ = mode; - return Err(ExecutorError::other(other_err)); - } - } - } - if retrieved_contracts.len() == required_contracts { - // try running again with all the related contracts retrieved - continue; - } else if start.elapsed() > Duration::from_secs(10) { - /* make this timeout configurable, and anyway should be controlled globally*/ - return Err(RequestError::Timeout.into()); - } - } else { - // state wasn't updated - break state; - } - } + async fn op_request(&mut self, request: M) -> Result + where + Op: Operation + Send + TryFrom + 'static, + ::Result: TryFrom, + M: ComposeNetworkMessage, + { + let Some(ch) = &mut self.event_loop_channel else { + return Err(ExecutorError::other("missing event loop channel")); }; - // in the network impl this would be sent over the network - let summary = self - .runtime - .summarize_state(&key, ¶meters, &new_state) + let transaction = ch + .send_to_event_loop(request) + .await .map_err(ExecutorError::other)?; - self.send_update_notification(&key, ¶meters, &new_state) - .await?; - - #[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - { - #[cfg(any( - all(feature = "local-mode", feature = "network-mode"), - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - { - if self.mode == OperationMode::Local { - return Ok(ContractResponse::UpdateResponse { key, summary }.into()); - } - } - // notify peers with deltas from summary in network - let request = UpdateContract { - key: key.clone(), - new_state, - }; - let op: operations::update::UpdateOp = self - .op_request(request) - .await - .map_err(ExecutorError::other)?; - let _update: operations::update::UpdateResult = - op.try_into().map_err(ExecutorError::other)?; - } - - Ok(ContractResponse::UpdateResponse { key, summary }.into()) - } - - async fn perform_contract_get(&mut self, contract: bool, key: ContractKey) -> Response { - let mut got_contract = None; - - #[cfg(any( - all(not(feature = "local-mode"), not(feature = "network-mode")), - all(feature = "local-mode", feature = "network-mode"), - ))] - { - if contract && self.mode == OperationMode::Local { - let Some(contract) = self.get_contract_locally(&key).await? else { - return Err(ExecutorError::other(RequestError::from( - CoreContractError::Get { - key: key.clone(), - cause: "Missing contract or parameters".into(), - }, - ))); - }; - got_contract = Some(contract); - } else if contract { - got_contract = self.get_contract_from_network(key.clone()).await?; - } - } - - #[cfg(all(feature = "local-mode", not(feature = "network-mode")))] - if contract { - let Some(contract) = self - .get_contract_locally(&key) - .await - .map_err(Either::Left)? - else { - return Err(ExecutorError::other(RequestError::from( - CoreContractError::Get { - key: key.clone(), - cause: "Missing contract or parameters".into(), - }, - ))); - }; - got_contract = Some(contract); - } - - #[cfg(all(feature = "network-mode", not(feature = "local-mode")))] - if contract { - if let Ok(Some(contract)) = self.get_contract_locally(&key).await { - got_contract = Some(contract); - } else { - got_contract = self.get_contract_from_network(key.clone()).await?; - } - } - - match self.state_store.get(&key).await { - Ok(state) => Ok(ContractResponse::GetResponse { - key, - contract: got_contract, - state, - } - .into()), - Err(StateStoreError::MissingContract(_)) => Err(ExecutorError::other( - RequestError::from(CoreContractError::Get { - key, - cause: "Missing contract state".into(), - }), - )), - Err(err) => Err(ExecutorError::other(RequestError::from( - CoreContractError::Get { - key, - cause: format!("{err}"), - }, - ))), - } - } - - #[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - async fn get_contract_from_network( - &mut self, - key: ContractKey, - ) -> Result, ExecutorError> { - loop { - if let Ok(Some(contract)) = self.get_contract_locally(&key).await { - break Ok(Some(contract)); - } else { - #[cfg(any( - all(not(feature = "local-mode"), not(feature = "network-mode")), - all(feature = "local-mode", feature = "network-mode") - ))] - { - if self.mode == OperationMode::Local { - return Err(ExecutorError::request(RequestError::ContractError( - CoreContractError::MissingRelated { key: *key.id() }, - ))); - } - } - match self - .local_state_or_from_network(&key.clone().into()) - .await? - { - Either::Right(GetResult { state, contract }) => { - let contract = contract.unwrap(); // fixme: deal with unwrap - self.verify_and_store_contract( - state, - contract.clone(), - RelatedContracts::default(), - ) - .await?; - break Ok(Some(contract)); - } - Either::Left(_state) => continue, - } - } - } - } - - async fn get_local_contract( - &self, - id: &ContractInstanceId, - ) -> Result, Either, DynError>> { - let Ok(contract) = self.state_store.get(&(*id).into()).await else { - return Err(Either::Right( - CoreContractError::MissingRelated { key: *id }.into(), - )); - }; - let state: &[u8] = unsafe { - // Safety: this is fine since this will never scape this scope - std::mem::transmute::<&[u8], &'_ [u8]>(contract.as_ref()) - }; - Ok(State::from(state)) - } - - async fn verify_and_store_contract<'a>( - &mut self, - state: WrappedState, - trying_container: ContractContainer, - mut related_contracts: RelatedContracts<'a>, - ) -> Result<(), ExecutorError> { - let key = trying_container.key(); - let params = trying_container.params(); - - const DEPENDENCY_CYCLE_LIMIT_GUARD: usize = 100; - let mut iterations = 0; - - let original_key = key.clone(); - let original_state = state.clone(); - let original_params = params.clone(); - let mut trying_key = key; - let mut trying_state = state; - let mut trying_params = params; - let mut trying_contract = Some(trying_container); - - while iterations < DEPENDENCY_CYCLE_LIMIT_GUARD { - if let Some(contract) = trying_contract.take() { - self.runtime - .contract_store - .store_contract(contract) - .map_err(ExecutorError::other)?; - } - - let result = self - .runtime - .validate_state( - &trying_key, - &trying_params, - &trying_state, - &related_contracts, - ) - .map_err(|err| { - let _ = self.runtime.contract_store.remove_contract(&trying_key); - ExecutorError::other(err) - })?; - - let is_valid = match result { - ValidateResult::Valid => true, - ValidateResult::Invalid => false, - ValidateResult::RequestRelated(related) => { - iterations += 1; - related_contracts.missing(related); - for (id, related) in related_contracts.update() { - if related.is_none() { - #[cfg(all(feature = "local-mode", not(feature = "network-mode")))] - { - let current_state = self.get_local_contract(id).await?; - *related = Some(current_state); - } - - #[cfg(any( - all(not(feature = "local-mode"), not(feature = "network-mode")), - all(feature = "local-mode", feature = "network-mode"), - all(not(feature = "local-mode"), feature = "network-mode") - ))] - { - match self.local_state_or_from_network(id).await? { - Either::Left(state) => { - *related = Some(state.into()); - } - Either::Right(result) => { - let contract = result.contract.unwrap(); // fixme: deal with unwrap - trying_key = (*id).into(); - trying_params = contract.params(); - trying_state = result.state; - trying_contract = Some(contract); - continue; - } - } - } - } - } + // FIXME: must add a way to suspend a request while waiting for result and resume upon getting + // an answer back so we don't block the executor itself. + // otherwise it may be possible to end up in a deadlock waiting for a tree of contract + // dependencies to be resolved + let result = loop { + match ch.receive_op_result::(transaction).await { + Ok(result) => break result, + Err(CallbackError::MissingResult) => { + tokio::time::sleep(Duration::from_secs(10)).await; continue; } - }; - - if !is_valid { - return Err(ExecutorError::request(CoreContractError::Put { - key: trying_key.clone(), - cause: "not valid".to_owned(), - })); - } - - self.state_store - .store( - trying_key.clone(), - trying_state.clone(), - trying_params.clone(), - ) - .await - .map_err(ExecutorError::other)?; - if trying_key != original_key { - trying_key = original_key.clone(); - trying_params = original_params.clone(); - trying_state = original_state.clone(); - continue; - } - break; - } - if iterations == DEPENDENCY_CYCLE_LIMIT_GUARD { - return Err(ExecutorError::request(CoreContractError::MissingRelated { - key: *original_key.id(), - })); - } - Ok(()) - } - - async fn send_update_notification<'a>( - &mut self, - key: &ContractKey, - params: &Parameters<'a>, - new_state: &WrappedState, - ) -> Result<(), ExecutorError> { - tracing::debug!(contract = %key, "notify of contract update"); - if let Some(notifiers) = self.update_notifications.get_mut(key) { - let summaries = self.subscriber_summaries.get_mut(key).unwrap(); - // in general there should be less than 32 failures - let mut failures = Vec::with_capacity(32); - for (peer_key, notifier) in notifiers.iter() { - let peer_summary = summaries.get_mut(peer_key).unwrap(); - let update = match peer_summary { - Some(summary) => self - .runtime - .get_state_delta(key, params, new_state, &*summary) - .map_err(|err| { - tracing::error!("{err}"); - let is_contract_exec_error = err.is_contract_exec_error(); - match is_contract_exec_error { - true => ExecutorError::request(CoreContractError::Put { - key: key.clone(), - cause: format!("{err}"), - }), - _ => ExecutorError::other(err), - } - })? - .to_owned() - .into(), - None => UpdateData::State(State::from(new_state.as_ref()).into_owned()), - }; - if let Err(err) = notifier.send(Ok(ContractResponse::UpdateNotification { - key: key.clone(), - update, - } - .into())) - { - failures.push(*peer_key); - tracing::error!(cli_id = %peer_key, "{err}"); - } else { - tracing::debug!(cli_id = %peer_key, contract = %key, "notified of update"); + Err(CallbackError::Conversion(err)) => { + tracing::error!("expect message of one type but got an other: {err}"); + return Err(ExecutorError::other(err)); } + Err(CallbackError::Err(other)) => return Err(other), } - if !failures.is_empty() { - notifiers.retain(|(c, _)| !failures.contains(c)); - } - } - Ok(()) - } - - async fn get_contract_locally( - &self, - key: &ContractKey, - ) -> Result, ExecutorError> { - let Some(parameters) = self - .state_store - .get_params(key) - .await - .map_err(ExecutorError::other)? - else { - return Ok(None); - }; - let Some(contract) = self.runtime.contract_store.fetch_contract(key, ¶meters) else { - return Ok(None); - }; - Ok(Some(contract)) - } -} - -impl Executor { - pub async fn new_mock(identifier: &str) -> Result { - let data_dir = std::env::temp_dir().join(format!("freenet-executor-{identifier}")); - - let contracts_data_dir = data_dir.join("contracts"); - std::fs::create_dir_all(&contracts_data_dir).expect("directory created"); - let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; - - let db_path = data_dir.join("db"); - std::fs::create_dir_all(&db_path).expect("directory created"); - let log_file = data_dir.join("_EVENT_LOG_LOCAL"); - crate::config::Config::set_event_log(log_file); - let state_store = - StateStore::new(Storage::new(Some(&db_path)).await?, u16::MAX as u32).unwrap(); - - let executor = Executor::new( - state_store, - || Ok(()), - OperationMode::Local, - super::MockRuntime { contract_store }, - ) - .await?; - Ok(executor) - } - - pub async fn handle_request<'a>( - &mut self, - _id: ClientId, - _req: ClientRequest<'a>, - _updates: Option>>, - ) -> Response { - unreachable!() - } -} - -#[async_trait::async_trait] -impl ContractExecutor for Executor { - async fn fetch_contract( - &mut self, - key: ContractKey, - fetch_contract: bool, - ) -> Result<(WrappedState, Option), ExecutorError> { - match self.perform_contract_get(fetch_contract, key).await { - Ok(HostResponse::ContractResponse(ContractResponse::GetResponse { - contract, - state, - .. - })) => Ok((state, contract)), - Err(err) => Err(err), - Ok(_) => { - // Safety: check `perform_contract_get` to indeed check this should never happen - unsafe { std::hint::unreachable_unchecked() } - } - } - } - - async fn store_contract( - &mut self, - contract: ContractContainer, - ) -> Result<(), crate::runtime::ContractError> { - self.runtime.contract_store.store_contract(contract) - } - - async fn upsert_contract_state( - &mut self, - _key: ContractKey, - _state: Either>, - _related_contracts: RelatedContracts<'static>, - _params: Option>, - ) -> Result { - todo!() - } -} - -#[async_trait::async_trait] -impl ContractExecutor for Executor { - async fn fetch_contract( - &mut self, - key: ContractKey, - _fetch_contract: bool, - ) -> Result<(WrappedState, Option), ExecutorError> { - let Some(parameters) = self - .state_store - .get_params(&key) - .await - .map_err(ExecutorError::other)? - else { - return Err(ExecutorError::other(format!( - "missing parameters for contract {key}" - ))); - }; - let Ok(state) = self.state_store.get(&key).await else { - return Err(ExecutorError::other(format!( - "missing state for contract {key}" - ))); }; - let contract = self - .runtime - .contract_store - .fetch_contract(&key, ¶meters); - Ok((state, contract)) - } - - async fn store_contract( - &mut self, - contract: ContractContainer, - ) -> Result<(), crate::runtime::ContractError> { - self.runtime.contract_store.store_contract(contract)?; - Ok(()) - } - - async fn upsert_contract_state( - &mut self, - key: ContractKey, - state: Either>, - _related_contracts: RelatedContracts<'static>, - params: Option>, - ) -> Result { - // todo: instead allow to perform mutations per contract based on incoming value so we can track - // state values over the network - match (state, params) { - (Either::Left(state), Some(params)) => { - self.state_store - .store(key, state.clone(), params.into_owned()) - .await?; - return Ok(state); - } - _ => unreachable!(), - } - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::{ - contract::MockRuntime, - runtime::{ContractStore, StateStore}, - }; - - #[tokio::test(flavor = "multi_thread")] - async fn local_node_handle() -> Result<(), Box> { - const MAX_SIZE: i64 = 10 * 1024 * 1024; - const MAX_MEM_CACHE: u32 = 10_000_000; - let tmp_dir = tempfile::tempdir()?; - let contract_store = ContractStore::new(tmp_dir.path().join("executor-test"), MAX_SIZE)?; - let state_store = StateStore::new(Storage::new(None).await?, MAX_MEM_CACHE).unwrap(); - let mut counter = 0; - Executor::new( - state_store, - || { - counter += 1; - Ok(()) - }, - OperationMode::Local, - MockRuntime { contract_store }, - ) - .await - .expect("local node with handle"); - - assert_eq!(counter, 1); - Ok(()) + let result = ::try_from(result).map_err(|err| { + tracing::debug!("didn't get result back: {err}"); + ExecutorError::other(err) + })?; + Ok(result) } } diff --git a/crates/core/src/contract/executor/mock_runtime.rs b/crates/core/src/contract/executor/mock_runtime.rs new file mode 100644 index 000000000..ad89c3797 --- /dev/null +++ b/crates/core/src/contract/executor/mock_runtime.rs @@ -0,0 +1,151 @@ +use super::*; + +pub(crate) struct MockRuntime { + pub contract_store: ContractStore, +} + +impl Executor { + pub async fn new_mock( + identifier: &str, + event_loop_channel: ExecutorToEventLoopChannel, + ) -> Result { + let data_dir = std::env::temp_dir().join(format!("freenet-executor-{identifier}")); + + let contracts_data_dir = data_dir.join("contracts"); + std::fs::create_dir_all(&contracts_data_dir).expect("directory created"); + let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; + + let db_path = data_dir.join("db"); + std::fs::create_dir_all(&db_path).expect("directory created"); + let log_file = data_dir.join("_EVENT_LOG_LOCAL"); + crate::config::Config::set_event_log(log_file); + let state_store = + StateStore::new(Storage::new(Some(&db_path)).await?, u16::MAX as u32).unwrap(); + + let executor = Executor::new( + state_store, + || Ok(()), + OperationMode::Local, + MockRuntime { contract_store }, + Some(event_loop_channel), + ) + .await?; + Ok(executor) + } + + pub async fn handle_request<'a>( + &mut self, + _id: ClientId, + _req: ClientRequest<'a>, + _updates: Option>>, + ) -> Response { + unreachable!() + } +} + +#[async_trait::async_trait] +impl ContractExecutor for Executor { + async fn fetch_contract( + &mut self, + key: ContractKey, + _fetch_contract: bool, + ) -> Result<(WrappedState, Option), ExecutorError> { + let Some(parameters) = self + .state_store + .get_params(&key) + .await + .map_err(ExecutorError::other)? + else { + return Err(ExecutorError::other(format!( + "missing parameters for contract {key}" + ))); + }; + let Ok(state) = self.state_store.get(&key).await else { + return Err(ExecutorError::other(format!( + "missing state for contract {key}" + ))); + }; + let contract = self + .runtime + .contract_store + .fetch_contract(&key, ¶meters); + Ok((state, contract)) + } + + async fn store_contract(&mut self, contract: ContractContainer) -> Result<(), ExecutorError> { + self.runtime + .contract_store + .store_contract(contract) + .map_err(ExecutorError::other)?; + Ok(()) + } + + async fn upsert_contract_state( + &mut self, + key: ContractKey, + state: Either>, + related_contracts: RelatedContracts<'static>, + code: Option, + ) -> Result { + // todo: instead allow to perform mutations per contract based on incoming value so we can track + // state values over the network + match (state, code) { + (Either::Left(incoming_state), Some(contract)) => { + self.state_store + .store(key, incoming_state.clone(), contract.params().into_owned()) + .await + .map_err(ExecutorError::other)?; + + let request = PutContract { + contract, + state: incoming_state.clone(), + related_contracts, + }; + let _op: Result = self.op_request(request).await; + + return Ok(incoming_state); + } + _ => unreachable!(), + } + } + + async fn subscribe_to_contract( + &mut self, + key: ContractKey, + ) -> Result { + let request = SubscribeContract { key }; + let result: operations::subscribe::SubscribeResult = self.op_request(request).await?; + Ok(result.subscribed_to) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::wasm_runtime::{ContractStore, StateStore}; + + #[tokio::test(flavor = "multi_thread")] + async fn local_node_handle() -> Result<(), Box> { + const MAX_SIZE: i64 = 10 * 1024 * 1024; + const MAX_MEM_CACHE: u32 = 10_000_000; + let tmp_dir = tempfile::tempdir()?; + let contract_store = ContractStore::new(tmp_dir.path().join("executor-test"), MAX_SIZE)?; + let state_store = StateStore::new(Storage::new(None).await?, MAX_MEM_CACHE).unwrap(); + let mut counter = 0; + Executor::new( + state_store, + || { + counter += 1; + Ok(()) + }, + OperationMode::Local, + MockRuntime { contract_store }, + None, + ) + .await + .expect("local node with handle"); + + assert_eq!(counter, 1); + Ok(()) + } +} diff --git a/crates/core/src/contract/executor/runtime.rs b/crates/core/src/contract/executor/runtime.rs new file mode 100644 index 000000000..864191e5d --- /dev/null +++ b/crates/core/src/contract/executor/runtime.rs @@ -0,0 +1,1022 @@ +use super::*; + +#[async_trait::async_trait] +impl ContractExecutor for Executor { + async fn fetch_contract( + &mut self, + key: ContractKey, + fetch_contract: bool, + ) -> Result<(WrappedState, Option), ExecutorError> { + match self.perform_contract_get(fetch_contract, key).await { + Ok(HostResponse::ContractResponse(ContractResponse::GetResponse { + contract, + state, + .. + })) => Ok((state, contract)), + Err(err) => Err(err), + Ok(_) => { + // Safety: check `perform_contract_get` to indeed check this should never happen + unsafe { std::hint::unreachable_unchecked() } + } + } + } + + async fn store_contract(&mut self, contract: ContractContainer) -> Result<(), ExecutorError> { + self.runtime + .contract_store + .store_contract(contract) + .map_err(ExecutorError::other) + } + + async fn upsert_contract_state( + &mut self, + key: ContractKey, + update: Either>, + related_contracts: RelatedContracts<'static>, + code: Option, + ) -> Result { + let params = if let Some(code) = &code { + code.params() + } else { + self.state_store + .get_params(&key) + .await + .map_err(ExecutorError::other)? + .ok_or_else(|| { + ExecutorError::request(StdContractError::Put { + key: key.clone(), + cause: "missing contract parameters".into(), + }) + })? + }; + + let contract = if let Some(code) = self.runtime.contract_store.fetch_contract(&key, ¶ms) + { + code + } else { + let code = code.ok_or_else(|| { + ExecutorError::request(StdContractError::MissingContract { + key: key.clone().into(), + }) + })?; + self.runtime + .contract_store + .store_contract(code.clone()) + .map_err(ExecutorError::other)?; + code + }; + + let mut updates = match update { + Either::Left(incoming_state) => { + let result = self + .runtime + .validate_state(&key, ¶ms, &incoming_state, &related_contracts) + .map_err(|err| { + let _ = self.runtime.contract_store.remove_contract(&key); + ExecutorError::other(err) + })?; + match result { + ValidateResult::Valid => {} + ValidateResult::Invalid => { + return Err(ExecutorError::request(StdContractError::invalid_put(key))); + } + ValidateResult::RequestRelated(mut related) => { + if let Some(key) = related.pop() { + return Err(ExecutorError::request(StdContractError::MissingRelated { + key, + })); + } else { + return Err(ExecutorError::internal_error()); + } + } + } + + let request = PutContract { + contract, + state: incoming_state.clone(), + related_contracts: related_contracts.clone(), + }; + let _op: operations::put::PutResult = self.op_request(request).await?; + + vec![UpdateData::State(incoming_state.clone().into())] + } + Either::Right(delta) => { + let valid = self + .runtime + .validate_delta(&key, ¶ms, &delta) + .map_err(|err| { + let _ = self.runtime.contract_store.remove_contract(&key); + ExecutorError::other(err) + })?; + if !valid { + return Err(ExecutorError::request(StdContractError::invalid_update( + key, + ))); + } + // todo: forward delta like we are doing with puts + vec![UpdateData::Delta(delta)] + } + }; + + let current_state = match self.state_store.get(&key).await { + Ok(s) => s, + Err(StateStoreError::MissingContract(_)) => { + return Err(ExecutorError::request(StdContractError::MissingContract { + key: key.into(), + })); + } + Err(StateStoreError::Any(err)) => return Err(ExecutorError::other(err)), + }; + + for (id, state) in related_contracts + .states() + .filter_map(|(id, c)| c.map(|c| (id, c))) + { + updates.push(UpdateData::RelatedState { + related_to: id, + state, + }); + } + + let updated_state = match self + .attempt_state_update(¶ms, ¤t_state, &key, &updates) + .await? + { + Either::Left(s) => s, + Either::Right(mut r) => { + let Some(c) = r.pop() else { + // this branch should be unreachable since attempt_state_update should only + return Err(ExecutorError::internal_error()); + }; + return Err(ExecutorError::request(StdContractError::MissingRelated { + key: c.contract_instance_id, + })); + } + }; + Ok(updated_state) + } + + async fn subscribe_to_contract( + &mut self, + key: ContractKey, + ) -> Result { + let request = SubscribeContract { key }; + let result: operations::subscribe::SubscribeResult = self.op_request(request).await?; + Ok(result.subscribed_to) + } +} + +impl Executor { + pub async fn from_config( + config: PeerCliConfig, + event_loop_channel: Option>, + ) -> Result { + let (contract_store, delegate_store, secret_store, state_store) = + Self::get_stores(&config).await?; + let rt = Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(); + Executor::new( + state_store, + || { + crate::util::set_cleanup_on_exit()?; + Ok(()) + }, + OperationMode::Local, + rt, + event_loop_channel, + ) + .await + } + + pub fn register_contract_notifier( + &mut self, + key: ContractKey, + cli_id: ClientId, + notification_ch: tokio::sync::mpsc::UnboundedSender, + summary: Option>, + ) -> Result<(), Box> { + let channels = self.update_notifications.entry(key.clone()).or_default(); + if let Ok(i) = channels.binary_search_by_key(&&cli_id, |(p, _)| p) { + let (_, existing_ch) = &channels[i]; + if !existing_ch.same_channel(¬ification_ch) { + return Err(RequestError::from(StdContractError::Subscribe { + key, + cause: format!("Peer {cli_id} already subscribed").into(), + }) + .into()); + } + } else { + channels.push((cli_id, notification_ch)); + } + + if self + .subscriber_summaries + .entry(key.clone()) + .or_default() + .insert(cli_id, summary.map(StateSummary::into_owned)) + .is_some() + { + tracing::warn!( + "contract {key} already was registered for peer {cli_id}; replaced summary" + ); + } + Ok(()) + } + + pub async fn preload( + &mut self, + cli_id: ClientId, + contract: ContractContainer, + state: WrappedState, + related_contracts: RelatedContracts<'static>, + ) { + if let Err(err) = self + .contract_requests( + ContractRequest::Put { + contract, + state, + related_contracts, + }, + cli_id, + None, + ) + .await + { + match err.0 { + Either::Left(err) => tracing::error!("req error: {err}"), + Either::Right(err) => tracing::error!("other error: {err}"), + } + } + } + + pub async fn handle_request<'a>( + &mut self, + id: ClientId, + req: ClientRequest<'a>, + updates: Option>>, + ) -> Response { + match req { + ClientRequest::ContractOp(op) => self.contract_requests(op, id, updates).await, + ClientRequest::DelegateOp(op) => self.delegate_request(op, None), + ClientRequest::Disconnect { cause } => { + if let Some(cause) = cause { + tracing::info!("disconnecting cause: {cause}"); + } + Err(RequestError::Disconnect.into()) + } + _ => Err(ExecutorError::other("not supported")), + } + } + + /// Responde to requests made through any API's from client applications locally. + pub async fn contract_requests( + &mut self, + req: ContractRequest<'_>, + cli_id: ClientId, + updates: Option>>, + ) -> Response { + match req { + ContractRequest::Put { + contract, + state, + related_contracts, + } => { + self.perform_contract_put(contract, state, related_contracts) + .await + } + ContractRequest::Update { key, data } => self.perform_contract_update(key, data).await, + ContractRequest::Get { + key, + fetch_contract: contract, + } => self.perform_contract_get(contract, key).await, + ContractRequest::Subscribe { key, summary } => { + let updates = + updates.ok_or_else(|| ExecutorError::other("missing update channel"))?; + self.register_contract_notifier(key.clone(), cli_id, updates, summary)?; + // by default a subscribe op has an implicit get + let res = self.perform_contract_get(false, key.clone()).await?; + #[cfg(any( + all(not(feature = "local-mode"), not(feature = "network-mode")), + all(feature = "local-mode", feature = "network-mode"), + all(feature = "network-mode", not(feature = "network-mode")) + ))] + { + self.subscribe(key).await?; + } + Ok(res) + } + _ => Err(ExecutorError::other("not supported")), + } + } + + pub fn delegate_request( + &mut self, + req: DelegateRequest<'_>, + attestaded_contract: Option<&ContractInstanceId>, + ) -> Response { + match req { + DelegateRequest::RegisterDelegate { + delegate, + cipher, + nonce, + } => { + use chacha20poly1305::{KeyInit, XChaCha20Poly1305}; + let key = delegate.key().clone(); + let arr = GenericArray::from_slice(&cipher); + let cipher = XChaCha20Poly1305::new(arr); + let nonce = GenericArray::from_slice(&nonce).to_owned(); + tracing::debug!("registering delegate `{key}"); + if let Some(contract) = attestaded_contract { + self.delegate_attested_ids + .entry(key.clone()) + .or_default() + .push(*contract); + } + match self.runtime.register_delegate(delegate, cipher, nonce) { + Ok(_) => Ok(DelegateResponse { + key, + values: Vec::new(), + }), + Err(err) => { + tracing::error!("failed registering delegate `{key}`: {err}"); + Err(ExecutorError::other(StdDelegateError::RegisterError(key))) + } + } + } + DelegateRequest::UnregisterDelegate(key) => { + self.delegate_attested_ids.remove(&key); + match self.runtime.unregister_delegate(&key) { + Ok(_) => Ok(HostResponse::Ok), + Err(err) => { + tracing::error!("failed unregistering delegate `{key}`: {err}"); + Ok(HostResponse::Ok) + } + } + } + DelegateRequest::GetSecretRequest { + key, + params, + get_request, + } => { + let attested = attestaded_contract.and_then(|contract| { + self.delegate_attested_ids + .get(&key) + .and_then(|contracts| contracts.iter().find(|c| *c == contract)) + }); + match self.runtime.inbound_app_message( + &key, + ¶ms, + attested.map(|c| c.as_bytes()), + vec![InboundDelegateMsg::GetSecretRequest(get_request)], + ) { + Ok(values) => Ok(HostResponse::DelegateResponse { key, values }), + Err(err) => Err(ExecutorError::execution( + err, + Some(InnerOpError::Delegate(key.clone())), + )), + } + } + DelegateRequest::ApplicationMessages { + key, + inbound, + params, + } => { + let attested = attestaded_contract.and_then(|contract| { + self.delegate_attested_ids + .get(&key) + .and_then(|contracts| contracts.iter().find(|c| *c == contract)) + }); + match self.runtime.inbound_app_message( + &key, + ¶ms, + attested.map(|c| c.as_bytes()), + inbound + .into_iter() + .map(InboundDelegateMsg::into_owned) + .collect(), + ) { + Ok(values) => Ok(HostResponse::DelegateResponse { key, values }), + + Err(err) => { + tracing::error!("failed executing delegate `{key}`: {err}"); + Err(ExecutorError::other(format!( + "uncontrolled error while executing `{key}`" + ))) + } + } + } + _ => Err(ExecutorError::other("not supported")), + } + } + + async fn perform_contract_put( + &mut self, + contract: ContractContainer, + state: WrappedState, + related_contracts: RelatedContracts<'_>, + ) -> Response { + let key = contract.key(); + if self.update_notifications.get(&key).is_none() { + return Ok(ContractResponse::PutResponse { key }.into()); + } + let params = contract.params(); + + if self.get_local_contract(key.id()).await.is_ok() { + // already existing contract, just try to merge states + return self + .perform_contract_update(key, UpdateData::State(state.into())) + .await; + } + + self.verify_and_store_contract(state.clone(), contract, related_contracts) + .await?; + + self.send_update_notification(&key, ¶ms, &state) + .await + .map_err(|_| { + ExecutorError::request(StdContractError::Put { + key: key.clone(), + cause: "failed while sending notifications".into(), + }) + })?; + Ok(ContractResponse::PutResponse { key }.into()) + } + + async fn perform_contract_update( + &mut self, + key: ContractKey, + update: UpdateData<'_>, + ) -> Response { + let parameters = { + self.state_store + .get_params(&key) + .await + .map_err(ExecutorError::other)? + .ok_or_else(|| { + RequestError::ContractError(StdContractError::Update { + cause: "missing contract parameters".into(), + key: key.clone(), + }) + })? + }; + + let current_state = self + .state_store + .get(&key) + .await + .map_err(ExecutorError::other)? + .clone(); + + let updates = vec![update]; + let new_state = self + .get_updated_state(¶meters, current_state, key.clone(), updates) + .await?; + + // in the network impl this would be sent over the network + let summary = self + .runtime + .summarize_state(&key, ¶meters, &new_state) + .map_err(ExecutorError::other)?; + self.send_update_notification(&key, ¶meters, &new_state) + .await?; + + #[cfg(any( + not(feature = "local-mode"), + feature = "network-mode", + all(not(feature = "local-mode"), not(feature = "network-mode")) + ))] + { + #[cfg(any( + all(feature = "local-mode", feature = "network-mode"), + all(not(feature = "local-mode"), not(feature = "network-mode")) + ))] + { + if self.mode == OperationMode::Local { + return Ok(ContractResponse::UpdateResponse { key, summary }.into()); + } + } + // notify peers with deltas from summary in network + let request = UpdateContract { + key: key.clone(), + new_state, + }; + let _op: operations::update::UpdateResult = self.op_request(request).await?; + } + + Ok(ContractResponse::UpdateResponse { key, summary }.into()) + } + + /// Attempts to update the state with the provided updates. + /// If there were no updates, it will return the current state. + async fn attempt_state_update( + &mut self, + parameters: &Parameters<'_>, + current_state: &WrappedState, + key: &ContractKey, + updates: &[UpdateData<'_>], + ) -> Result>, ExecutorError> { + let update_modification = + match self + .runtime + .update_state(key, parameters, current_state, updates) + { + Ok(result) => result, + Err(err) => { + return Err(ExecutorError::execution( + err, + Some(InnerOpError::Upsert(key.clone())), + )) + } + }; + let UpdateModification { + new_state, related, .. + } = update_modification; + let Some(new_state) = new_state else { + if related.is_empty() { + // no updates were made, just return old state + return Ok(Either::Left(current_state.clone())); + } else { + return Ok(Either::Right(related)); + } + }; + let new_state = WrappedState::new(new_state.into_bytes()); + self.state_store + .update(key, new_state.clone()) + .await + .map_err(ExecutorError::other)?; + Ok(Either::Left(new_state)) + } + + /// Given a contract and a series of delta updates, it will try to perform an update + /// to the contract state and return the new state. If it fails to update the state, + /// it will return an error. + /// + /// If there are missing updates for related contracts, it will try to fetch them from the network. + async fn get_updated_state( + &mut self, + parameters: &Parameters<'_>, + current_state: WrappedState, + key: ContractKey, + mut updates: Vec>, + ) -> Result { + let new_state = { + let start = Instant::now(); + loop { + let state_update_res = self + .attempt_state_update(parameters, ¤t_state, &key, &updates) + .await?; + let missing = match state_update_res { + Either::Left(new_state) => { + self.state_store + .update(&key, new_state.clone()) + .await + .map_err(ExecutorError::other)?; + break new_state; + } + Either::Right(missing) => missing, + }; + // some required contracts are missing + let required_contracts = missing.len() + 1; + for RelatedContract { + contract_instance_id: id, + mode, + } in missing + { + match self.state_store.get(&id.into()).await { + Ok(state) => { + // in this case we are already subscribed to and are updating this contract, + // we can try first with the existing value + updates.push(UpdateData::RelatedState { + related_to: id, + state: state.into(), + }); + } + #[cfg(any( + all(not(feature = "local-mode"), not(feature = "network-mode")), + all(feature = "local-mode", feature = "network-mode") + ))] + Err(StateStoreError::MissingContract(_)) + if self.mode == OperationMode::Network => + { + let state = match self.local_state_or_from_network(&id).await? { + Either::Left(state) => state, + Either::Right(GetResult { state, contract }) => { + let Some(contract) = contract else { + return Err(ExecutorError::request( + RequestError::ContractError(StdContractError::Get { + key, + cause: "missing-contract".into(), + }), + )); + }; + self.verify_and_store_contract( + state.clone(), + contract, + RelatedContracts::default(), + ) + .await?; + state + } + }; + updates.push(UpdateData::State(state.into())); + match mode { + RelatedMode::StateOnce => {} + RelatedMode::StateThenSubscribe => { + self.subscribe(id.into()).await?; + } + } + } + #[cfg(all(not(feature = "local-mode"), feature = "network-mode"))] + Err(StateStoreError::MissingContract(_)) => { + let state = match self.local_state_or_from_network(&id).await? { + Either::Left(state) => current_state, + Either::Right(GetResult { state, contract }) => { + self.verify_and_store_contract( + current_state.clone(), + contract, + RelatedContracts::default(), + ) + .await?; + current_state + } + }; + updates.push(UpdateData::State(current_state.into())); + match mode { + RelatedMode::StateOnce => {} + RelatedMode::StateThenSubscribe => { + self.subscribe(id.into()).await?; + } + } + } + Err(other_err) => { + let _ = mode; + return Err(ExecutorError::other(other_err)); + } + } + } + if updates.len() + 1 /* includes the original contract being updated update */ >= required_contracts + { + // try running again with all the related contracts retrieved + continue; + } else if start.elapsed() > Duration::from_secs(10) { + /* make this timeout configurable, and anyway should be controlled globally*/ + return Err(RequestError::Timeout.into()); + } + } + }; + Ok(new_state) + } + + async fn perform_contract_get(&mut self, contract: bool, key: ContractKey) -> Response { + let mut got_contract = None; + + #[cfg(any( + all(not(feature = "local-mode"), not(feature = "network-mode")), + all(feature = "local-mode", feature = "network-mode"), + ))] + { + if contract && self.mode == OperationMode::Local { + let Some(contract) = self.get_contract_locally(&key).await? else { + return Err(ExecutorError::request(RequestError::from( + StdContractError::Get { + key: key.clone(), + cause: "Missing contract or parameters".into(), + }, + ))); + }; + got_contract = Some(contract); + } else if contract { + got_contract = self.get_contract_from_network(key.clone()).await?; + } + } + + #[cfg(all(feature = "local-mode", not(feature = "network-mode")))] + if contract { + let Some(contract) = self + .get_contract_locally(&key) + .await + .map_err(Either::Left)? + else { + return Err(ExecutorError::request(RequestError::from( + StdContractError::Get { + key: key.clone(), + cause: "Missing contract or parameters".into(), + }, + ))); + }; + got_contract = Some(contract); + } + + #[cfg(all(feature = "network-mode", not(feature = "local-mode")))] + if contract { + if let Ok(Some(contract)) = self.get_contract_locally(&key).await { + got_contract = Some(contract); + } else { + got_contract = self.get_contract_from_network(key.clone()).await?; + } + } + + match self.state_store.get(&key).await { + Ok(state) => Ok(ContractResponse::GetResponse { + key, + contract: got_contract, + state, + } + .into()), + Err(StateStoreError::MissingContract(_)) => Err(ExecutorError::request( + RequestError::from(StdContractError::Get { + key, + cause: "Missing contract state".into(), + }), + )), + Err(err) => Err(ExecutorError::request(RequestError::from( + StdContractError::Get { + key, + cause: format!("{err}").into(), + }, + ))), + } + } + + async fn get_local_contract( + &self, + id: &ContractInstanceId, + ) -> Result, Either, DynError>> { + let Ok(contract) = self.state_store.get(&(*id).into()).await else { + return Err(Either::Right( + StdContractError::MissingRelated { key: *id }.into(), + )); + }; + let state: &[u8] = unsafe { + // Safety: this is fine since this will never scape this scope + std::mem::transmute::<&[u8], &'_ [u8]>(contract.as_ref()) + }; + Ok(State::from(state)) + } + + async fn verify_and_store_contract<'a>( + &mut self, + state: WrappedState, + trying_container: ContractContainer, + mut related_contracts: RelatedContracts<'a>, + ) -> Result<(), ExecutorError> { + let key = trying_container.key(); + let params = trying_container.params(); + + const DEPENDENCY_CYCLE_LIMIT_GUARD: usize = 100; + let mut iterations = 0; + + let original_key = key.clone(); + let original_state = state.clone(); + let original_params = params.clone(); + let mut trying_key = key; + let mut trying_state = state; + let mut trying_params = params; + let mut trying_contract = Some(trying_container); + + while iterations < DEPENDENCY_CYCLE_LIMIT_GUARD { + if let Some(contract) = trying_contract.take() { + self.runtime + .contract_store + .store_contract(contract) + .map_err(ExecutorError::other)?; + } + + let result = self + .runtime + .validate_state( + &trying_key, + &trying_params, + &trying_state, + &related_contracts, + ) + .map_err(|err| { + let _ = self.runtime.contract_store.remove_contract(&trying_key); + ExecutorError::other(err) + })?; + + let is_valid = match result { + ValidateResult::Valid => true, + ValidateResult::Invalid => false, + ValidateResult::RequestRelated(related) => { + iterations += 1; + related_contracts.missing(related); + for (id, related) in related_contracts.update() { + if related.is_none() { + #[cfg(all(feature = "local-mode", not(feature = "network-mode")))] + { + let current_state = self.get_local_contract(id).await?; + *related = Some(current_state); + } + + #[cfg(any( + all(not(feature = "local-mode"), not(feature = "network-mode")), + all(feature = "local-mode", feature = "network-mode"), + all(not(feature = "local-mode"), feature = "network-mode") + ))] + { + match self.local_state_or_from_network(id).await? { + Either::Left(state) => { + *related = Some(state.into()); + } + Either::Right(result) => { + let Some(contract) = result.contract else { + return Err(ExecutorError::request( + RequestError::ContractError( + StdContractError::Get { + key: (*id).into(), + cause: "missing-contract".into(), + }, + ), + )); + }; + trying_key = (*id).into(); + trying_params = contract.params(); + trying_state = result.state; + trying_contract = Some(contract); + continue; + } + } + } + } + } + continue; + } + }; + + if !is_valid { + return Err(ExecutorError::request(StdContractError::Put { + key: trying_key.clone(), + cause: "not valid".into(), + })); + } + + self.state_store + .store( + trying_key.clone(), + trying_state.clone(), + trying_params.clone(), + ) + .await + .map_err(ExecutorError::other)?; + if trying_key != original_key { + trying_key = original_key.clone(); + trying_params = original_params.clone(); + trying_state = original_state.clone(); + continue; + } + break; + } + if iterations == DEPENDENCY_CYCLE_LIMIT_GUARD { + return Err(ExecutorError::request(StdContractError::MissingRelated { + key: *original_key.id(), + })); + } + Ok(()) + } + + async fn send_update_notification<'a>( + &mut self, + key: &ContractKey, + params: &Parameters<'a>, + new_state: &WrappedState, + ) -> Result<(), ExecutorError> { + tracing::debug!(contract = %key, "notify of contract update"); + if let Some(notifiers) = self.update_notifications.get_mut(key) { + let summaries = self.subscriber_summaries.get_mut(key).unwrap(); + // in general there should be less than 32 failures + let mut failures = Vec::with_capacity(32); + for (peer_key, notifier) in notifiers.iter() { + let peer_summary = summaries.get_mut(peer_key).unwrap(); + let update = match peer_summary { + Some(summary) => self + .runtime + .get_state_delta(key, params, new_state, &*summary) + .map_err(|err| { + tracing::error!("{err}"); + ExecutorError::execution(err, Some(InnerOpError::Upsert(key.clone()))) + })? + .to_owned() + .into(), + None => UpdateData::State(State::from(new_state.as_ref()).into_owned()), + }; + if let Err(err) = notifier.send(Ok(ContractResponse::UpdateNotification { + key: key.clone(), + update, + } + .into())) + { + failures.push(*peer_key); + tracing::error!(cli_id = %peer_key, "{err}"); + } else { + tracing::debug!(cli_id = %peer_key, contract = %key, "notified of update"); + } + } + if !failures.is_empty() { + notifiers.retain(|(c, _)| !failures.contains(c)); + } + } + Ok(()) + } + + async fn get_contract_locally( + &self, + key: &ContractKey, + ) -> Result, ExecutorError> { + let Some(parameters) = self + .state_store + .get_params(key) + .await + .map_err(ExecutorError::other)? + else { + return Ok(None); + }; + let Some(contract) = self.runtime.contract_store.fetch_contract(key, ¶meters) else { + return Ok(None); + }; + Ok(Some(contract)) + } +} + +#[cfg(any( + not(feature = "local-mode"), + feature = "network-mode", + all(not(feature = "local-mode"), not(feature = "network-mode")) +))] +impl Executor { + async fn subscribe(&mut self, key: ContractKey) -> Result<(), ExecutorError> { + #[cfg(any( + all(not(feature = "local-mode"), not(feature = "network-mode")), + all(feature = "local-mode", feature = "network-mode") + ))] + { + if self.mode == OperationMode::Local { + return Ok(()); + } + } + let request = SubscribeContract { key }; + let _sub: operations::subscribe::SubscribeResult = self.op_request(request).await?; + Ok(()) + } + + #[inline] + async fn local_state_or_from_network( + &mut self, + id: &ContractInstanceId, + ) -> Result, ExecutorError> { + if let Ok(contract) = self.state_store.get(&(*id).into()).await { + return Ok(Either::Left(contract)); + }; + let request: GetContract = GetContract { + key: (*id).into(), + fetch_contract: true, + }; + let get_result: operations::get::GetResult = self.op_request(request).await?; + Ok(Either::Right(get_result)) + } + + async fn get_contract_from_network( + &mut self, + key: ContractKey, + ) -> Result, ExecutorError> { + loop { + if let Ok(Some(contract)) = self.get_contract_locally(&key).await { + break Ok(Some(contract)); + } else { + #[cfg(any( + all(not(feature = "local-mode"), not(feature = "network-mode")), + all(feature = "local-mode", feature = "network-mode") + ))] + { + if self.mode == OperationMode::Local { + return Err(ExecutorError::request(RequestError::ContractError( + StdContractError::MissingRelated { key: *key.id() }, + ))); + } + } + match self + .local_state_or_from_network(&key.clone().into()) + .await? + { + Either::Right(GetResult { state, contract }) => { + let Some(contract) = contract else { + return Err(ExecutorError::request(RequestError::ContractError( + StdContractError::Get { + key, + cause: "missing-contract".into(), + }, + ))); + }; + self.verify_and_store_contract( + state, + contract.clone(), + RelatedContracts::default(), + ) + .await?; + break Ok(Some(contract)); + } + Either::Left(_state) => continue, + } + } + } + } +} diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 8716e213e..ae217e604 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -1,9 +1,7 @@ -#![allow(unused)] // FIXME: remove this use std::collections::BTreeMap; use std::hash::Hash; -use std::marker::PhantomData; use std::sync::atomic::{AtomicU64, Ordering::SeqCst}; -use std::time::{Duration, Instant}; +use std::time::Duration; use freenet_stdlib::client_api::{ClientError, ClientRequest, HostResponse}; use freenet_stdlib::prelude::*; @@ -12,24 +10,24 @@ use serde::{Deserialize, Serialize}; use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; use super::executor::{ExecutorHalve, ExecutorToEventLoopChannel}; +use super::ExecutorError; use super::{ executor::{ContractExecutor, Executor}, ContractError, }; use crate::client_events::HostResult; use crate::message::Transaction; -use crate::{client_events::ClientId, node::PeerCliConfig, runtime::Runtime, DynError}; +use crate::ring::PeerKeyLocation; +use crate::{client_events::ClientId, node::PeerCliConfig, wasm_runtime::Runtime, DynError}; -pub(crate) struct ClientResponses(UnboundedReceiver<(ClientId, HostResult)>); +pub(crate) struct ClientResponsesReceiver(UnboundedReceiver<(ClientId, HostResult)>); -impl ClientResponses { - pub fn channel() -> (Self, ClientResponsesSender) { - let (tx, rx) = mpsc::unbounded_channel(); - (Self(rx), ClientResponsesSender(tx)) - } +pub fn client_responses_channel() -> (ClientResponsesReceiver, ClientResponsesSender) { + let (tx, rx) = mpsc::unbounded_channel(); + (ClientResponsesReceiver(rx), ClientResponsesSender(tx)) } -impl std::ops::Deref for ClientResponses { +impl std::ops::Deref for ClientResponsesReceiver { type Target = UnboundedReceiver<(ClientId, HostResult)>; fn deref(&self) -> &Self::Target { @@ -37,7 +35,7 @@ impl std::ops::Deref for ClientResponses { } } -impl std::ops::DerefMut for ClientResponses { +impl std::ops::DerefMut for ClientResponsesReceiver { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } @@ -98,8 +96,7 @@ impl ContractHandler for NetworkContractHandler { Self: Sized + 'static, { async { - let mut executor = Executor::from_config(config).await?; - executor.event_loop_channel(executor_request_sender); + let executor = Executor::from_config(config, Some(executor_request_sender)).await?; Ok(Self { executor, channel }) } .boxed() @@ -137,14 +134,14 @@ impl ContractHandler for NetworkContractHandler { fn build( channel: ContractHandlerChannel, - _executor_request_sender: ExecutorToEventLoopChannel, + executor_request_sender: ExecutorToEventLoopChannel, identifier: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { async move { - let executor = Executor::new_mock(&identifier).await?; + let executor = Executor::new_mock(&identifier, executor_request_sender).await?; Ok(Self { executor, channel }) } .boxed() @@ -178,24 +175,6 @@ impl ContractHandler for NetworkContractHandler { #[derive(Eq)] pub(crate) struct EventId { id: u64, - client_id: Option, - transaction: Option, -} - -impl EventId { - pub fn client_id(&self) -> Option { - self.client_id - } - - pub fn transaction(&self) -> Option { - self.transaction - } - - // FIXME: this should be used somewhere to inform than an event is pending - // a transaction resolution - pub fn with_transaction(&mut self, transaction: Transaction) { - self.transaction = Some(transaction); - } } impl PartialEq for EventId { @@ -213,46 +192,71 @@ impl Hash for EventId { /// A bidirectional channel which keeps track of the initiator half /// and sends the corresponding response to the listener of the operation. pub(crate) struct ContractHandlerChannel { - rx: mpsc::UnboundedReceiver, - tx: mpsc::UnboundedSender, - queue: BTreeMap, - _halve: PhantomData, + end: End, } -pub(crate) struct ContractHandlerHalve; -pub(crate) struct SenderHalve; +pub(crate) struct ContractHandlerHalve { + event_receiver: mpsc::UnboundedReceiver, + waiting_response: BTreeMap>, +} + +pub(crate) struct SenderHalve { + event_sender: mpsc::UnboundedSender, + wait_for_res_tx: mpsc::Sender<(ClientId, Transaction)>, +} + +/// Communicates that a client is waiting for a transaction resolution +/// to continue processing this event. +pub(crate) struct WaitingResolution { + wait_for_res_rx: mpsc::Receiver<(ClientId, Transaction)>, +} mod sealed { - use super::{ContractHandlerHalve, SenderHalve}; + use super::{ContractHandlerHalve, SenderHalve, WaitingResolution}; pub(crate) trait ChannelHalve {} impl ChannelHalve for ContractHandlerHalve {} impl ChannelHalve for SenderHalve {} + impl ChannelHalve for WaitingResolution {} } pub(crate) fn contract_handler_channel() -> ( ContractHandlerChannel, ContractHandlerChannel, + ContractHandlerChannel, ) { - let (notification_tx, notification_channel) = mpsc::unbounded_channel(); - let (ch_tx, ch_listener) = mpsc::unbounded_channel(); + let (event_sender, event_receiver) = mpsc::unbounded_channel(); + let (wait_for_res_tx, wait_for_res_rx) = mpsc::channel(10); ( ContractHandlerChannel { - rx: notification_channel, - tx: ch_tx, - queue: BTreeMap::new(), - _halve: PhantomData, + end: SenderHalve { + event_sender, + wait_for_res_tx, + }, + }, + ContractHandlerChannel { + end: ContractHandlerHalve { + event_receiver, + waiting_response: BTreeMap::new(), + }, }, ContractHandlerChannel { - rx: ch_listener, - tx: notification_tx, - queue: BTreeMap::new(), - _halve: PhantomData, + end: WaitingResolution { wait_for_res_rx }, }, ) } static EV_ID: AtomicU64 = AtomicU64::new(0); +impl ContractHandlerChannel { + pub async fn recv_from_client_event(&mut self) -> Result<(ClientId, Transaction), DynError> { + self.end + .wait_for_res_rx + .recv() + .await + .ok_or_else(|| "channel dropped".into()) + } +} + impl ContractHandlerChannel { // TODO: the timeout should be derived from whatever is the worst // case we are willing to accept for waiting out for an event; @@ -263,66 +267,55 @@ impl ContractHandlerChannel { /// Send an event to the contract handler and receive a response event if successful. pub async fn send_to_handler( - &mut self, + &self, ev: ContractHandlerEvent, - client_id: Option, ) -> Result { let id = EV_ID.fetch_add(1, SeqCst); - self.tx - .send(InternalCHEvent { ev, id, client_id }) + let (result, result_receiver) = tokio::sync::oneshot::channel(); + self.end + .event_sender + .send(InternalCHEvent { ev, id, result }) .map_err(|err| ContractError::ChannelDropped(Box::new(err.0.ev)))?; - if let Some(handler) = self.queue.remove(&id) { - Ok(handler) - } else { - let started_op = Instant::now(); - loop { - if started_op.elapsed() > Self::CH_EV_RESPONSE_TIME_OUT { - break Err(ContractError::NoEvHandlerResponse); - } - while let Some(msg) = self.rx.recv().await { - if msg.id == id { - return Ok(msg.ev); - } else { - self.queue.insert(id, msg.ev); // should never be duplicates - } - } - tokio::time::sleep(Duration::from_nanos(100)).await; - } + match tokio::time::timeout(Self::CH_EV_RESPONSE_TIME_OUT, result_receiver).await { + Ok(Ok((_, res))) => Ok(res), + Ok(Err(_)) | Err(_) => Err(ContractError::NoEvHandlerResponse), } } - pub async fn recv_from_handler(&mut self) -> EventId { - todo!() + pub async fn waiting_for_transaction( + &self, + transaction: Transaction, + client_id: ClientId, + ) -> Result<(), ContractError> { + self.end + .wait_for_res_tx + .send((client_id, transaction)) + .await + .map_err(|_| ContractError::NoEvHandlerResponse) } } impl ContractHandlerChannel { pub async fn send_to_sender( - &self, + &mut self, id: EventId, ev: ContractHandlerEvent, ) -> Result<(), ContractError> { - self.tx - .send(InternalCHEvent { - ev, - id: id.id, - client_id: id.client_id, - }) - .map_err(|err| ContractError::ChannelDropped(Box::new(err.0.ev))) + if let Some(response) = self.end.waiting_response.remove(&id.id) { + response + .send((id, ev)) + .map_err(|_| ContractError::NoEvHandlerResponse) + } else { + Err(ContractError::NoEvHandlerResponse) + } } pub async fn recv_from_sender( &mut self, ) -> Result<(EventId, ContractHandlerEvent), ContractError> { - if let Some(msg) = self.rx.recv().await { - return Ok(( - EventId { - id: msg.id, - client_id: msg.client_id, - transaction: None, - }, - msg.ev, - )); + if let Some(InternalCHEvent { ev, id, result }) = self.end.event_receiver.recv().await { + self.end.waiting_response.insert(id, result); + return Ok((EventId { id }, ev)); } Err(ContractError::NoEvHandlerResponse) } @@ -337,46 +330,49 @@ pub(crate) struct StoreResponse { struct InternalCHEvent { ev: ContractHandlerEvent, id: u64, - client_id: Option, + // client_id: Option, + result: tokio::sync::oneshot::Sender<(EventId, ContractHandlerEvent)>, } #[derive(Debug)] pub(crate) enum ContractHandlerEvent { - /// Try to push/put a new value into the contract. + /// Try to push/put a new value into the contract PutQuery { key: ContractKey, state: WrappedState, related_contracts: RelatedContracts<'static>, - parameters: Option>, + contract: Option, }, - /// The response to a push query. + /// The response to a push query PutResponse { - new_value: Result, + new_value: Result, }, - /// Fetch a supposedly existing contract value in this node, and optionally the contract itself. + /// Fetch a supposedly existing contract value in this node, and optionally the contract itself GetQuery { key: ContractKey, fetch_contract: bool, }, - /// The response to a FetchQuery event + /// The response to a get query event GetResponse { key: ContractKey, - response: Result, + response: Result, + }, + /// Subscribe to a contract. + Subscribe { key: ContractKey }, + /// The response to a subscribe event + SubscribeResponse { + key: ContractKey, + /// If successful, returns the peer to which it subscribed to for updates. + response: Result, }, - /// Store a contract in the local store. - Cache(ContractContainer), - /// Result of a caching operation. - CacheResult(Result<(), ContractError>), } impl std::fmt::Display for ContractHandlerEvent { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - ContractHandlerEvent::PutQuery { - key, parameters, .. - } => { - if let Some(params) = parameters { - write!(f, "put query {{ {key}, params: {:?} }}", params.as_ref()) + ContractHandlerEvent::PutQuery { key, contract, .. } => { + if let Some(contract) = contract { + write!(f, "put query {{ {key}, params: {:?} }}", contract.params()) } else { write!(f, "put query {{ {key} }}") } @@ -403,12 +399,23 @@ impl std::fmt::Display for ContractHandlerEvent { write!(f, "get query failed {{ {key} }}",) } }, - ContractHandlerEvent::Cache(container) => { - write!(f, "caching {{ {} }}", container.key()) - } - ContractHandlerEvent::CacheResult(r) => { - write!(f, "caching result {{ {} }}", r.is_ok()) + ContractHandlerEvent::Subscribe { key } => { + write!(f, "subscribe {{ {key} }}") } + ContractHandlerEvent::SubscribeResponse { key, response } => match response { + Ok(_) => { + write!(f, "subscribe response {{ {key} }}",) + } + Err(_) => { + write!(f, "subscribe failed {{ {key} }}",) + } + }, + // ContractHandlerEvent::Cache(container) => { + // write!(f, "caching {{ {} }}", container.key()) + // } + // ContractHandlerEvent::CacheResult(r) => { + // write!(f, "caching result {{ {} }}", r.is_ok()) + // } } } } @@ -424,38 +431,134 @@ pub mod test { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn channel_test() -> Result<(), anyhow::Error> { - let (mut send_halve, mut rcv_halve) = contract_handler_channel(); + let (send_halve, mut rcv_halve, _) = contract_handler_channel(); let contract = ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( Arc::new(ContractCode::from(vec![0, 1, 2, 3])), Parameters::from(vec![4, 5]), ))); - let contract_cp = contract.clone(); let h = GlobalExecutor::spawn(async move { send_halve - .send_to_handler(ContractHandlerEvent::Cache(contract_cp), None) + .send_to_handler(ContractHandlerEvent::PutQuery { + key: contract.key(), + state: vec![6, 7, 8].into(), + related_contracts: RelatedContracts::default(), + contract: Some(contract), + }) .await }); let (id, ev) = tokio::time::timeout(Duration::from_millis(100), rcv_halve.recv_from_sender()) .await??; - let ContractHandlerEvent::Cache(contract) = ev else { + let ContractHandlerEvent::PutQuery { state, .. } = ev else { anyhow::bail!("invalid event"); }; - assert_eq!(contract.data(), vec![0, 1, 2, 3]); + assert_eq!(state.as_ref(), &[6, 7, 8]); tokio::time::timeout( Duration::from_millis(100), - rcv_halve.send_to_sender(id, ContractHandlerEvent::Cache(contract)), + rcv_halve.send_to_sender( + id, + ContractHandlerEvent::PutResponse { + new_value: Ok(vec![0, 7].into()), + }, + ), ) .await??; - let ContractHandlerEvent::Cache(contract) = h.await?? else { + let ContractHandlerEvent::PutResponse { new_value } = h.await?? else { anyhow::bail!("invalid event!"); }; - assert_eq!(contract.data(), vec![0, 1, 2, 3]); + let new_value = new_value.map_err(|e| anyhow::anyhow!(e))?; + assert_eq!(new_value.as_ref(), &[0, 7]); + + Ok(()) + } +} + +pub(super) mod in_memory { + use crate::client_events::ClientId; + use freenet_stdlib::client_api::{ClientError, ClientRequest, HostResponse}; + use futures::{future::BoxFuture, FutureExt}; + use tokio::sync::mpsc::UnboundedSender; + + use super::{ + super::{ + executor::{ExecutorHalve, ExecutorToEventLoopChannel}, + Executor, MockRuntime, + }, + ContractHandler, ContractHandlerChannel, ContractHandlerHalve, + }; + use crate::DynError; + + pub(crate) struct MemoryContractHandler { + channel: ContractHandlerChannel, + runtime: Executor, + } + + impl MemoryContractHandler { + pub async fn new( + channel: ContractHandlerChannel, + executor_request_sender: ExecutorToEventLoopChannel, + identifier: &str, + ) -> Self { + MemoryContractHandler { + channel, + runtime: Executor::new_mock(identifier, executor_request_sender) + .await + .unwrap(), + } + } + } + + impl ContractHandler for MemoryContractHandler { + type Builder = String; + type ContractExecutor = Executor; + + fn build( + channel: ContractHandlerChannel, + executor_request_sender: ExecutorToEventLoopChannel, + identifier: Self::Builder, + ) -> BoxFuture<'static, Result> + where + Self: Sized + 'static, + { + async move { + Ok(MemoryContractHandler::new(channel, executor_request_sender, &identifier).await) + } + .boxed() + } + + fn channel(&mut self) -> &mut ContractHandlerChannel { + &mut self.channel + } + + fn handle_request<'a, 's: 'a>( + &'s mut self, + _req: ClientRequest<'a>, + _client_id: ClientId, + _updates: Option>>, + ) -> BoxFuture<'static, Result> { + unreachable!() + } + + fn executor(&mut self) -> &mut Self::ContractExecutor { + &mut self.runtime + } + } + #[test] + fn serialization() -> Result<(), anyhow::Error> { + use freenet_stdlib::prelude::WrappedContract; + let bytes = crate::util::test::random_bytes_1kb(); + let mut gen = arbitrary::Unstructured::new(&bytes); + let contract: WrappedContract = gen.arbitrary()?; + + let serialized = bincode::serialize(&contract)?; + let deser: WrappedContract = bincode::deserialize(&serialized)?; + assert_eq!(deser.code(), contract.code()); + assert_eq!(deser.key(), contract.key()); Ok(()) } } diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs deleted file mode 100644 index f832b5ccd..000000000 --- a/crates/core/src/contract/in_memory.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::{client_events::ClientId, runtime::ContractStore}; -use freenet_stdlib::client_api::{ClientError, ClientRequest, HostResponse}; -use futures::{future::BoxFuture, FutureExt}; -use tokio::sync::mpsc::UnboundedSender; - -use super::{ - executor::{ExecutorHalve, ExecutorToEventLoopChannel}, - handler::{ContractHandler, ContractHandlerChannel, ContractHandlerHalve}, - Executor, -}; -use crate::DynError; - -pub(crate) struct MockRuntime { - pub contract_store: ContractStore, -} - -pub(crate) struct MemoryContractHandler { - channel: ContractHandlerChannel, - runtime: Executor, -} - -impl MemoryContractHandler { - pub async fn new( - channel: ContractHandlerChannel, - identifier: &str, - ) -> Self { - MemoryContractHandler { - channel, - runtime: Executor::new_mock(identifier).await.unwrap(), - } - } -} - -impl ContractHandler for MemoryContractHandler { - type Builder = String; - type ContractExecutor = Executor; - - fn build( - channel: ContractHandlerChannel, - _executor_request_sender: ExecutorToEventLoopChannel, - identifier: Self::Builder, - ) -> BoxFuture<'static, Result> - where - Self: Sized + 'static, - { - async move { Ok(MemoryContractHandler::new(channel, &identifier).await) }.boxed() - } - - fn channel(&mut self) -> &mut ContractHandlerChannel { - &mut self.channel - } - - fn handle_request<'a, 's: 'a>( - &'s mut self, - _req: ClientRequest<'a>, - _client_id: ClientId, - _updates: Option>>, - ) -> BoxFuture<'static, Result> { - unreachable!() - } - - fn executor(&mut self) -> &mut Self::ContractExecutor { - &mut self.runtime - } -} - -#[test] -fn serialization() -> Result<(), anyhow::Error> { - use freenet_stdlib::prelude::WrappedContract; - let bytes = crate::util::test::random_bytes_1kb(); - let mut gen = arbitrary::Unstructured::new(&bytes); - let contract: WrappedContract = gen.arbitrary()?; - - let serialized = bincode::serialize(&contract)?; - let deser: WrappedContract = bincode::deserialize(&serialized)?; - assert_eq!(deser.code(), contract.code()); - assert_eq!(deser.key(), contract.key()); - Ok(()) -} diff --git a/crates/core/src/contract/storages/rocks_db.rs b/crates/core/src/contract/storages/rocks_db.rs index 1e5f94e2d..489be5303 100644 --- a/crates/core/src/contract/storages/rocks_db.rs +++ b/crates/core/src/contract/storages/rocks_db.rs @@ -4,7 +4,7 @@ use freenet_stdlib::prelude::*; use rocksdb::{Options, DB}; use crate::contract::ContractKey; -use crate::runtime::StateStorage; +use crate::wasm_runtime::StateStorage; pub struct RocksDb(DB); diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index 2e05f2820..1fa2ca05c 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -8,7 +8,7 @@ use sqlx::{ use crate::{ contract::ContractKey, - runtime::{ContractError, StateStorage, StateStoreError}, + wasm_runtime::{ContractError, StateStorage, StateStoreError}, }; async fn create_contracts_table(pool: &SqlitePool) -> Result<(), SqlDbError> { diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 63c7fd215..0eeb93872 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,19 +1,33 @@ +/// Clients events related logic and type definitions. pub(crate) mod client_events; +/// Peer node configuration. pub mod config; +/// Handling of contracts and delegates functionality. mod contract; +/// Generated messages from the flatbuffers schema for the network monitor. pub mod generated; +/// Network messages for transactions. mod message; +/// Node configuration, implementations and execution (entry points for the binaries). mod node; +/// Network operation/transaction state machines. mod operations; +/// Resource usage tracking. mod resources; +/// Ring connections and routing. mod ring; +/// Router implementation. mod router; -mod runtime; +/// Local server used to communicate with the peer core. #[cfg(feature = "websocket")] pub mod server; +/// Local network topology management. mod topology; +/// Tracing and loging infrastructure. Includes our custom event log register. Tracing collectors, etc. mod tracing; pub mod util; +/// WASM code execution runtime, tailored for the contract and delegate APIs. +mod wasm_runtime; type DynError = Box; @@ -37,5 +51,5 @@ pub mod dev_tool { InitPeerNode, InterProcessConnManager, NodeConfig, PeerCliConfig, PeerId, }; pub use ring::Location; - pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; + pub use wasm_runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 1e1e13c59..aa78fc410 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -1,6 +1,7 @@ //! Network messaging between peers. use std::{ + borrow::Cow, fmt::Display, time::{Duration, SystemTime}, }; @@ -29,7 +30,7 @@ pub(crate) use sealed_msg_type::{TransactionType, TransactionTypeId}; /// /// A transaction may span different messages sent across the network. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy)] -pub struct Transaction { +pub(crate) struct Transaction { id: Ulid, } @@ -257,7 +258,7 @@ pub(crate) enum NodeEvent { #[serde(skip)] Error(ConnectionError), Disconnect { - cause: Option, + cause: Option>, }, } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 5fe682e26..3829d1558 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -5,7 +5,8 @@ //! Node comes with different underlying implementations that can be used upon construction. //! Those implementations are: //! - libp2p: all the connection is handled by libp2p. -//! - In memory: a simplifying node used for emulation purposes mainly. +//! - in-memory: a simplifying node used for emulation purposes mainly. +//! - inter-process: similar to in-memory, but can be rana cross multiple processes, closer to the real p2p impl use std::{ fmt::Display, @@ -28,18 +29,17 @@ use crate::{ config::Config, config::GlobalExecutor, contract::{ - ClientResponses, ClientResponsesSender, ContractError, ExecutorToEventLoopChannel, - NetworkContractHandler, NetworkEventListenerHalve, OperationMode, + Callback, ClientResponsesReceiver, ClientResponsesSender, ContractError, + ExecutorToEventLoopChannel, NetworkContractHandler, OperationMode, }, message::{NetMessage, NodeEvent, Transaction, TransactionType}, operations::{ - connect::{self, ConnectMsg, ConnectOp}, + connect::{self, ConnectOp}, get, put, subscribe, OpEnum, OpError, OpOutcome, }, ring::{Location, PeerKeyLocation}, router::{RouteEvent, RouteOutcome}, tracing::{EventRegister, NetEventLog, NetEventRegister}, - util::ExponentialBackoff, DynError, }; @@ -212,6 +212,10 @@ impl NodeConfig { Ok(Node(node)) } + pub fn is_gateway(&self) -> bool { + self.local_ip.is_some() && self.local_port.is_some() && self.location.is_some() + } + /// Returns all specified gateways for this peer. Returns an error if the peer is not a gateway /// and no gateways are specified. fn get_gateways(&self) -> Result, anyhow::Error> { @@ -300,37 +304,11 @@ impl InitPeerNode { } } -async fn join_ring_request( - backoff: Option, - peer_key: PeerId, - gateway: &PeerKeyLocation, - op_manager: &OpManager, - conn_manager: &mut CM, -) -> Result<(), OpError> -where - CM: NetworkBridge + Send, -{ - let tx_id = Transaction::new::(); - let mut op = - connect::initial_request(peer_key, *gateway, op_manager.ring.max_hops_to_live, tx_id); - if let Some(mut backoff) = backoff { - // backoff to retry later in case it failed - tracing::warn!("Performing a new join, attempt {}", backoff.retries() + 1); - if backoff.sleep().await.is_none() { - tracing::error!("Max number of retries reached"); - return Err(OpError::MaxRetriesExceeded(tx_id, tx_id.transaction_type())); - } - op.backoff = Some(backoff); - } - connect::connect_request(tx_id, op_manager, conn_manager, op).await?; - Ok(()) -} - /// Process client events. async fn client_event_handling( op_manager: Arc, mut client_events: ClientEv, - mut client_responses: ClientResponses, + mut client_responses: ClientResponsesReceiver, node_controller: tokio::sync::mpsc::Sender, ) where ClientEv: ClientEventsProxy + Send + 'static, @@ -378,6 +356,7 @@ async fn process_open_request(request: OpenRequest<'static>, op_manager: Arc match ops { @@ -397,7 +376,11 @@ async fn process_open_request(request: OpenRequest<'static>, op_manager: Arc, op_manager: Arc { - // FIXME: DO THIS + // FIXME: perform updates tracing::debug!( this_peer = %op_manager.ring.peer_key, "Received update from user event", @@ -421,7 +404,11 @@ async fn process_open_request(request: OpenRequest<'static>, op_manager: Arc, op_manager: Arc { tracing::info!(%key, "Trying to subscribe to a contract not present, requesting it first"); missing_contract = true; let get_op = get::start_op(key.clone(), true); - if let Err(error) = - get::request_get(&op_manager, get_op, Some(client_id)).await + if let Err(error) = get::request_get(&op_manager, get_op).await { tracing::error!(%key, %error, "Failed getting the contract while previously trying to subscribe; bailing"); break; @@ -502,7 +490,7 @@ async fn report_result( tx: Option, op_result: Result, OpError>, op_manager: &OpManager, - executor_callback: Option>, + executor_callback: Option>, client_req_handler_callback: Option<(ClientId, ClientResponsesSender)>, event_listener: &mut dyn NetEventRegister, ) { @@ -534,7 +522,7 @@ async fn report_result( event_listener .register_events(Either::Left(NetEventLog::route_event( op_res.id(), - op_manager, + &op_manager.ring, &event, ))) .await; @@ -610,7 +598,7 @@ async fn process_message( op_manager: Arc, mut conn_manager: CB, mut event_listener: Box, - executor_callback: Option>, + executor_callback: Option>, client_req_handler_callback: Option, client_id: Option, ) where @@ -626,13 +614,9 @@ async fn process_message( match &msg { NetMessage::Connect(op) => { // log_handling_msg!("join", op.id(), op_manager); - let op_result = handle_op_request::( - &op_manager, - &mut conn_manager, - op, - client_id, - ) - .await; + let op_result = + handle_op_request::(&op_manager, &mut conn_manager, op) + .await; handle_op_not_available!(op_result); break report_result( tx, @@ -646,13 +630,8 @@ async fn process_message( } NetMessage::Put(op) => { // log_handling_msg!("put", *op.id(), op_manager); - let op_result = handle_op_request::( - &op_manager, - &mut conn_manager, - op, - client_id, - ) - .await; + let op_result = + handle_op_request::(&op_manager, &mut conn_manager, op).await; handle_op_not_available!(op_result); break report_result( tx, @@ -666,13 +645,8 @@ async fn process_message( } NetMessage::Get(op) => { // log_handling_msg!("get", op.id(), op_manager); - let op_result = handle_op_request::( - &op_manager, - &mut conn_manager, - op, - client_id, - ) - .await; + let op_result = + handle_op_request::(&op_manager, &mut conn_manager, op).await; handle_op_not_available!(op_result); break report_result( tx, @@ -690,7 +664,6 @@ async fn process_message( &op_manager, &mut conn_manager, op, - client_id, ) .await; handle_op_not_available!(op_result); @@ -709,32 +682,62 @@ async fn process_message( } } -async fn handle_cancelled_op( +async fn handle_aborted_op( tx: Transaction, - peer_key: PeerId, + this_peer: PeerId, op_manager: &OpManager, conn_manager: &mut CM, + gateways: &[PeerKeyLocation], ) -> Result<(), OpError> where CM: NetworkBridge + Send, { + use crate::util::IterExt; if let TransactionType::Connect = tx.transaction_type() { - // the attempt to join the network failed, this could be a fatal error since the node + // attempt to establish a connection failed, this could be a fatal error since the node // is useless without connecting to the network, we will retry with exponential backoff + // if necessary match op_manager.pop(&tx) { - Ok(Some(OpEnum::Connect(op))) if op.has_backoff() => { + // only keep attempting to connect if the node hasn't got enough connections yet + Ok(Some(OpEnum::Connect(op))) + if op.has_backoff() + && op_manager.ring.open_connections() < op_manager.ring.min_connections => + { let ConnectOp { gateway, backoff, .. } = *op; if let Some(gateway) = gateway { - let backoff = backoff.expect("infallible"); tracing::warn!("Retry connecting to gateway {}", gateway.peer); - join_ring_request(Some(backoff), peer_key, &gateway, op_manager, conn_manager) - .await?; + connect::join_ring_request( + backoff, + this_peer, + &gateway, + op_manager, + conn_manager, + ) + .await?; } } Ok(Some(OpEnum::Connect(_))) => { - return Err(OpError::MaxRetriesExceeded(tx, tx.transaction_type())); + // if no connections were achieved just fail + if op_manager.ring.open_connections() == 0 { + tracing::warn!("Retrying joining the ring with an other gateway"); + if let Some(gateway) = gateways + .iter() + .shuffle() + .next() + .filter(|p| p.peer != this_peer) + { + connect::join_ring_request( + None, + this_peer, + gateway, + op_manager, + conn_manager, + ) + .await? + } + } } _ => {} } diff --git a/crates/core/src/node/network_bridge.rs b/crates/core/src/node/network_bridge.rs index 19506ce5d..577de16f9 100644 --- a/crates/core/src/node/network_bridge.rs +++ b/crates/core/src/node/network_bridge.rs @@ -1,4 +1,4 @@ -//! Types and definitions to handle all socket communication for the peer nodes. +//! Types and definitions to handle all inter-peer communication. use std::ops::{Deref, DerefMut}; @@ -8,10 +8,7 @@ use serde::{Deserialize, Serialize}; use tokio::sync::mpsc::{self, Receiver, Sender}; use super::PeerId; -use crate::{ - client_events::ClientId, - message::{NetMessage, NodeEvent}, -}; +use crate::message::{NetMessage, NodeEvent}; pub(crate) mod in_memory; pub(crate) mod inter_process; @@ -86,41 +83,35 @@ impl Clone for ConnectionError { } } -pub(super) struct EventLoopNotifications( - Receiver), NodeEvent>>, -); - -impl EventLoopNotifications { - pub fn channel() -> (EventLoopNotifications, EventLoopNotificationsSender) { - let (notification_tx, notification_rx) = mpsc::channel(100); - ( - EventLoopNotifications(notification_rx), - EventLoopNotificationsSender(notification_tx), - ) - } +pub fn event_loop_notification_channel( +) -> (EventLoopNotificationsReceiver, EventLoopNotificationsSender) { + let (notification_tx, notification_rx) = mpsc::channel(100); + ( + EventLoopNotificationsReceiver(notification_rx), + EventLoopNotificationsSender(notification_tx), + ) } +pub(super) struct EventLoopNotificationsReceiver(Receiver>); -impl Deref for EventLoopNotifications { - type Target = Receiver), NodeEvent>>; +impl Deref for EventLoopNotificationsReceiver { + type Target = Receiver>; fn deref(&self) -> &Self::Target { &self.0 } } -impl DerefMut for EventLoopNotifications { +impl DerefMut for EventLoopNotificationsReceiver { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } } #[derive(Clone)] -pub(crate) struct EventLoopNotificationsSender( - Sender), NodeEvent>>, -); +pub(crate) struct EventLoopNotificationsSender(Sender>); impl Deref for EventLoopNotificationsSender { - type Target = Sender), NodeEvent>>; + type Target = Sender>; fn deref(&self) -> &Self::Target { &self.0 diff --git a/crates/core/src/node/network_bridge/p2p_protoc.rs b/crates/core/src/node/network_bridge/p2p_protoc.rs index 1cda18763..1521346fe 100644 --- a/crates/core/src/node/network_bridge/p2p_protoc.rs +++ b/crates/core/src/node/network_bridge/p2p_protoc.rs @@ -35,20 +35,21 @@ use tokio::sync::mpsc::{self, Receiver, Sender}; use tracing::Instrument; use unsigned_varint::codec::UviBytes; -use super::{ConnectionError, EventLoopNotifications, NetworkBridge}; +use super::{ConnectionError, EventLoopNotificationsReceiver, NetworkBridge}; use crate::{ client_events::ClientId, config::{self, GlobalExecutor}, - contract::{ClientResponsesSender, ExecutorToEventLoopChannel, NetworkEventListenerHalve}, - message::{NetMessage, NodeEvent, Transaction, TransactionType}, + contract::{ + ClientResponsesSender, ContractHandlerChannel, ExecutorToEventLoopChannel, + NetworkEventListenerHalve, WaitingResolution, + }, + message::{NetMessage, NodeEvent, Transaction}, node::{ - handle_cancelled_op, join_ring_request, process_message, InitPeerNode, NetEventRegister, - NodeConfig, OpManager, PeerId as FreenetPeerId, + handle_aborted_op, process_message, InitPeerNode, NetEventRegister, NodeConfig, OpManager, + PeerId as FreenetPeerId, }, - operations::OpError, ring::PeerKeyLocation, tracing::NetEventLog, - util::IterExt, }; /// The default maximum size for a varint length-delimited packet. @@ -270,8 +271,9 @@ impl P2pConnManager { pub async fn run_event_listener( mut self, op_manager: Arc, - mut notification_channel: EventLoopNotifications, - mut executor_channel: ExecutorToEventLoopChannel, + mut client_wait_for_transaction: ContractHandlerChannel, + mut notification_channel: EventLoopNotificationsReceiver, + mut executor_listener: ExecutorToEventLoopChannel, cli_response_sender: ClientResponsesSender, mut node_controller: Receiver, ) -> Result<(), anyhow::Error> { @@ -291,7 +293,7 @@ impl P2pConnManager { let network_msg = self.swarm.select_next_some().map(|event| match event { SwarmEvent::Behaviour(NetEvent::Freenet(msg)) => { tracing::debug!("Message inbound: {:?}", msg); - Ok(Left((*msg, None))) + Ok(Left(*msg)) } SwarmEvent::ConnectionClosed { peer_id, .. } => { Ok(Right(ConnMngrActions::ConnectionClosed { @@ -362,7 +364,7 @@ impl P2pConnManager { let notification_msg = notification_channel.0.recv().map(|m| match m { None => Ok(Right(ClosedChannel)), - Some(Left((msg, cli_id))) => Ok(Left((msg, cli_id))), + Some(Left(msg)) => Ok(Left(msg)), Some(Right(action)) => Ok(Right(NodeAction(action))), }); @@ -386,59 +388,39 @@ impl P2pConnManager { Ok(Right(ClosedChannel)) } } - event_id = op_manager.recv_from_handler() => { - if let Some((client_id, transaction)) = event_id.client_id().zip(event_id.transaction()) { - tx_to_client.insert(transaction, client_id); - } + event_id = client_wait_for_transaction.recv_from_client_event() => { + let (client_id, transaction) = event_id.map_err(|err| anyhow::anyhow!(err))?; + tx_to_client.insert(transaction, client_id); continue; } - id = executor_channel.transaction_from_executor() => { + id = executor_listener.transaction_from_executor() => { + let id = id.map_err(|err| anyhow::anyhow!(err))?; pending_from_executor.insert(id); continue; } }; match msg { - Ok(Left((msg, client_id))) => { + Ok(Left(msg)) => { let cb = self.bridge.clone(); match msg { NetMessage::Aborted(tx) => { - let tx_type = tx.transaction_type(); - let res = handle_cancelled_op( + handle_aborted_op( tx, op_manager.ring.peer_key, &op_manager, &mut self.bridge, + &self.gateways, ) - .await; - match res { - Err(OpError::MaxRetriesExceeded(_, _)) - if tx_type == TransactionType::Connect - && self.public_addr.is_none() /* FIXME: this should be not a gateway instead */ => - { - tracing::warn!("Retrying joining the ring with an other peer"); - let gateway = self.gateways.iter().shuffle().next().unwrap(); - join_ring_request( - None, - op_manager.ring.peer_key, - gateway, - &op_manager, - &mut self.bridge, - ) - .await? - } - Err(err) => return Err(anyhow::anyhow!(err)), - Ok(_) => {} - } + .await?; continue; } msg => { let executor_callback = pending_from_executor .remove(msg.id()) - .then(|| executor_channel.clone()); + .then(|| executor_listener.callback()); let pending_client_req = tx_to_client.get(msg.id()).copied(); let client_req_handler_callback = if pending_client_req.is_some() { - debug_assert!(client_id.is_none()); Some(cli_response_sender.clone()) } else { None @@ -458,7 +440,7 @@ impl P2pConnManager { self.event_listener.trait_clone(), executor_callback, client_req_handler_callback, - client_id, + pending_client_req, ) .instrument(span), ); diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 4a10d06a5..589f0e0e7 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -2,19 +2,17 @@ use std::{cmp::Reverse, collections::BTreeSet, sync::Arc, time::Duration}; use dashmap::{DashMap, DashSet}; use either::Either; -use tokio::sync::Mutex; use tracing::Instrument; use crate::{ config::GlobalExecutor, contract::{ContractError, ContractHandlerChannel, ContractHandlerEvent, SenderHalve}, - dev_tool::ClientId, message::{NetMessage, Transaction, TransactionType}, operations::{ connect::ConnectOp, get::GetOp, put::PutOp, subscribe::SubscribeOp, update::UpdateOp, OpEnum, OpError, }, - ring::{LiveTransactionTracker, PeerKeyLocation, Ring}, + ring::{LiveTransactionTracker, Ring}, }; use super::{network_bridge::EventLoopNotificationsSender, NetEventRegister, NodeConfig, PeerId}; @@ -53,24 +51,22 @@ pub(crate) struct OpManager { pub ring: Arc, ops: Arc, to_event_listener: EventLoopNotificationsSender, - // todo: remove the need for a mutex here if possible - ch_outbound: Mutex>, + pub ch_outbound: ContractHandlerChannel, new_transactions: tokio::sync::mpsc::Sender, } impl OpManager { pub(super) fn new( notification_channel: EventLoopNotificationsSender, - contract_handler: ContractHandlerChannel, + ch_outbound: ContractHandlerChannel, config: &NodeConfig, - gateways: &[PeerKeyLocation], event_register: ER, ) -> Result { let ring = Ring::new( config, - gateways, notification_channel.clone(), event_register.clone(), + config.is_gateway(), )?; let ops = Arc::new(Ops::default()); @@ -95,7 +91,7 @@ impl OpManager { ring, ops, to_event_listener: notification_channel, - ch_outbound: Mutex::new(contract_handler), + ch_outbound, new_transactions, }) } @@ -105,16 +101,11 @@ impl OpManager { /// /// Useful when transitioning between states that do not require any network communication /// with other nodes, like intermediate states before returning. - pub async fn notify_op_change( - &self, - msg: NetMessage, - op: OpEnum, - client_id: Option, - ) -> Result<(), OpError> { + pub async fn notify_op_change(&self, msg: NetMessage, op: OpEnum) -> Result<(), OpError> { // push back the state to the stack self.push(*msg.id(), op).await?; self.to_event_listener - .send(Either::Left((msg, client_id))) + .send(Either::Left(msg)) .await .map_err(Into::into) } @@ -123,18 +114,8 @@ impl OpManager { pub async fn notify_contract_handler( &self, msg: ContractHandlerEvent, - client_id: Option, ) -> Result { - self.ch_outbound - .lock() - .await - .send_to_handler(msg, client_id) - .await - } - - pub async fn recv_from_handler(&self) -> crate::contract::EventId { - // self.ch_outbound.lock().await.recv_from_handler(); - todo!() + self.ch_outbound.send_to_handler(msg).await } pub async fn push(&self, id: Transaction, op: OpEnum) -> Result<(), OpError> { diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index 0419f2e06..06b247348 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -12,20 +12,22 @@ use libp2p::{ use tracing::Instrument; use super::{ - client_event_handling, join_ring_request, - network_bridge::{p2p_protoc::P2pConnManager, EventLoopNotifications}, + client_event_handling, + network_bridge::{ + event_loop_notification_channel, p2p_protoc::P2pConnManager, EventLoopNotificationsReceiver, + }, NetEventRegister, PeerId as FreenetPeerId, }; use crate::{ client_events::{combinator::ClientEventsCombinator, BoxedClient}, config::{self, GlobalExecutor}, contract::{ - self, ClientResponsesSender, ContractHandler, ExecutorToEventLoopChannel, - NetworkEventListenerHalve, + self, ClientResponsesSender, ContractHandler, ContractHandlerChannel, + ExecutorToEventLoopChannel, NetworkEventListenerHalve, WaitingResolution, }, message::NodeEvent, node::NodeConfig, - util::IterExt, + operations::connect, }; use super::OpManager; @@ -33,12 +35,13 @@ use super::OpManager; pub(super) struct NodeP2P { pub(crate) peer_key: FreenetPeerId, pub(crate) op_manager: Arc, - notification_channel: EventLoopNotifications, + notification_channel: EventLoopNotificationsReceiver, + client_wait_for_transaction: ContractHandlerChannel, pub(super) conn_manager: P2pConnManager, - is_gateway: bool, executor_listener: ExecutorToEventLoopChannel, cli_response_sender: ClientResponsesSender, node_controller: tokio::sync::mpsc::Receiver, + is_gateway: bool, } impl NodeP2P { @@ -48,25 +51,19 @@ impl NodeP2P { self.conn_manager.listen_on()?; } - if !self.is_gateway { - if let Some(gateway) = self.conn_manager.gateways.iter().shuffle().take(1).next() { - join_ring_request( - None, - self.peer_key, - gateway, - &self.op_manager, - &mut self.conn_manager.bridge, - ) - .await?; - } else { - anyhow::bail!("requires at least one gateway"); - } - } + connect::initial_join_procedure( + &self.op_manager, + &mut self.conn_manager.bridge, + self.peer_key, + &self.conn_manager.gateways, + ) + .await?; // start the p2p event loop self.conn_manager .run_event_listener( self.op_manager.clone(), + self.client_wait_for_transaction, self.notification_channel, self.executor_listener, self.cli_response_sender, @@ -76,7 +73,7 @@ impl NodeP2P { } pub(crate) async fn build( - builder: NodeConfig, + config: NodeConfig, private_key: Keypair, clients: [BoxedClient; CLIENTS], event_register: ER, @@ -86,18 +83,16 @@ impl NodeP2P { CH: ContractHandler + Send + 'static, ER: NetEventRegister + Clone, { - let peer_key = builder.peer_id; - let gateways = builder.get_gateways()?; + let peer_key = config.peer_id; - let (notification_channel, notification_tx) = EventLoopNotifications::channel(); - let (ch_outbound, ch_inbound) = contract::contract_handler_channel(); - let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); + let (notification_channel, notification_tx) = event_loop_notification_channel(); + let (ch_outbound, ch_inbound, wait_for_event) = contract::contract_handler_channel(); + let (client_responses, cli_response_sender) = contract::client_responses_channel(); let op_manager = Arc::new(OpManager::new( notification_tx, ch_outbound, - &builder, - &gateways, + &config, event_register.clone(), )?); let (executor_listener, executor_sender) = contract::executor_channel(op_manager.clone()); @@ -109,7 +104,7 @@ impl NodeP2P { let transport = Self::config_transport(&private_key)?; P2pConnManager::build( transport, - &builder, + &config, op_manager.clone(), event_register, private_key, @@ -137,11 +132,12 @@ impl NodeP2P { peer_key, conn_manager, notification_channel, + client_wait_for_transaction: wait_for_event, op_manager, - is_gateway: builder.location.is_some(), executor_listener, cli_response_sender, node_controller: node_controller_rx, + is_gateway: config.is_gateway(), }) } diff --git a/crates/core/src/node/testing_impl.rs b/crates/core/src/node/testing_impl.rs index a4999f803..a2627224f 100644 --- a/crates/core/src/node/testing_impl.rs +++ b/crates/core/src/node/testing_impl.rs @@ -21,9 +21,13 @@ use crate::tracing::CombinedRegister; use crate::{ client_events::test::{MemoryEventsGen, RandomEventGenerator}, config::GlobalExecutor, - contract, - message::{NetMessage, NodeEvent}, + contract::{ + self, ContractHandlerChannel, ExecutorToEventLoopChannel, NetworkEventListenerHalve, + WaitingResolution, + }, + message::{NetMessage, NodeEvent, Transaction}, node::{InitPeerNode, NetEventRegister, NodeConfig}, + operations::connect, ring::{Distance, Location, PeerKeyLocation}, tracing::TestEventListener, }; @@ -33,7 +37,9 @@ mod inter_process; pub use self::inter_process::SimPeer; -use super::{network_bridge::EventLoopNotifications, ConnectionError, NetworkBridge, PeerId}; +use super::{ + network_bridge::EventLoopNotificationsReceiver, ConnectionError, NetworkBridge, PeerId, +}; pub fn get_free_port() -> Result { let mut port; @@ -121,7 +127,7 @@ impl<'a> From<&'a str> for NodeLabel { #[cfg(test)] #[derive(Clone)] pub(crate) struct NodeSpecification { - pub owned_contracts: Vec<(ContractContainer, WrappedState)>, + pub owned_contracts: Vec<(ContractContainer, WrappedState, Option)>, pub events_to_generate: HashMap>, pub contract_subscribers: HashMap>, } @@ -277,7 +283,7 @@ pub(super) struct Builder { contract_handler_name: String, add_noise: bool, event_register: ER, - contracts: Vec<(ContractContainer, WrappedState)>, + contracts: Vec<(ContractContainer, WrappedState, Option)>, contract_subscribers: HashMap>, } @@ -318,7 +324,7 @@ pub struct SimNetwork { rnd_if_htl_above: usize, max_connections: usize, min_connections: usize, - init_backoff: Duration, + start_backoff: Duration, add_noise: bool, } @@ -350,7 +356,7 @@ impl SimNetwork { rnd_if_htl_above, max_connections, min_connections, - init_backoff: Duration::from_millis(1), + start_backoff: Duration::from_millis(1), add_noise: false, }; net.config_gateways(gateways).await; @@ -361,7 +367,7 @@ impl SimNetwork { impl SimNetwork { pub fn with_start_backoff(&mut self, value: Duration) { - self.init_backoff = value; + self.start_backoff = value; } /// Simulates network random behaviour, like messages arriving delayed or out of order, throttling etc. @@ -536,7 +542,7 @@ impl SimNetwork { let node_task = async move { node.run_node(user_events, span).await }; GlobalExecutor::spawn(node_task); - tokio::time::sleep(self.init_backoff).await; + tokio::time::sleep(self.start_backoff).await; } self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); } @@ -569,7 +575,7 @@ impl SimNetwork { let handle = GlobalExecutor::spawn(node_task); peers.push(handle); - tokio::time::sleep(self.init_backoff).await; + tokio::time::sleep(self.start_backoff).await; } self.labels.sort_by(|(a, _), (b, _)| a.cmp(b)); peers @@ -621,19 +627,14 @@ impl SimNetwork { self.event_listener.is_connected(&self.labels[pos].1) } - pub fn has_put_contract( - &self, - peer: impl Into, - key: &ContractKey, - value: &WrappedState, - ) -> bool { + pub fn has_put_contract(&self, peer: impl Into, key: &ContractKey) -> bool { let peer = peer.into(); let pos = self .labels .binary_search_by(|(label, _)| label.cmp(&peer)) .expect("peer not found"); self.event_listener - .has_put_contract(&self.labels[pos].1, key, value) + .has_put_contract(&self.labels[pos].1, key) } pub fn has_got_contract(&self, peer: impl Into, key: &ContractKey) -> bool { @@ -775,7 +776,7 @@ impl SimNetwork { tracing::info!("Number of simulated nodes: {num_nodes}"); - let missing_percent = (num_nodes - missing.len()) as f64 / num_nodes as f64; + let missing_percent = 1.0 - ((num_nodes - missing.len()) as f64 / num_nodes as f64); if missing_percent > (percent + 0.01/* 1% error tolerance */) { missing.sort(); let show_max = missing.len().min(100); @@ -870,7 +871,7 @@ impl std::fmt::Debug for SimNetwork { .field("rnd_if_htl_above", &self.rnd_if_htl_above) .field("max_connections", &self.max_connections) .field("min_connections", &self.min_connections) - .field("init_backoff", &self.init_backoff) + .field("init_backoff", &self.start_backoff) .field("add_noise", &self.add_noise) .finish() } @@ -969,15 +970,16 @@ where UsrEv: ClientEventsProxy + Send + 'static, { peer_key: PeerId, - is_gateway: bool, - gateways: Vec, parent_span: Option, op_manager: Arc, conn_manager: NB, /// Set on creation, taken on run user_events: Option, - notification_channel: EventLoopNotifications, + notification_channel: EventLoopNotificationsReceiver, event_register: Box, + gateways: Vec, + executor_listener: ExecutorToEventLoopChannel, + client_wait_for_transaction: ContractHandlerChannel, } async fn run_node(mut config: RunnerConfig) -> Result<(), anyhow::Error> @@ -985,22 +987,14 @@ where NB: NetworkBridge + NetworkBridgeExt, UsrEv: ClientEventsProxy + Send + 'static, { - use crate::util::IterExt; - if !config.is_gateway { - if let Some(gateway) = config.gateways.iter().shuffle().take(1).next() { - super::join_ring_request( - None, - config.peer_key, - gateway, - &config.op_manager, - &mut config.conn_manager, - ) - .await?; - } else { - anyhow::bail!("requires at least one gateway"); - } - } - let (client_responses, cli_response_sender) = contract::ClientResponses::channel(); + connect::initial_join_procedure( + &config.op_manager, + &mut config.conn_manager, + config.peer_key, + &config.gateways, + ) + .await?; + let (client_responses, cli_response_sender) = contract::client_responses_channel(); let span = { config .parent_span @@ -1037,17 +1031,18 @@ where /// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. async fn run_event_listener( - _client_responses: contract::ClientResponsesSender, + cli_response_sender: contract::ClientResponsesSender, mut node_controller_rx: tokio::sync::mpsc::Receiver, RunnerConfig { peer_key, - is_gateway, gateways, parent_span, op_manager, mut conn_manager, mut notification_channel, mut event_register, + mut executor_listener, + client_wait_for_transaction: mut wait_for_event, .. }: RunnerConfig, ) -> Result<(), anyhow::Error> @@ -1055,13 +1050,14 @@ where NB: NetworkBridge + NetworkBridgeExt, UsrEv: ClientEventsProxy + Send + 'static, { - use crate::util::IterExt; + let mut pending_from_executor = HashSet::new(); + let mut tx_to_client: HashMap = HashMap::new(); loop { let msg = tokio::select! { msg = conn_manager.recv() => { msg.map(Either::Left) } msg = notification_channel.recv() => { if let Some(msg) = msg { - Ok(msg.map_left(|(msg, _cli_id)| msg)) + Ok(msg) } else { anyhow::bail!("notification channel shutdown, fatal error"); } @@ -1073,34 +1069,23 @@ where anyhow::bail!("node controller channel shutdown, fatal error"); } } + event_id = wait_for_event.recv_from_client_event() => { + if let Ok((client_id, transaction)) = event_id { + tx_to_client.insert(transaction, client_id); + } + continue; + } + id = executor_listener.transaction_from_executor() => { + if let Ok(res) = id { + pending_from_executor.insert(res); + } + continue; + } }; if let Ok(Either::Left(NetMessage::Aborted(tx))) = msg { - let tx_type = tx.transaction_type(); - let res = - super::handle_cancelled_op(tx, peer_key, &op_manager, &mut conn_manager).await; - match res { - Err(crate::operations::OpError::MaxRetriesExceeded(_, _)) - if tx_type == crate::message::TransactionType::Connect && !is_gateway => - { - tracing::warn!("Retrying joining the ring with an other peer"); - if let Some(gateway) = gateways.iter().shuffle().next() { - super::join_ring_request( - None, - peer_key, - gateway, - &op_manager, - &mut conn_manager, - ) - .await? - } else { - anyhow::bail!("requires at least one gateway"); - } - } - Err(err) => return Err(anyhow::anyhow!(err)), - Ok(_) => {} - } - continue; + super::handle_aborted_op(tx, peer_key, &op_manager, &mut conn_manager, &gateways) + .await?; } let msg = match msg { @@ -1164,14 +1149,24 @@ where }) }; + let executor_callback = pending_from_executor + .remove(msg.id()) + .then(|| executor_listener.callback()); + let pending_client_req = tx_to_client.get(msg.id()).copied(); + let client_req_handler_callback = if pending_client_req.is_some() { + Some(cli_response_sender.clone()) + } else { + None + }; + let msg = super::process_message( msg, op_manager, conn_manager.clone(), event_listener, - None, - None, - None, + executor_callback, + client_req_handler_callback, + pending_client_req, ) .instrument(span); GlobalExecutor::spawn(msg); diff --git a/crates/core/src/node/testing_impl/in_memory.rs b/crates/core/src/node/testing_impl/in_memory.rs index da8699c20..885bb0e5d 100644 --- a/crates/core/src/node/testing_impl/in_memory.rs +++ b/crates/core/src/node/testing_impl/in_memory.rs @@ -8,7 +8,7 @@ use crate::{ config::GlobalExecutor, contract::{self, executor_channel, ContractHandler, MemoryContractHandler}, node::{ - network_bridge::{in_memory::MemoryConnManager, EventLoopNotifications}, + network_bridge::{event_loop_notification_channel, in_memory::MemoryConnManager}, op_state_manager::OpManager, NetEventRegister, NetworkBridge, }, @@ -28,21 +28,19 @@ impl Builder { ER: NetEventRegister + Clone, { let gateways = self.config.get_gateways()?; - let is_gateway = self.config.local_ip.zip(self.config.local_port).is_some(); - let (notification_channel, notification_tx) = EventLoopNotifications::channel(); - let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); + let (notification_channel, notification_tx) = event_loop_notification_channel(); + let (ops_ch_channel, ch_channel, wait_for_event) = contract::contract_handler_channel(); let _guard = parent_span.enter(); let op_manager = Arc::new(OpManager::new( notification_tx, ops_ch_channel, &self.config, - &gateways, self.event_register.clone(), )?); std::mem::drop(_guard); - let (_executor_listener, executor_sender) = executor_channel(op_manager.clone()); + let (executor_listener, executor_sender) = executor_channel(op_manager.clone()); let contract_handler = MemoryContractHandler::build(ch_channel, executor_sender, self.contract_handler_name) .await @@ -61,7 +59,6 @@ impl Builder { ); let mut config = super::RunnerConfig { peer_key: self.peer_key, - is_gateway, gateways, parent_span: Some(parent_span), op_manager, @@ -69,6 +66,8 @@ impl Builder { user_events: Some(user_events), notification_channel, event_register: self.event_register.trait_clone(), + executor_listener, + client_wait_for_transaction: wait_for_event, }; config .append_contracts(self.contracts, self.contract_subscribers) @@ -79,7 +78,7 @@ impl Builder { #[cfg(test)] pub fn append_contracts( &mut self, - contracts: Vec<(ContractContainer, WrappedState)>, + contracts: Vec<(ContractContainer, WrappedState, Option)>, contract_subscribers: std::collections::HashMap>, ) { self.contracts.extend(contracts); @@ -94,33 +93,30 @@ where { async fn append_contracts( &mut self, - contracts: Vec<(ContractContainer, WrappedState)>, + contracts: Vec<(ContractContainer, WrappedState, Option)>, contract_subscribers: HashMap>, ) -> Result<(), anyhow::Error> { use crate::contract::ContractHandlerEvent; - for (contract, state) in contracts { + for (contract, state, subscription) in contracts { let key: ContractKey = contract.key(); - let parameters = contract.params(); self.op_manager - .notify_contract_handler(ContractHandlerEvent::Cache(contract.clone()), None) - .await?; - self.op_manager - .notify_contract_handler( - ContractHandlerEvent::PutQuery { - key: key.clone(), - state, - related_contracts: RelatedContracts::default(), - parameters: Some(parameters), - }, - None, - ) + .notify_contract_handler(ContractHandlerEvent::PutQuery { + key: key.clone(), + state, + related_contracts: RelatedContracts::default(), + contract: Some(contract), + }) .await?; tracing::debug!( "Appended contract {} to peer {}", key, self.op_manager.ring.peer_key ); - self.op_manager.ring.contract_cached(&key); + if let Some(subscription) = subscription { + self.op_manager + .ring + .add_subscription(key.clone(), subscription); + } if let Some(subscribers) = contract_subscribers.get(&key) { // add contract subscribers for subscriber in subscribers { diff --git a/crates/core/src/node/testing_impl/inter_process.rs b/crates/core/src/node/testing_impl/inter_process.rs index 4c115e2c1..efc4e5b1e 100644 --- a/crates/core/src/node/testing_impl/inter_process.rs +++ b/crates/core/src/node/testing_impl/inter_process.rs @@ -8,7 +8,7 @@ use crate::{ contract::{self, ContractHandler, MemoryContractHandler}, dev_tool::{ClientEventsProxy, NodeConfig}, node::{ - network_bridge::{inter_process::InterProcessConnManager, EventLoopNotifications}, + network_bridge::{event_loop_notification_channel, inter_process::InterProcessConnManager}, OpManager, }, tracing::{EventRegister, NetEventRegister}, @@ -52,19 +52,17 @@ impl SimPeer { ER: NetEventRegister + Send + Clone + 'static, { let gateways = self.config.get_gateways()?; - let is_gateway = self.config.local_ip.zip(self.config.local_port).is_some(); - let (notification_channel, notification_tx) = EventLoopNotifications::channel(); - let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); + let (notification_channel, notification_tx) = event_loop_notification_channel(); + let (ops_ch_channel, ch_channel, wait_for_event) = contract::contract_handler_channel(); let op_manager = Arc::new(OpManager::new( notification_tx, ops_ch_channel, &self.config, - &gateways, event_register.clone(), )?); - let (_executor_listener, executor_sender) = contract::executor_channel(op_manager.clone()); + let (executor_listener, executor_sender) = contract::executor_channel(op_manager.clone()); let contract_handler = MemoryContractHandler::build( ch_channel, executor_sender, @@ -83,13 +81,14 @@ impl SimPeer { let running_node = super::RunnerConfig { peer_key: self.config.peer_id, op_manager, - gateways, notification_channel, conn_manager, event_register: Box::new(event_register), - is_gateway, user_events: Some(event_generator), parent_span: None, + gateways, + executor_listener, + client_wait_for_transaction: wait_for_event, }; super::run_node(running_node).await } diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index cfa7d043d..6cd0a5858 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -7,11 +7,10 @@ use tokio::sync::mpsc::error::SendError; use crate::{ client_events::{ClientId, HostResult}, - contract::ContractError, + contract::{ContractError, ExecutorError}, message::{InnerMessage, NetMessage, Transaction, TransactionType}, node::{ConnectionError, NetworkBridge, OpManager, OpNotAvailable, PeerId}, ring::{Location, PeerKeyLocation, RingError}, - DynError, }; pub(crate) mod connect; @@ -36,7 +35,6 @@ pub(crate) async fn handle_op_request( op_manager: &OpManager, network_bridge: &mut NB, msg: &Op::Message, - client_id: Option, ) -> Result, OpError> where Op: Operation, @@ -47,12 +45,12 @@ where let result = { let OpInitialization { sender: s, op } = Op::load_or_init(op_manager, msg).await?; sender = s; - op.process_message(network_bridge, op_manager, msg, client_id) - .await + op.process_message(network_bridge, op_manager, msg).await }; handle_op_result(op_manager, network_bridge, result, tx, sender).await } +#[inline(always)] async fn handle_op_result( op_manager: &OpManager, network_bridge: &mut CB, @@ -154,6 +152,33 @@ impl OpEnum { } } +macro_rules! try_from_op_enum { + ($op_enum:path, $op_type:ty, $transaction_type:expr) => { + impl TryFrom for $op_type { + type Error = OpError; + + fn try_from(value: OpEnum) -> Result { + match value { + $op_enum(op) => Ok(op), + other => Err(OpError::IncorrectTxType( + $transaction_type, + other.id().transaction_type(), + )), + } + } + } + }; +} + +try_from_op_enum!(OpEnum::Put, put::PutOp, TransactionType::Put); +try_from_op_enum!(OpEnum::Get, get::GetOp, TransactionType::Get); +try_from_op_enum!( + OpEnum::Subscribe, + subscribe::SubscribeOp, + TransactionType::Subscribe +); +try_from_op_enum!(OpEnum::Update, update::UpdateOp, TransactionType::Update); + pub(crate) enum OpOutcome<'a> { /// An op which involves a contract completed successfully. ContractOpSuccess { @@ -187,7 +212,7 @@ pub(crate) enum OpError { #[error(transparent)] ContractError(#[from] ContractError), #[error(transparent)] - ExecutorError(DynError), + ExecutorError(#[from] ExecutorError), #[error("unexpected operation state")] UnexpectedOpState, @@ -201,7 +226,6 @@ pub(crate) enum OpError { }, #[error("failed notifying, channel closed")] NotificationError, - #[cfg(debug_assertions)] #[error("unspected transaction type, trying to get a {0:?} from a {1:?}")] IncorrectTxType(TransactionType, TransactionType), #[error("op not present: {0}")] @@ -211,7 +235,7 @@ pub(crate) enum OpError { #[error("op not available")] OpNotAvailable(#[from] OpNotAvailable), - // user for control flow + // used for control flow /// This is used as an early interrumpt of an op update when an op /// was sent throught the fast path back to the storage. #[error("early push of state into the op stack")] @@ -274,6 +298,6 @@ where conn_manager: &'a mut CB, op_manager: &'a OpManager, input: &'a Self::Message, - client_id: Option, + // client_id: Option, ) -> Pin> + Send + 'a>>; } diff --git a/crates/core/src/operations/connect.rs b/crates/core/src/operations/connect.rs index 45ac1d5ed..8afbaa588 100644 --- a/crates/core/src/operations/connect.rs +++ b/crates/core/src/operations/connect.rs @@ -6,7 +6,6 @@ use std::{collections::HashSet, time::Duration}; use super::{OpError, OpInitialization, OpOutcome, Operation, OperationResult}; use crate::{ - client_events::ClientId, message::{InnerMessage, NetMessage, Transaction}, node::{ConnectionError, NetworkBridge, OpManager, PeerId}, operations::OpEnum, @@ -120,7 +119,6 @@ impl Operation for ConnectOp { network_bridge: &'a mut NB, op_manager: &'a OpManager, input: &'a Self::Message, - _client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { let return_msg; @@ -233,7 +231,7 @@ impl Operation for ConnectOp { // todo: location should be based on your public IP let new_location = assigned_location.unwrap_or_else(Location::random); - let accepted_by = if op_manager.ring.should_accept(new_location) { + let accepted_by = if op_manager.ring.should_accept(new_location, joiner) { tracing::debug!(tx = %id, %joiner, "Accepting connection from"); HashSet::from_iter([*this_peer]) } else { @@ -313,10 +311,10 @@ impl Operation for ConnectOp { %hops_to_live, "Proxy connect request received to connect with peer", ); - if op_manager - .ring - .should_accept(joiner.location.ok_or(ConnectionError::LocationUnknown)?) - { + if op_manager.ring.should_accept( + joiner.location.ok_or(ConnectionError::LocationUnknown)?, + &joiner.peer, + ) { tracing::debug!(tx = %id, "Accepting proxy connection from {}", joiner.peer); accepted_by.insert(own_loc); } else { @@ -428,7 +426,7 @@ impl Operation for ConnectOp { op_manager.ring.update_location(Some(*your_location)); for other_peer in accepted_by.iter().filter(|pl| pl.peer != target.peer) { - let _ = propagate_oc_to_accepted_peers( + let _ = propagate_oc_to_responding_peers( network_bridge, op_manager, gateway, @@ -459,11 +457,7 @@ impl Operation for ConnectOp { backoff: self.backoff, }; op_manager - .notify_op_change( - NetMessage::Aborted(*id), - OpEnum::Connect(op.into()), - None, - ) + .notify_op_change(NetMessage::Aborted(*id), OpEnum::Connect(op.into())) .await?; return Err(OpError::StatePushed); } @@ -559,7 +553,7 @@ impl Operation for ConnectOp { new_state = None; } else { for peer in accepted_by { - propagate_oc_to_accepted_peers( + propagate_oc_to_responding_peers( network_bridge, op_manager, *sender, @@ -718,7 +712,7 @@ fn try_returning_proxy_connection( (new_state, return_msg) } -async fn propagate_oc_to_accepted_peers( +async fn propagate_oc_to_responding_peers( network_bridge: &mut NB, op_manager: &OpManager, sender: PeerKeyLocation, @@ -730,6 +724,7 @@ async fn propagate_oc_to_accepted_peers( other_peer .location .ok_or(ConnectionError::LocationUnknown)?, + &other_peer.peer, ) { tracing::info!(tx = %id, from = %sender.peer, to = %other_peer.peer, "Established connection"); network_bridge.add_connection(other_peer.peer).await?; @@ -801,7 +796,83 @@ impl ConnectState { } } -pub(crate) fn initial_request( +/// # Arguments +/// +/// - gateways: Inmutable list of known gateways. Passed when starting up the node. +/// After the initial connections through the gateways are established all other connections +/// (to gateways or regular peers) will be treated as regular connections. +/// +/// - is_gateway: Whether this peer is a gateway or not. +pub(crate) async fn initial_join_procedure( + op_manager: &OpManager, + conn_manager: &mut CM, + this_peer: PeerId, + gateways: &[PeerKeyLocation], +) -> Result<(), OpError> +where + CM: NetworkBridge + Send, +{ + use crate::util::IterExt; + let number_of_parallel_connections = { + let max_potential_conns_per_gw = op_manager.ring.max_hops_to_live; + // e.g. 10 gateways and htl 5 -> only need 2 connections in parallel + let needed_to_cover_max = gateways + .iter() + .filter(|conn| conn.peer != this_peer) + .count() + / max_potential_conns_per_gw; + needed_to_cover_max.max(1) + }; + tracing::info!( + "Attempting to connect to {} gateways in parallel", + number_of_parallel_connections + ); + for gateway in gateways + .iter() + .shuffle() + .filter(|conn| conn.peer != this_peer) + .take(number_of_parallel_connections) + { + join_ring_request(None, this_peer, gateway, op_manager, conn_manager).await?; + } + Ok(()) +} + +pub(crate) async fn join_ring_request( + backoff: Option, + peer_key: PeerId, + gateway: &PeerKeyLocation, + op_manager: &OpManager, + conn_manager: &mut CM, +) -> Result<(), OpError> +where + CM: NetworkBridge + Send, +{ + let tx_id = Transaction::new::(); + let mut op = initial_request(peer_key, *gateway, op_manager.ring.max_hops_to_live, tx_id); + if let Some(mut backoff) = backoff { + // backoff to retry later in case it failed + tracing::warn!("Performing a new join, attempt {}", backoff.retries() + 1); + if backoff.sleep().await.is_none() { + tracing::error!("Max number of retries reached"); + if op_manager.ring.open_connections() == 0 { + // only consider this a complete failure if no connections were established at all + // if connections where established the peer should incrementally acquire more over time + return Err(OpError::MaxRetriesExceeded(tx_id, tx_id.transaction_type())); + } else { + return Ok(()); + } + } + // on first run the backoff will be initialized at the `initial_request` function + // if the op was to fail and retried this function will be called with the previous backoff + // passed as an argument and advanced + op.backoff = Some(backoff); + } + connect_request(tx_id, op_manager, conn_manager, op).await?; + Ok(()) +} + +fn initial_request( this_peer: PeerId, gateway: PeerKeyLocation, max_hops_to_live: usize, @@ -831,7 +902,7 @@ pub(crate) fn initial_request( } /// Join ring routine, called upon performing a join operation for this node. -pub(crate) async fn connect_request( +async fn connect_request( tx: Transaction, op_manager: &OpManager, conn_bridge: &mut NB, diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 2419b14fb..d2c35e32a 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -5,14 +5,13 @@ use freenet_stdlib::prelude::*; use futures::future::BoxFuture; use futures::FutureExt; +use crate::contract::ExecutorError; use crate::{ - client_events::ClientId, contract::{ContractError, ContractHandlerEvent, StoreResponse}, message::{InnerMessage, NetMessage, Transaction}, node::{NetworkBridge, OpManager, PeerId}, operations::{OpInitialization, Operation}, ring::{Location, PeerKeyLocation, RingError}, - DynError, }; use super::{OpEnum, OpError, OpOutcome, OperationResult}; @@ -21,20 +20,115 @@ pub(crate) use self::messages::GetMsg; /// Maximum number of retries to get values. const MAX_RETRIES: usize = 10; -/// Maximum number of hops performed while trying to perform a get (a hop will be performed -/// when the current node cannot perform a get for whichever reason, eg. being out of the caching -/// distance for the contract) -const MAX_GET_RETRY_HOPS: usize = 1; -pub(crate) struct GetOp { - id: Transaction, - state: Option, - pub(super) result: Option, - stats: Option, +pub(crate) fn start_op(key: ContractKey, fetch_contract: bool) -> GetOp { + let contract_location = Location::from(&key); + let id = Transaction::new::(); + tracing::debug!(tx = %id, "Requesting get contract {key} @ loc({contract_location})"); + let state = Some(GetState::PrepareRequest { + key, + id, + fetch_contract, + }); + GetOp { + id, + state, + result: None, + stats: Some(GetStats { + contract_location, + next_peer: None, + transfer_time: None, + first_response_time: None, + step: Default::default(), + }), + } +} + +/// Request to get the current value from a contract. +pub(crate) async fn request_get(op_manager: &OpManager, get_op: GetOp) -> Result<(), OpError> { + let (target, id) = if let Some(GetState::PrepareRequest { key, id, .. }) = &get_op.state { + const EMPTY: &[PeerId] = &[]; + // the initial request must provide: + // - a location in the network where the contract resides + // - and the key of the contract value to get + ( + op_manager + .ring + .closest_potentially_caching(key, EMPTY) + .into_iter() + .next() + .ok_or(RingError::EmptyRing)?, + *id, + ) + } else { + return Err(OpError::UnexpectedOpState); + }; + tracing::debug!( + tx = %id, + target = %target.peer, + "Preparing get contract request", + ); + + match get_op.state { + Some(GetState::PrepareRequest { + fetch_contract, + key, + id, + .. + }) => { + let new_state = Some(GetState::AwaitingResponse { + retries: 0, + fetch_contract, + requester: None, + }); + + let msg = GetMsg::RequestGet { + id, + key, + target, + fetch_contract, + }; + + let op = GetOp { + id, + state: new_state, + result: None, + stats: get_op.stats.map(|mut s| { + s.next_peer = Some(target); + s + }), + }; + + op_manager + .notify_op_change(NetMessage::from(msg), OpEnum::Get(op)) + .await?; + } + _ => return Err(OpError::invalid_transition(get_op.id)), + } + Ok(()) +} + +enum GetState { + /// A new petition for a get op. + ReceivedRequest, + /// Preparing request for get op. + PrepareRequest { + key: ContractKey, + id: Transaction, + fetch_contract: bool, + }, + /// Awaiting response from petition. + AwaitingResponse { + /// If specified the peer waiting for the response upstream + requester: Option, + fetch_contract: bool, + retries: usize, + }, } struct GetStats { - caching_peer: Option, + /// Next peer in get path to be targeted + next_peer: Option, contract_location: Location, /// (start, end) first_response_time: Option<(Instant, Option)>, @@ -54,12 +148,35 @@ enum RecordingStats { Completed, } +pub(crate) struct GetResult { + pub state: WrappedState, + pub contract: Option, +} + +impl TryFrom for GetResult { + type Error = OpError; + + fn try_from(value: GetOp) -> Result { + match value.result { + Some(r) => Ok(r), + _ => Err(OpError::UnexpectedOpState), + } + } +} + +pub(crate) struct GetOp { + pub id: Transaction, + state: Option, + pub(super) result: Option, + stats: Option, +} + impl GetOp { pub(super) fn outcome(&self) -> OpOutcome { if let Some(( GetResult { state, contract }, GetStats { - caching_peer: Some(target_peer), + next_peer: Some(target_peer), contract_location, first_response_time: Some((response_start, Some(response_end))), transfer_time: Some((transfer_start, Some(transfer_end))), @@ -117,22 +234,6 @@ impl GetOp { } } -pub(crate) struct GetResult { - pub state: WrappedState, - pub contract: Option, -} - -impl TryFrom for GetResult { - type Error = OpError; - - fn try_from(value: GetOp) -> Result { - match value.result { - Some(r) => Ok(r), - _ => Err(OpError::UnexpectedOpState), - } - } -} - impl Operation for GetOp { type Message = GetMsg; type Result = GetResult; @@ -183,7 +284,6 @@ impl Operation for GetOp { conn_manager: &'a mut NB, op_manager: &'a OpManager, input: &'a Self::Message, - client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { let return_msg; @@ -207,7 +307,7 @@ impl Operation for GetOp { new_state = self.state; stats = Some(GetStats { contract_location: Location::from(key), - caching_peer: None, + next_peer: None, transfer_time: None, first_response_time: None, step: Default::default(), @@ -219,7 +319,8 @@ impl Operation for GetOp { target: *target, sender: own_loc, fetch_contract: *fetch_contract, - htl: MAX_GET_RETRY_HOPS, + htl: op_manager.ring.max_hops_to_live, + skip_list: vec![own_loc.peer], }); } GetMsg::SeekNode { @@ -229,22 +330,24 @@ impl Operation for GetOp { sender, target, htl, + skip_list, } => { let htl = *htl; let id = *id; let key: ContractKey = key.clone(); let fetch_contract = *fetch_contract; + let this_peer = *target; - let is_cached_contract = op_manager.ring.is_contract_cached(&key); + let is_cached_contract = op_manager.ring.is_subscribed_to_contract(&key); if let Some(s) = stats.as_mut() { - s.caching_peer = Some(*target); + s.next_peer = Some(this_peer); } if !is_cached_contract { tracing::warn!( tx = %id, %key, - this_peer = %target.peer, + this_peer = %this_peer.peer, "Contract not found while processing a get request", ); @@ -268,6 +371,7 @@ impl Operation for GetOp { }, sender: op_manager.ring.own_location(), target: *sender, // return to requester + skip_list: skip_list.clone(), }), None, stats, @@ -282,11 +386,13 @@ impl Operation for GetOp { tracing::warn!( tx = %id, %key, - target = %target.peer, + this_peer = %this_peer.peer, "No other peers found while trying getting contract", ); return Err(OpError::RingError(RingError::NoCachingPeers(key))); }; + let mut new_skip_list = skip_list.clone(); + new_skip_list.push(target.peer); continue_seeking( conn_manager, &new_target, @@ -294,27 +400,29 @@ impl Operation for GetOp { id, key, fetch_contract, - sender: *sender, + sender: this_peer, target: new_target, htl: new_htl, + skip_list: new_skip_list, }) .into(), ) .await?; return_msg = None; - new_state = None; + new_state = Some(GetState::AwaitingResponse { + requester: Some(*sender), + retries: 0, + fetch_contract, + }); } else if let ContractHandlerEvent::GetResponse { key: returned_key, response: value, } = op_manager - .notify_contract_handler( - ContractHandlerEvent::GetQuery { - key: key.clone(), - fetch_contract, - }, - client_id, - ) + .notify_contract_handler(ContractHandlerEvent::GetQuery { + key: key.clone(), + fetch_contract, + }) .await? { match check_contract_found( @@ -331,14 +439,34 @@ impl Operation for GetOp { tracing::debug!(tx = %id, "Contract {returned_key} found @ peer {}", target.peer); match self.state { - Some(GetState::AwaitingResponse { .. }) => { - tracing::debug!( - tx = %id, - "Completed operation, get response received for contract {key}" - ); - // Completed op - new_state = None; - return_msg = None; + Some(GetState::AwaitingResponse { requester, .. }) => { + if let Some(requester) = requester { + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); + new_state = None; + let value = match value { + Ok(res) => res, + Err(err) => { + tracing::error!(tx = %id, "error: {err}"); + return Err(OpError::ExecutorError(err)); + } + }; + return_msg = Some(GetMsg::ReturnGet { + id, + key, + value, + sender: *target, + target: requester, + skip_list: skip_list.clone(), + }); + } else { + tracing::debug!( + tx = %id, + "Completed operation, get response received for contract {key}" + ); + // Completed op + new_state = None; + return_msg = None; + } } Some(GetState::ReceivedRequest) => { tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); @@ -356,6 +484,7 @@ impl Operation for GetOp { value, sender: *target, target: *sender, + skip_list: skip_list.clone(), }); } _ => return Err(OpError::invalid_transition(self.id)), @@ -374,6 +503,7 @@ impl Operation for GetOp { }, sender, target, + skip_list, } => { let this_peer = target; tracing::warn!( @@ -387,18 +517,19 @@ impl Operation for GetOp { match self.state { Some(GetState::AwaitingResponse { - mut skip_list, - retries, fetch_contract, + retries, + requester, .. }) => { // todo: register in the stats for the outcome of the op that failed to get a response from this peer if retries < MAX_RETRIES { // no response received from this peer, so skip it in the next iteration - skip_list.push(target.peer); + let mut new_skip_list = skip_list.clone(); + new_skip_list.push(target.peer); if let Some(target) = op_manager .ring - .closest_potentially_caching(key, skip_list.as_slice()) + .closest_potentially_caching(key, new_skip_list.as_slice()) .into_iter() .next() { @@ -408,15 +539,16 @@ impl Operation for GetOp { target, sender: *this_peer, fetch_contract, - htl: MAX_GET_RETRY_HOPS, + htl: op_manager.ring.max_hops_to_live, + skip_list: new_skip_list.clone(), }); } else { return Err(RingError::NoCachingPeers(key.clone()).into()); } new_state = Some(GetState::AwaitingResponse { - skip_list, retries: retries + 1, fetch_contract, + requester, }); } else { tracing::error!( @@ -442,6 +574,7 @@ impl Operation for GetOp { }, sender: *sender, target: *target, + skip_list: skip_list.clone(), }); } _ => return Err(OpError::invalid_transition(self.id)), @@ -457,6 +590,7 @@ impl Operation for GetOp { }, sender, target, + skip_list, } => { let id = *id; let key = key.clone(); @@ -469,72 +603,49 @@ impl Operation for GetOp { ); // received a response with a contract value - if require_contract { - if let Some(contract) = &contract { - // store contract first - let res = op_manager - .notify_contract_handler( - ContractHandlerEvent::Cache(contract.clone()), - client_id, - ) - .await?; - match res { - ContractHandlerEvent::CacheResult(Ok(_)) => { - op_manager.ring.contract_cached(&key); - } - ContractHandlerEvent::CacheResult(Err(err)) => { - return Err(OpError::ContractError(err)); - } - _ => unreachable!(), - } - let key = contract.key(); - tracing::debug!(tx = %id, "Contract `{}` successfully cached", key); - } else { - // no contract, consider this like an error ignoring the incoming update value - tracing::warn!( - tx = %id, - "Contract not received from peer {} while required", - sender.peer - ); + if require_contract && contract.is_none() { + // no contract, consider this like an error ignoring the incoming update value + tracing::warn!( + tx = %id, + "Contract not received from peer {} while required", + sender.peer + ); - let op = GetOp { - id, - state: self.state, - result: None, - stats, - }; + let op = GetOp { + id, + state: self.state, + result: None, + stats, + }; - op_manager - .notify_op_change( - NetMessage::from(GetMsg::ReturnGet { - id, - key, - value: StoreResponse { - state: None, - contract: None, - }, - sender: *sender, - target: *target, - }), - OpEnum::Get(op), - None, - ) - .await?; - return Err(OpError::StatePushed); - } + let mut new_skip_list = skip_list.clone(); + new_skip_list.push(sender.peer); + op_manager + .notify_op_change( + NetMessage::from(GetMsg::ReturnGet { + id, + key, + value: StoreResponse { + state: None, + contract: None, + }, + sender: *sender, + target: *target, + skip_list: new_skip_list, + }), + OpEnum::Get(op), + ) + .await?; + return Err(OpError::StatePushed); } - let parameters = contract.as_ref().map(|c| c.params()); let res = op_manager - .notify_contract_handler( - ContractHandlerEvent::PutQuery { - key: key.clone(), - state: value.clone(), - related_contracts: RelatedContracts::default(), - parameters, - }, - client_id, - ) + .notify_contract_handler(ContractHandlerEvent::PutQuery { + key: key.clone(), + state: value.clone(), + related_contracts: RelatedContracts::default(), + contract: contract.clone(), + }) .await?; match res { ContractHandlerEvent::PutResponse { new_value: Ok(_) } => {} @@ -582,6 +693,7 @@ impl Operation for GetOp { }, sender: *sender, target: *target, + skip_list: skip_list.clone(), }); } _ => return Err(OpError::invalid_transition(self.id)), @@ -632,7 +744,7 @@ fn check_contract_found( key: ContractKey, id: Transaction, fetch_contract: bool, - value: &Result, + value: &Result, returned_key: ContractKey, ) -> Result<(), OpError> { if returned_key != key { @@ -659,114 +771,6 @@ fn check_contract_found( } } -pub(crate) fn start_op(key: ContractKey, fetch_contract: bool) -> GetOp { - let contract_location = Location::from(&key); - let id = Transaction::new::(); - tracing::debug!(tx = %id, "Requesting get contract {key} @ loc({contract_location})"); - let state = Some(GetState::PrepareRequest { - key, - id, - fetch_contract, - }); - GetOp { - id, - state, - result: None, - stats: Some(GetStats { - contract_location, - caching_peer: None, - transfer_time: None, - first_response_time: None, - step: Default::default(), - }), - } -} - -enum GetState { - /// A new petition for a get op. - ReceivedRequest, - /// Preparing request for get op. - PrepareRequest { - key: ContractKey, - id: Transaction, - fetch_contract: bool, - }, - /// Awaiting response from petition. - AwaitingResponse { - skip_list: Vec, - retries: usize, - fetch_contract: bool, - }, -} - -/// Request to get the current value from a contract. -pub(crate) async fn request_get( - op_manager: &OpManager, - get_op: GetOp, - client_id: Option, -) -> Result<(), OpError> { - let (target, id) = if let Some(GetState::PrepareRequest { key, id, .. }) = &get_op.state { - const EMPTY: &[PeerId] = &[]; - // the initial request must provide: - // - a location in the network where the contract resides - // - and the key of the contract value to get - ( - op_manager - .ring - .closest_potentially_caching(key, EMPTY) - .into_iter() - .next() - .ok_or(RingError::EmptyRing)?, - *id, - ) - } else { - return Err(OpError::UnexpectedOpState); - }; - tracing::debug!( - tx = %id, - target = %target.peer, - "Preparing get contract request", - ); - - match get_op.state { - Some(GetState::PrepareRequest { - fetch_contract, - key, - id, - .. - }) => { - let new_state = Some(GetState::AwaitingResponse { - skip_list: vec![], - retries: 0, - fetch_contract, - }); - - let msg = GetMsg::RequestGet { - id, - key, - target, - fetch_contract, - }; - - let op = GetOp { - id, - state: new_state, - result: None, - stats: get_op.stats.map(|mut s| { - s.caching_peer = Some(target); - s - }), - }; - - op_manager - .notify_op_change(NetMessage::from(msg), OpEnum::Get(op), client_id) - .await?; - } - _ => return Err(OpError::invalid_transition(get_op.id)), - } - Ok(()) -} - mod messages { use std::fmt::Display; @@ -791,6 +795,7 @@ mod messages { target: PeerKeyLocation, sender: PeerKeyLocation, htl: usize, + skip_list: Vec, }, ReturnGet { id: Transaction, @@ -798,6 +803,7 @@ mod messages { value: StoreResponse, sender: PeerKeyLocation, target: PeerKeyLocation, + skip_list: Vec, }, } @@ -886,6 +892,7 @@ mod test { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, + None, )], events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), @@ -980,6 +987,7 @@ mod test { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, + None, )], events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 609a52e1a..231cb5a07 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -4,7 +4,7 @@ use std::future::Future; use std::pin::Pin; -use std::{collections::HashSet, time::Instant}; +use std::time::Instant; pub(crate) use self::messages::PutMsg; use freenet_stdlib::prelude::*; @@ -13,7 +13,6 @@ use futures::FutureExt; use super::{OpEnum, OpError, OpInitialization, OpOutcome, Operation, OperationResult}; use crate::{ - client_events::ClientId, contract::ContractHandlerEvent, message::{InnerMessage, NetMessage, Transaction}, node::{NetworkBridge, OpManager, PeerId}, @@ -21,7 +20,7 @@ use crate::{ }; pub(crate) struct PutOp { - id: Transaction, + pub id: Transaction, state: Option, stats: Option, } @@ -169,7 +168,6 @@ impl Operation for PutOp { conn_manager: &'a mut NB, op_manager: &'a OpManager, input: &'a Self::Message, - client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { let return_msg; @@ -211,15 +209,15 @@ impl Operation for PutOp { } PutMsg::SeekNode { id, - sender, value, contract, related_contracts, htl, target, + sender, } => { let key = contract.key(); - let is_cached_contract = op_manager.ring.is_contract_cached(&key); + let is_subscribed_contract = op_manager.ring.is_subscribed_to_contract(&key); tracing::debug!( tx = %id, @@ -228,84 +226,68 @@ impl Operation for PutOp { "Puttting contract at target peer", ); - if !is_cached_contract - && op_manager + if is_subscribed_contract + || op_manager .ring - .within_caching_distance(&Location::from(&key)) + .within_subscribing_distance(&Location::from(&key)) { - tracing::debug!(tx = %id, %key, "Contract not cached @ peer {}", target.peer); - match try_to_cache_contract(op_manager, contract, &key, client_id).await { - Ok(_) => {} - Err(err) => return Err(err), + if !is_subscribed_contract { + tracing::debug!(tx = %id, %key, "Contract not cached @ peer {}", target.peer); + match try_subscribing_to_contract(op_manager, key.clone()).await { + Ok(_) => {} + Err(err) => return Err(err), + } } - } else if !is_cached_contract { - // FIXME - // in this case forward to a closer node to the target location and just wait for a response - // to give back to requesting peer - tracing::warn!( + // after the contract has been cached, push the update query + tracing::debug!(tx = %id, "Attempting contract value update"); + put_contract( + op_manager, + key.clone(), + value.clone(), + related_contracts.clone(), + contract, + ) + .await?; + tracing::debug!( tx = %id, - %key, - "Contract not found while processing info, forwarding", + "Successfully updated a value for contract {} @ {:?}", + key, + target.location ); } - // after the contract has been cached, push the update query - tracing::debug!(tx = %id, "Attempting contract value update"); - let parameters = contract.params(); - let new_value = put_contract( - op_manager, - key.clone(), - value.clone(), - related_contracts.clone(), - parameters, - client_id, - ) - .await?; - tracing::debug!(tx = %id, "Contract successfully updated"); - // if the change was successful, communicate this back to the requestor and broadcast the change - conn_manager - .send( - &sender.peer, - (PutMsg::SuccessfulUpdate { - id: *id, - new_value: new_value.clone(), - }) - .into(), - ) - .await?; - if let Some(new_htl) = htl.checked_sub(1) { // forward changes in the contract to nodes closer to the contract location, if possible - forward_changes( + let put_here = forward_put( op_manager, conn_manager, contract, - new_value.clone(), + value.clone(), *id, new_htl, ) .await; + if put_here && !is_subscribed_contract { + // if already subscribed the value was already put and merging succeeded + put_contract( + op_manager, + key.clone(), + value.clone(), + RelatedContracts::default(), + contract, + ) + .await?; + } } - let broadcast_to = op_manager - .ring - .subscribers_of(&key) - .map(|i| i.value().to_vec()) - .unwrap_or_default(); - tracing::debug!( - tx = %id, - "Successfully updated a value for contract {} @ {:?}", - key, - target.location - ); - + let broadcast_to = op_manager.get_broadcast_targets(&key, &sender.peer); match try_to_broadcast( - (*id, client_id), + *id, op_manager, self.state, broadcast_to, key.clone(), - (contract.params(), new_value), + (contract.clone(), value.clone()), ) .await { @@ -320,9 +302,8 @@ impl Operation for PutOp { id, key, new_value, - parameters, + contract, sender, - sender_subscribers, } => { let target = op_manager.ring.own_location(); @@ -332,25 +313,12 @@ impl Operation for PutOp { key.clone(), new_value.clone(), RelatedContracts::default(), - parameters.clone(), - client_id, + contract, ) .await?; tracing::debug!("Contract successfully updated"); - let broadcast_to = op_manager - .ring - .subscribers_of(key) - .map(|i| { - // Avoid already broadcast nodes and sender from broadcasting - let mut subscribers: Vec = i.value().to_vec(); - let mut avoid_list: HashSet = - sender_subscribers.iter().map(|pl| pl.peer).collect(); - avoid_list.insert(sender.peer); - subscribers.retain(|s| !avoid_list.contains(&s.peer)); - subscribers - }) - .unwrap_or_default(); + let broadcast_to = op_manager.get_broadcast_targets(key, &sender.peer); tracing::debug!( "Successfully updated a value for contract {} @ {:?}", key, @@ -358,12 +326,12 @@ impl Operation for PutOp { ); match try_to_broadcast( - (*id, client_id), + *id, op_manager, self.state, broadcast_to, key.clone(), - (parameters.clone(), new_value), + (contract.clone(), new_value), ) .await { @@ -380,24 +348,19 @@ impl Operation for PutOp { broadcasted_to, key, new_value, - parameters, + contract, } => { let sender = op_manager.ring.own_location(); let mut broadcasted_to = *broadcasted_to; let mut broadcasting = Vec::with_capacity(broadcast_to.len()); - let mut filtered_broadcast = broadcast_to - .iter() - .filter(|pk| pk.peer != sender.peer) - .collect::>(); - for peer in filtered_broadcast.iter() { + for peer in broadcast_to.iter() { let msg = PutMsg::BroadcastTo { id: *id, key: key.clone(), new_value: new_value.clone(), sender, - sender_subscribers: broadcast_to.clone(), - parameters: parameters.clone(), + contract: contract.clone(), }; let f = conn_manager.send(&peer.peer, msg.into()); broadcasting.push(f); @@ -412,18 +375,18 @@ impl Operation for PutOp { } else { None } - }) - .rev(); + }); let mut incorrect_results = 0; for (peer_num, err) in error_futures { // remove the failed peers in reverse order - let peer = filtered_broadcast.remove(peer_num); + let peer = broadcast_to.get(peer_num).unwrap(); tracing::warn!( "failed broadcasting put change to {} with error {}; dropping connection", peer.peer, err ); + // todo: review this, maybe we should just dropping this subscription conn_manager.drop_connection(&peer.peer).await?; incorrect_results += 1; } @@ -434,10 +397,10 @@ impl Operation for PutOp { ); // Subscriber nodes have been notified of the change, the operation is completed - return_msg = None; + return_msg = Some(PutMsg::SuccessfulPut { id: *id }); new_state = None; } - PutMsg::SuccessfulUpdate { .. } => { + PutMsg::SuccessfulPut { .. } => { match self.state { Some(PutState::AwaitingResponse { contract, .. }) => { tracing::debug!("Successfully updated value for {}", contract,); @@ -456,6 +419,7 @@ impl Operation for PutOp { contract, new_value, htl, + sender, } => { let key = contract.key(); let peer_loc = op_manager.ring.own_location(); @@ -466,47 +430,70 @@ impl Operation for PutOp { "Forwarding changes, trying put the contract" ); - let cached_contract = op_manager.ring.is_contract_cached(&key); + let is_subscribed_contract = op_manager.ring.is_subscribed_to_contract(&key); let within_caching_dist = op_manager .ring - .within_caching_distance(&Location::from(&key)); - if !cached_contract && within_caching_dist { - match try_to_cache_contract(op_manager, contract, &key, client_id).await { - Ok(_) => {} - Err(err) => return Err(err), + .within_subscribing_distance(&Location::from(&key)); + if is_subscribed_contract || within_caching_dist { + if !is_subscribed_contract { + tracing::debug!(%key, "Contract not cached @ peer {}", peer_loc.peer); + match try_subscribing_to_contract(op_manager, key.clone()).await { + Ok(_) => {} + Err(err) => return Err(err), + } } - } else if !within_caching_dist { - // not a contract this node cares about; do nothing - return Ok(OperationResult { - return_msg: None, - state: None, - }); + // after the contract has been cached, push the update query + put_contract( + op_manager, + key.clone(), + new_value.clone(), + RelatedContracts::default(), + contract, + ) + .await?; } - // after the contract has been cached, push the update query - let new_value = put_contract( - op_manager, - key, - new_value.clone(), - RelatedContracts::default(), - contract.params(), - client_id, - ) - .await?; // if successful, forward to the next closest peers (if any) if let Some(new_htl) = htl.checked_sub(1) { - forward_changes( + let put_here = forward_put( op_manager, conn_manager, contract, - new_value, + new_value.clone(), *id, new_htl, ) .await; + if put_here && !is_subscribed_contract { + // if already subscribed the value was already put and merging succeeded + put_contract( + op_manager, + key.clone(), + new_value.clone(), + RelatedContracts::default(), + contract, + ) + .await?; + } + } + + let broadcast_to = op_manager.get_broadcast_targets(&key, &sender.peer); + match try_to_broadcast( + *id, + op_manager, + self.state, + broadcast_to, + key.clone(), + (contract.clone(), new_value.clone()), + ) + .await + { + Ok((state, msg)) => { + new_state = state; + return_msg = msg; + } + Err(err) => return Err(err), } - return_msg = None; - new_state = None; } _ => return Err(OpError::UnexpectedOpState), } @@ -516,6 +503,26 @@ impl Operation for PutOp { } } +impl OpManager { + fn get_broadcast_targets(&self, key: &ContractKey, sender: &PeerId) -> Vec { + let mut subscribers = self + .ring + .subscribers_of(key) + .map(|subs| { + subs.value() + .iter() + .filter(|pk| &pk.peer != sender) + .copied() + .collect::>() + }) + .unwrap_or_default(); + if let Some(peer) = self.ring.subscribed_to_contract(key) { + subscribers.push(peer); + } + subscribers + } +} + fn build_op_result( id: Transaction, state: Option, @@ -529,18 +536,20 @@ fn build_op_result( }) } -pub(super) async fn try_to_cache_contract<'a>( - op_manager: &'a OpManager, - contract: &ContractContainer, - key: &ContractKey, - client_id: Option, +pub(super) async fn try_subscribing_to_contract( + op_manager: &OpManager, + key: ContractKey, ) -> Result<(), OpError> { // this node does not have the contract, so instead store the contract and execute the put op. let res = op_manager - .notify_contract_handler(ContractHandlerEvent::Cache(contract.clone()), client_id) + .notify_contract_handler(ContractHandlerEvent::Subscribe { key: key.clone() }) .await?; - if let ContractHandlerEvent::CacheResult(Ok(_)) = res { - op_manager.ring.contract_cached(key); + if let ContractHandlerEvent::SubscribeResponse { + response: Ok(subcribed_to), + .. + } = res + { + op_manager.ring.add_subscription(key, subcribed_to); tracing::debug!("Contract successfully cached"); Ok(()) } else { @@ -552,12 +561,12 @@ pub(super) async fn try_to_cache_contract<'a>( } async fn try_to_broadcast( - (id, client_id): (Transaction, Option), + id: Transaction, op_manager: &OpManager, state: Option, broadcast_to: Vec, key: ContractKey, - (parameters, new_value): (Parameters<'static>, WrappedState), + (contract, new_value): (ContractContainer, WrappedState), ) -> Result<(Option, Option), OpError> { let new_state; let return_msg; @@ -571,18 +580,18 @@ async fn try_to_broadcast( key ); // means the whole tx finished so can return early - new_state = None; - return_msg = Some(PutMsg::SuccessfulUpdate { id, new_value }); + new_state = Some(PutState::AwaitingResponse { contract: key }); + return_msg = None; } else { tracing::debug!("Callback to start broadcasting to other nodes"); new_state = Some(PutState::BroadcastOngoing); return_msg = Some(PutMsg::Broadcasting { id, new_value, - parameters, broadcasted_to: 0, broadcast_to, key, + contract, }); let op = PutOp { @@ -591,11 +600,7 @@ async fn try_to_broadcast( stats: None, }; op_manager - .notify_op_change( - NetMessage::from(return_msg.unwrap()), - OpEnum::Put(op), - client_id, - ) + .notify_op_change(NetMessage::from(return_msg.unwrap()), OpEnum::Put(op)) .await?; return Err(OpError::StatePushed); } @@ -657,11 +662,7 @@ enum PutState { } /// Request to insert/update a value into a contract. -pub(crate) async fn request_put( - op_manager: &OpManager, - mut put_op: PutOp, - client_id: Option, -) -> Result<(), OpError> { +pub(crate) async fn request_put(op_manager: &OpManager, mut put_op: PutOp) -> Result<(), OpError> { let key = if let Some(PutState::PrepareRequest { contract, .. }) = &put_op.state { contract.key() } else { @@ -710,7 +711,7 @@ pub(crate) async fn request_put( }; op_manager - .notify_op_change(NetMessage::from(msg), OpEnum::Put(op), client_id) + .notify_op_change(NetMessage::from(msg), OpEnum::Put(op)) .await?; } _ => return Err(OpError::invalid_transition(put_op.id)), @@ -724,20 +725,16 @@ async fn put_contract( key: ContractKey, state: WrappedState, related_contracts: RelatedContracts<'static>, - parameters: Parameters<'static>, - client_id: Option, + contract: &ContractContainer, ) -> Result { // after the contract has been cached, push the update query match op_manager - .notify_contract_handler( - ContractHandlerEvent::PutQuery { - key, - state, - related_contracts, - parameters: Some(parameters), - }, - client_id, - ) + .notify_contract_handler(ContractHandlerEvent::PutQuery { + key, + state, + related_contracts, + contract: Some(contract.clone()), + }) .await { Ok(ContractHandlerEvent::PutResponse { @@ -754,25 +751,28 @@ async fn put_contract( } } -// TODO: keep track of who is supposed to have the contract, and only send if necessary -// since sending the contract over and over, will be expensive; this can be done via subscriptions -/// Communicate changes in the contract to other peers nearby the contract location. +/// Forwards the put request to a peer which is closer to the assigned contract location if possible. +/// If is not possible to forward the request, then this peer is the final target and should store the contract. +/// It returns whether this peer should be storing the contract or not. +/// /// This operation is "fire and forget" and the node does not keep track if is successful or not. -async fn forward_changes( +async fn forward_put( op_manager: &OpManager, conn_manager: &CB, contract: &ContractContainer, new_value: WrappedState, id: Transaction, htl: usize, -) where +) -> bool +where CB: NetworkBridge, { let key = contract.key(); let contract_loc = Location::from(&key); const EMPTY: &[PeerId] = &[]; let forward_to = op_manager.ring.closest_potentially_caching(&key, EMPTY); - let own_loc = op_manager.ring.own_location().location.expect("infallible"); + let own_pkloc = op_manager.ring.own_location(); + let own_loc = own_pkloc.location.expect("infallible"); if let Some(peer) = forward_to { let other_loc = peer.location.as_ref().expect("infallible"); let other_distance = contract_loc.distance(other_loc); @@ -785,6 +785,7 @@ async fn forward_changes( &peer.peer, (PutMsg::PutForward { id, + sender: own_pkloc, contract: contract.clone(), new_value: new_value.clone(), htl, @@ -792,8 +793,10 @@ async fn forward_changes( .into(), ) .await; + return false; } } + true } mod messages { @@ -822,16 +825,14 @@ mod messages { /// Forward a contract and it's latest value to an other node PutForward { id: Transaction, + sender: PeerKeyLocation, contract: ContractContainer, new_value: WrappedState, /// current htl, reduced by one at each hop htl: usize, }, /// Value successfully inserted/updated. - SuccessfulUpdate { - id: Transaction, - new_value: WrappedState, - }, + SuccessfulPut { id: Transaction }, /// Target the node which is closest to the key SeekNode { id: Transaction, @@ -851,8 +852,7 @@ mod messages { broadcast_to: Vec, key: ContractKey, new_value: WrappedState, - #[serde(deserialize_with = "Parameters::deser_params")] - parameters: Parameters<'static>, + contract: ContractContainer, }, /// Broadcasting a change to a peer, which then will relay the changes to other peers. BroadcastTo { @@ -860,9 +860,7 @@ mod messages { sender: PeerKeyLocation, key: ContractKey, new_value: WrappedState, - #[serde(deserialize_with = "Parameters::deser_params")] - parameters: Parameters<'static>, - sender_subscribers: Vec, + contract: ContractContainer, }, } @@ -872,7 +870,7 @@ mod messages { Self::SeekNode { id, .. } => id, Self::RequestPut { id, .. } => id, Self::Broadcasting { id, .. } => id, - Self::SuccessfulUpdate { id, .. } => id, + Self::SuccessfulPut { id, .. } => id, Self::PutForward { id, .. } => id, Self::AwaitPut { id } => id, Self::BroadcastTo { id, .. } => id, @@ -891,7 +889,7 @@ mod messages { use PutMsg::*; matches!( self, - SuccessfulUpdate { .. } | SeekNode { .. } | PutForward { .. } + SuccessfulPut { .. } | SeekNode { .. } | PutForward { .. } ) } @@ -924,7 +922,7 @@ mod messages { Self::SeekNode { .. } => write!(f, "SeekNode(id: {id})"), Self::RequestPut { .. } => write!(f, "RequestPut(id: {id})"), Self::Broadcasting { .. } => write!(f, "Broadcasting(id: {id})"), - Self::SuccessfulUpdate { .. } => write!(f, "SusscessfulUpdate(id: {id})"), + Self::SuccessfulPut { .. } => write!(f, "SusscessfulUpdate(id: {id})"), Self::PutForward { .. } => write!(f, "PutForward(id: {id})"), Self::AwaitPut { .. } => write!(f, "AwaitPut(id: {id})"), Self::BroadcastTo { .. } => write!(f, "BroadcastTo(id: {id})"), @@ -965,6 +963,7 @@ mod test { ) .await; let mut locations = sim_nw.get_locations_by_node(); + let gw0_loc = locations.remove(&"gateway-0".into()).unwrap(); let node0_loc = locations.remove(&"node-1".into()).unwrap(); let node1_loc = locations.remove(&"node-2".into()).unwrap(); @@ -973,18 +972,20 @@ mod test { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val.clone(), + Some(gw0_loc), )], events_to_generate: HashMap::new(), - contract_subscribers: HashMap::from_iter([(key.clone(), vec![node1_loc])]), + contract_subscribers: HashMap::new(), }; let node_2 = NodeSpecification { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val.clone(), + Some(gw0_loc), )], events_to_generate: HashMap::new(), - contract_subscribers: HashMap::from_iter([(key.clone(), vec![node0_loc])]), + contract_subscribers: HashMap::new(), }; let put_event = ContractRequest::Put { @@ -998,9 +999,10 @@ mod test { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract.clone())), contract_val, + None, )], events_to_generate: HashMap::from_iter([(1, put_event)]), - contract_subscribers: HashMap::new(), + contract_subscribers: HashMap::from_iter([(key.clone(), vec![node0_loc, node1_loc])]), }; // establish network @@ -1017,7 +1019,7 @@ mod test { sim_nw .trigger_event("gateway-0", 1, Some(Duration::from_secs(1))) .await?; - assert!(sim_nw.has_put_contract("gateway-0", &key, &new_value)); + assert!(sim_nw.has_put_contract("gateway-0", &key)); assert!(sim_nw.event_listener.contract_broadcasted(&key)); Ok(()) } diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 1c789a4e0..cd6f8478e 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -7,7 +7,6 @@ use serde::{Deserialize, Serialize}; use super::{OpEnum, OpError, OpInitialization, OpOutcome, Operation, OperationResult}; use crate::{ - client_events::ClientId, contract::ContractError, message::{InnerMessage, NetMessage, Transaction}, node::{NetworkBridge, OpManager, PeerId}, @@ -18,8 +17,97 @@ pub(crate) use self::messages::SubscribeMsg; const MAX_RETRIES: usize = 10; +#[derive(Debug)] +enum SubscribeState { + /// Prepare the request to subscribe. + PrepareRequest { + id: Transaction, + key: ContractKey, + }, + /// Received a request to subscribe to this network. + ReceivedRequest, + /// Awaitinh response from petition. + AwaitingResponse { + skip_list: Vec, + retries: usize, + upstream_subscriber: Option, + }, + Completed { + subscribed_to: PeerKeyLocation, + }, +} + +pub(crate) struct SubscribeResult { + pub subscribed_to: PeerKeyLocation, +} + +impl TryFrom for SubscribeResult { + type Error = OpError; + + fn try_from(value: SubscribeOp) -> Result { + if let Some(SubscribeState::Completed { subscribed_to }) = value.state { + Ok(SubscribeResult { subscribed_to }) + } else { + Err(OpError::UnexpectedOpState) + } + } +} + +pub(crate) fn start_op(key: ContractKey) -> SubscribeOp { + let id = Transaction::new::(); + let state = Some(SubscribeState::PrepareRequest { id, key }); + SubscribeOp { id, state } +} + +/// Request to subscribe to value changes from a contract. +pub(crate) async fn request_subscribe( + op_manager: &OpManager, + sub_op: SubscribeOp, +) -> Result<(), OpError> { + let (target, _id) = if let Some(SubscribeState::PrepareRequest { id, key }) = &sub_op.state { + if !op_manager.ring.is_subscribed_to_contract(key) { + return Err(OpError::ContractError(ContractError::ContractNotFound( + key.clone(), + ))); + } + const EMPTY: &[PeerId] = &[]; + ( + op_manager + .ring + .closest_potentially_caching(key, EMPTY) + .into_iter() + .next() + .ok_or_else(|| RingError::NoCachingPeers(key.clone()))?, + *id, + ) + } else { + return Err(OpError::UnexpectedOpState); + }; + + match sub_op.state { + Some(SubscribeState::PrepareRequest { id, key, .. }) => { + let new_state = Some(SubscribeState::AwaitingResponse { + skip_list: vec![], + retries: 0, + upstream_subscriber: None, + }); + let msg = SubscribeMsg::RequestSub { id, key, target }; + let op = SubscribeOp { + id, + state: new_state, + }; + op_manager + .notify_op_change(NetMessage::from(msg), OpEnum::Subscribe(op)) + .await?; + } + _ => return Err(OpError::invalid_transition(sub_op.id)), + } + + Ok(()) +} + pub(crate) struct SubscribeOp { - id: Transaction, + pub id: Transaction, state: Option, } @@ -29,25 +117,12 @@ impl SubscribeOp { } pub(super) fn finalized(&self) -> bool { - matches!(self.state, Some(SubscribeState::Completed)) + matches!(self.state, Some(SubscribeState::Completed { .. })) } pub(super) fn record_transfer(&mut self) {} } -pub(crate) struct SubscribeResult {} - -impl TryFrom for SubscribeResult { - type Error = OpError; - - fn try_from(value: SubscribeOp) -> Result { - value - .finalized() - .then_some(SubscribeResult {}) - .ok_or(OpError::UnexpectedOpState) - } -} - impl Operation for SubscribeOp { type Message = SubscribeMsg; type Result = SubscribeResult; @@ -97,10 +172,9 @@ impl Operation for SubscribeOp { fn process_message<'a, NB: NetworkBridge>( self, - conn_manager: &'a mut NB, + _conn_manager: &'a mut NB, op_manager: &'a OpManager, input: &'a Self::Message, - client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { let return_msg; @@ -121,7 +195,8 @@ impl Operation for SubscribeOp { target: *target, subscriber: sender, skip_list: vec![sender.peer], - htl: 0, + htl: op_manager.ring.max_hops_to_live, + retries: 0, }); } SubscribeMsg::SeekNode { @@ -131,34 +206,35 @@ impl Operation for SubscribeOp { target, skip_list, htl, + retries, } => { - let sender = op_manager.ring.own_location(); + let this_peer = op_manager.ring.own_location(); let return_err = || -> OperationResult { OperationResult { return_msg: Some(NetMessage::from(SubscribeMsg::ReturnSub { key: key.clone(), id: *id, subscribed: false, - sender, + sender: this_peer, target: *subscriber, })), state: None, } }; - if !op_manager.ring.is_contract_cached(key) { + if !op_manager.ring.is_subscribed_to_contract(key) { tracing::debug!(tx = %id, "Contract {} not found at {}, trying other peer", key, target.peer); let Some(new_target) = op_manager .ring - .closest_potentially_caching(key, [&sender.peer].as_slice()) + .closest_potentially_caching(key, skip_list.as_slice()) else { tracing::warn!(tx = %id, "No peer found while trying getting contract {key}"); return Err(OpError::RingError(RingError::NoCachingPeers(key.clone()))); }; - let new_htl = htl + 1; + let new_htl = htl - 1; - if new_htl > MAX_RETRIES { + if new_htl == 0 { return Ok(return_err()); } @@ -167,21 +243,27 @@ impl Operation for SubscribeOp { tracing::debug!(tx = %id, "Forward request to peer: {}", new_target.peer); // Retry seek node when the contract to subscribe has not been found in this node - conn_manager - .send( - &new_target.peer, - (SubscribeMsg::SeekNode { - id: *id, - key: key.clone(), - subscriber: *subscriber, - target: new_target, - skip_list: new_skip_list.clone(), - htl: new_htl, - }) - .into(), - ) - .await?; - } else if op_manager.ring.add_subscriber(key, *subscriber).is_err() { + return build_op_result( + *id, + Some(SubscribeState::AwaitingResponse { + skip_list: new_skip_list.clone(), + retries: *retries, + upstream_subscriber: Some(*subscriber), + }), + (SubscribeMsg::SeekNode { + id: *id, + key: key.clone(), + subscriber: this_peer, + target: new_target, + skip_list: new_skip_list, + htl: new_htl, + retries: *retries, + }) + .into(), + ); + } + + if op_manager.ring.add_subscriber(key, *subscriber).is_err() { // max number of subscribers for this contract reached return Ok(return_err()); } @@ -195,7 +277,6 @@ impl Operation for SubscribeOp { "Peer successfully subscribed to contract", ); new_state = None; - // TODO review behaviour, if the contract is not cached should return subscribed false? return_msg = Some(SubscribeMsg::ReturnSub { sender: *target, target: *subscriber, @@ -225,7 +306,7 @@ impl Operation for SubscribeOp { Some(SubscribeState::AwaitingResponse { mut skip_list, retries, - .. + upstream_subscriber, }) => { if retries < MAX_RETRIES { skip_list.push(sender.peer); @@ -241,8 +322,9 @@ impl Operation for SubscribeOp { key: key.clone(), subscriber, target, - skip_list: vec![target.peer], - htl: 0, + skip_list: skip_list.clone(), + htl: op_manager.ring.max_hops_to_live, + retries: retries + 1, }); } else { return Err(RingError::NoCachingPeers(key.clone()).into()); @@ -250,6 +332,7 @@ impl Operation for SubscribeOp { new_state = Some(SubscribeState::AwaitingResponse { skip_list, retries: retries + 1, + upstream_subscriber, }); } else { return Err(OpError::MaxRetriesExceeded( @@ -268,27 +351,39 @@ impl Operation for SubscribeOp { id, target, .. - } => { - match self.state { - Some(SubscribeState::AwaitingResponse { .. }) => { - tracing::info!( - tx = %id, - %key, - this_peer = %target.peer, - provider = %sender.peer, - "Subscribed to contract" - ); - op_manager.ring.add_subscription(key.clone()); - // fixme: should inform back to the network event loop in case a client is waiting for response - let _ = client_id; - new_state = Some(SubscribeState::Completed); + } => match self.state { + Some(SubscribeState::AwaitingResponse { + upstream_subscriber, + .. + }) => { + tracing::info!( + tx = %id, + %key, + this_peer = %target.peer, + provider = %sender.peer, + "Subscribed to contract" + ); + op_manager.ring.add_subscription(key.clone(), *sender); + + new_state = Some(SubscribeState::Completed { + subscribed_to: *sender, + }); + if let Some(upstream_subscriber) = upstream_subscriber { + return_msg = Some(SubscribeMsg::ReturnSub { + id: *id, + key: key.clone(), + sender: *target, + target: upstream_subscriber, + subscribed: true, + }); + } else { return_msg = None; } - _other => { - return Err(OpError::invalid_transition(self.id)); - } } - } + _other => { + return Err(OpError::invalid_transition(self.id)); + } + }, _ => return Err(OpError::UnexpectedOpState), } @@ -312,76 +407,6 @@ fn build_op_result( }) } -pub(crate) fn start_op(key: ContractKey) -> SubscribeOp { - let id = Transaction::new::(); - let state = Some(SubscribeState::PrepareRequest { id, key }); - SubscribeOp { id, state } -} - -#[derive(Debug)] -enum SubscribeState { - /// Prepare the request to subscribe. - PrepareRequest { - id: Transaction, - key: ContractKey, - }, - /// Received a request to subscribe to this network. - ReceivedRequest, - /// Awaitinh response from petition. - AwaitingResponse { - skip_list: Vec, - retries: usize, - }, - Completed, -} - -/// Request to subscribe to value changes from a contract. -pub(crate) async fn request_subscribe( - op_manager: &OpManager, - sub_op: SubscribeOp, - client_id: Option, -) -> Result<(), OpError> { - let (target, _id) = if let Some(SubscribeState::PrepareRequest { id, key }) = &sub_op.state { - if !op_manager.ring.is_contract_cached(key) { - return Err(OpError::ContractError(ContractError::ContractNotFound( - key.clone(), - ))); - } - const EMPTY: &[PeerId] = &[]; - ( - op_manager - .ring - .closest_potentially_caching(key, EMPTY) - .into_iter() - .next() - .ok_or_else(|| RingError::NoCachingPeers(key.clone()))?, - *id, - ) - } else { - return Err(OpError::UnexpectedOpState); - }; - - match sub_op.state { - Some(SubscribeState::PrepareRequest { id, key, .. }) => { - let new_state = Some(SubscribeState::AwaitingResponse { - skip_list: vec![], - retries: 0, - }); - let msg = SubscribeMsg::RequestSub { id, key, target }; - let op = SubscribeOp { - id, - state: new_state, - }; - op_manager - .notify_op_change(NetMessage::from(msg), OpEnum::Subscribe(op), client_id) - .await?; - } - _ => return Err(OpError::invalid_transition(sub_op.id)), - } - - Ok(()) -} - mod messages { use crate::message::InnerMessage; use std::fmt::Display; @@ -406,6 +431,7 @@ mod messages { subscriber: PeerKeyLocation, skip_list: Vec, htl: usize, + retries: usize, }, ReturnSub { id: Transaction, @@ -500,6 +526,7 @@ mod test { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, + None, )], events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index 6e87cae6e..342161a50 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -1,9 +1,13 @@ // TODO: complete update logic in the network +use freenet_stdlib::prelude::*; use futures::future::BoxFuture; use super::{OpError, OpOutcome, Operation}; -use crate::{client_events::ClientId, node::NetworkBridge}; +use crate::{ + client_events::ClientId, + node::{NetworkBridge, OpManager}, +}; pub(crate) use self::messages::UpdateMsg; @@ -51,7 +55,7 @@ impl Operation for UpdateOp { _conn_manager: &'a mut NB, _op_manager: &'a crate::node::OpManager, _input: &Self::Message, - _client_id: Option, + // _client_id: Option, ) -> std::pin::Pin< Box> + Send + 'a>, > { @@ -59,6 +63,19 @@ impl Operation for UpdateOp { } } +// todo: new_state should be a delta when possible! +pub(crate) fn start_op(_key: ContractKey, _new_state: WrappedState, _htl: usize) -> UpdateOp { + todo!() +} + +pub(crate) async fn request_update( + _op_manager: &OpManager, + _update_op: UpdateOp, + _client_id: Option, +) -> Result<(), OpError> { + todo!() +} + mod messages { use std::fmt::Display; diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index 1715e53f3..d74bf1c11 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -1,14 +1,7 @@ //! Ring protocol logic and supporting types. //! -//! # Routing -//! The routing mechanism consist in a greedy routing algorithm which just targets -//! the closest location to the target destination iteratively in each hop, until it reaches -//! the destination. -//! -//! Path is limited to local knowledge, at any given point only 3 data points are known: -//! - previous node -//! - next node -//! - final location +//! Mainly maintains a healthy and optimal pool of connections to other peers in the network +//! and routes requests to the optimal peers. use std::hash::Hash; use std::sync::atomic::AtomicBool; @@ -27,7 +20,7 @@ use std::{ }; use anyhow::bail; -use dashmap::{mapref::one::Ref as DmRef, DashMap, DashSet}; +use dashmap::{mapref::one::Ref as DmRef, DashMap}; use either::Either; use freenet_stdlib::prelude::{ContractInstanceId, ContractKey}; use parking_lot::RwLock; @@ -137,10 +130,6 @@ impl LiveTransactionTracker { } } - fn prune_transactions_from_peer(&self, peer: &PeerId) { - self.tx_per_peer.remove(peer); - } - fn new() -> (Self, sync::mpsc::Receiver) { let (missing_peer, rx) = sync::mpsc::channel(10); ( @@ -152,6 +141,10 @@ impl LiveTransactionTracker { ) } + fn prune_transactions_from_peer(&self, peer: &PeerId) { + self.tx_per_peer.remove(peer); + } + fn has_live_connection(&self, peer: &PeerId) -> bool { self.tx_per_peer.contains_key(peer) } @@ -164,15 +157,15 @@ impl LiveTransactionTracker { /// Thread safe and friendly data structure to keep track of the local knowledge /// of the state of the ring. /// -/// Note: For now internally we wrap some of the types internally with locks and/or use -/// multithreaded maps. In the future if performance requires it some of this can be moved -/// towards a more lock-free multithreading model if necessary. +// Note: For now internally we wrap some of the types internally with locks and/or use +// multithreaded maps. In the future if performance requires it some of this can be moved +// towards a more lock-free multithreading model if necessary. pub(crate) struct Ring { pub rnd_if_htl_above: usize, pub max_hops_to_live: usize, pub peer_key: PeerId, - max_connections: usize, - min_connections: usize, + pub max_connections: usize, + pub min_connections: usize, router: Arc>, topology_manager: RwLock, /// Fast is for when there are less than our target number of connections so we want to acquire new connections quickly. @@ -180,8 +173,6 @@ pub(crate) struct Ring { fast_acquisition: AtomicBool, connections_by_location: RwLock>>, location_for_peer: RwLock>, - /// contracts in the ring cached by this node - cached_contracts: DashSet, own_location: AtomicU64, /// The container for subscriber is a vec instead of something like a hashset /// that would allow for blind inserts of duplicate peers subscribing because @@ -189,7 +180,8 @@ pub(crate) struct Ring { /// of subscribers more often than inserting, and anyways is a relatively short sequence /// then is more optimal to just use a vector for it's compact memory layout. subscribers: DashMap>, - subscriptions: RwLock>, + /// Contracts this peer is subscribed to. + subscriptions: DashMap, /// Interim connections ongoing handshake or successfully open connections /// Is important to keep track of this so no more connections are accepted prematurely. open_connections: AtomicUsize, @@ -198,6 +190,9 @@ pub(crate) struct Ring { // todo: add blacklist // contract_blacklist: Arc>>, event_register: Box, + /// Whether this peer is a gateway or not. This will affect behavior of the node when acquiring + /// and dropping connections. + is_gateway: bool, } // /// A data type that represents the fact that a peer has been blacklisted @@ -209,26 +204,24 @@ pub(crate) struct Ring { // } impl Ring { - const MIN_CONNECTIONS: usize = 10; + const DEFAULT_MIN_CONNECTIONS: usize = 10; - const MAX_CONNECTIONS: usize = 20; + const DEFAULT_MAX_CONNECTIONS: usize = 20; /// Max number of subscribers for a contract. const MAX_SUBSCRIBERS: usize = 10; - /// Above this number of remaining hops, - /// randomize which of node a message which be forwarded to. - const RAND_WALK_ABOVE_HTL: usize = 7; + /// Above this number of remaining hops, randomize which node a message which be forwarded to. + const DEFAULT_RAND_WALK_ABOVE_HTL: usize = 7; - /// Max hops to be performed for certain operations (e.g. propagating - /// connection of a peer in the network). - const MAX_HOPS_TO_LIVE: usize = 10; + /// Max hops to be performed for certain operations (e.g. propagating connection of a peer in the network). + const DEFAULT_MAX_HOPS_TO_LIVE: usize = 10; pub fn new( config: &NodeConfig, - gateways: &[PeerKeyLocation], event_loop_notifier: EventLoopNotificationsSender, event_register: ER, + is_gateway: bool, ) -> Result, anyhow::Error> { let (live_tx_tracker, missing_candidate_rx) = LiveTransactionTracker::new(); @@ -240,25 +233,25 @@ impl Ring { let max_hops_to_live = if let Some(v) = config.max_hops_to_live { v } else { - Self::MAX_HOPS_TO_LIVE + Self::DEFAULT_MAX_HOPS_TO_LIVE }; let rnd_if_htl_above = if let Some(v) = config.rnd_if_htl_above { v } else { - Self::RAND_WALK_ABOVE_HTL + Self::DEFAULT_RAND_WALK_ABOVE_HTL }; let min_connections = if let Some(v) = config.min_number_conn { v } else { - Self::MIN_CONNECTIONS + Self::DEFAULT_MIN_CONNECTIONS }; let max_connections = if let Some(v) = config.max_number_conn { v } else { - Self::MAX_CONNECTIONS + Self::DEFAULT_MAX_CONNECTIONS }; let router = Arc::new(RwLock::new(Router::new(&[]))); @@ -277,27 +270,21 @@ impl Ring { fast_acquisition: AtomicBool::new(true), connections_by_location: RwLock::new(BTreeMap::new()), location_for_peer: RwLock::new(BTreeMap::new()), - cached_contracts: DashSet::new(), own_location, peer_key, subscribers: DashMap::new(), - subscriptions: RwLock::new(Vec::new()), + subscriptions: DashMap::new(), open_connections: AtomicUsize::new(0), live_tx_tracker: live_tx_tracker.clone(), event_register: Box::new(event_register), + is_gateway, }; if let Some(loc) = config.location { if config.local_ip.is_none() || config.local_port.is_none() { return Err(anyhow::anyhow!("IP and port are required for gateways")); } - ring.update_location(Some(loc)); - for PeerKeyLocation { peer, location } in gateways { - // FIXME: this is problematic cause gateways will take all spots then! - // all gateways are aware of each other - ring.add_connection((*location).unwrap(), *peer); - } } let ring = Arc::new(ring); @@ -315,6 +302,11 @@ impl Ring { Ok(ring) } + pub fn open_connections(&self) -> usize { + self.open_connections + .load(std::sync::atomic::Ordering::Acquire) + } + async fn refresh_router(router: Arc>) { let mut interval = tokio::time::interval(Duration::from_secs(60 * 5)); interval.tick().await; @@ -344,25 +336,28 @@ impl Ring { } } - #[inline(always)] - /// Return if a location is within appropiate caching distance. - pub fn within_caching_distance(&self, _loc: &Location) -> bool { - // This always returns true as of current version since LRU cache will make sure - // to remove contracts when capacity is fully utilized. - // So all nodes along the path will be caching all the contracts. - // This will be changed in the future as the caching logic gets more complicated. - true + /// Return if a location is within appropiate subscription distance. + pub fn within_subscribing_distance(&self, loc: &Location) -> bool { + const CACHING_DISTANCE: f64 = 0.05; + const MAX_CACHED: usize = 100; + let caching_distance = Distance::new(CACHING_DISTANCE); + self.subscriptions.len() < MAX_CACHED + && self + .own_location() + .location + .map(|own_loc| own_loc.distance(loc) <= caching_distance) + .unwrap_or(false) } - /// Whether this node already has this contract cached or not. + /// Whether this node already is subscribed to this contract or not. #[inline] - pub fn is_contract_cached(&self, key: &ContractKey) -> bool { - self.cached_contracts.contains(key) + pub fn is_subscribed_to_contract(&self, key: &ContractKey) -> bool { + self.subscriptions.contains_key(key) } #[inline] - pub fn contract_cached(&self, key: &ContractKey) { - self.cached_contracts.insert(key.clone()); + pub fn subscribed_to_contract(&self, key: &ContractKey) -> Option { + self.subscriptions.get(key).map(|v| *v.value()) } /// Update this node location. @@ -404,10 +399,16 @@ impl Ring { /// /// # Panic /// Will panic if the node checking for this condition has no location assigned. - pub fn should_accept(&self, location: Location) -> bool { + pub fn should_accept(&self, location: Location, peer: &PeerId) -> bool { let open_conn = self .open_connections .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.location_for_peer.read().get(peer).is_some() { + // avoid connecting mroe than once to the same peer + self.open_connections + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + return false; + } let my_location = self .own_location() .location @@ -560,8 +561,8 @@ impl Ring { } /// Add a new subscription for this peer. - pub fn add_subscription(&self, contract: ContractKey) { - self.subscriptions.write().push(contract); + pub fn add_subscription(&self, contract: ContractKey, peer: PeerKeyLocation) { + self.subscriptions.insert(contract, peer); } pub fn subscribers_of( @@ -632,9 +633,14 @@ impl Ring { live_tx_tracker: LiveTransactionTracker, mut missing_candidates: sync::mpsc::Receiver, ) -> Result<(), DynError> { - /// Peers whose connection should be acquired. - fn should_swap<'a>(_connections: impl Iterator) -> Vec { + /// Peers whose connection should be dropped. + fn should_swap<'a>( + _connections: impl Iterator, + is_gateway: bool, + ) -> Vec { // todo: instead we should be using ConnectionEvaluator here + // todo: if the peer is a gateway behaviour on how quickly we drop connections may be different + let _ = is_gateway; vec![] } @@ -743,6 +749,7 @@ impl Ring { && !live_tx_tracker.has_live_connection(&conn.location.peer) }) .map(|conn| &conn.location), + self.is_gateway, ) }; if !should_swap.is_empty() { @@ -822,7 +829,7 @@ impl Ring { }, }; let id = *msg.id(); - notifier.send(Either::Left((msg.into(), None))).await?; + notifier.send(Either::Left(msg.into())).await?; Ok(Some(id)) } } diff --git a/crates/core/src/server.rs b/crates/core/src/server.rs index 573b92378..84cf29fe5 100644 --- a/crates/core/src/server.rs +++ b/crates/core/src/server.rs @@ -162,7 +162,7 @@ pub mod local_node { Err(err) => { tracing::error!("{err}"); let err = Err(ErrorKind::Unhandled { - cause: format!("{err}"), + cause: format!("{err}").into(), } .into()); match receiver { diff --git a/crates/core/src/tracing.rs b/crates/core/src/tracing.rs index a225fb634..d4633963b 100644 --- a/crates/core/src/tracing.rs +++ b/crates/core/src/tracing.rs @@ -104,14 +104,10 @@ pub(crate) struct NetEventLog<'a> { } impl<'a> NetEventLog<'a> { - pub fn route_event( - tx: &'a Transaction, - op_manager: &'a OpManager, - route_event: &RouteEvent, - ) -> Self { + pub fn route_event(tx: &'a Transaction, ring: &'a Ring, route_event: &RouteEvent) -> Self { NetEventLog { tx, - peer_id: &op_manager.ring.peer_key, + peer_id: &ring.peer_key, kind: EventKind::Route(route_event.clone()), } } @@ -241,16 +237,13 @@ impl<'a> NetEventLog<'a> { }) => { let key = contract.key(); EventKind::Put(PutEvent::Request { - performer: target.peer, + requester: target.peer, key, }) } - NetMessage::Put(PutMsg::SuccessfulUpdate { new_value, .. }) => { - EventKind::Put(PutEvent::PutSuccess { - requester: op_manager.ring.peer_key, - value: new_value.clone(), - }) - } + NetMessage::Put(PutMsg::SuccessfulPut { .. }) => EventKind::Put(PutEvent::PutSuccess { + requester: op_manager.ring.peer_key, + }), NetMessage::Put(PutMsg::Broadcasting { new_value, broadcast_to, @@ -416,7 +409,7 @@ impl EventRegister { Ok(file) => file, Err(err) => { tracing::error!("Failed openning log file {:?} with: {err}", event_log_path); - panic!("Failed openning log file"); // fixme: propagate this to the main thread + panic!("Failed openning log file"); // fixme: propagate this to the main event loop } }; let mut num_written = 0; @@ -779,6 +772,7 @@ async fn send_to_metrics_server( let msg = PeerChange::removed_connection_msg(*from, send_msg.peer_id); ws_stream.send(Message::Binary(msg)).await } + // todo: send op events too 8put, get, update, etc) so we can keep track of transactions _ => Ok(()), }; if let Err(error) = res { @@ -1109,12 +1103,11 @@ enum ConnectEvent { #[cfg_attr(test, derive(arbitrary::Arbitrary))] enum PutEvent { Request { - performer: PeerId, + requester: PeerId, key: ContractKey, }, PutSuccess { requester: PeerId, - value: WrappedState, }, BroadcastEmitted { /// subscribed peers @@ -1304,12 +1297,7 @@ pub(super) mod test { }) } - pub fn has_put_contract( - &self, - peer: &PeerId, - for_key: &ContractKey, - expected_value: &WrappedState, - ) -> bool { + pub fn has_put_contract(&self, peer: &PeerId, for_key: &ContractKey) -> bool { let Ok(logs) = self.logs.try_lock() else { return false; }; @@ -1323,7 +1311,6 @@ pub(super) mod test { }); for (_tx, events) in put_ops { - let mut is_expected_value = false; let mut is_expected_key = false; let mut is_expected_peer = false; for ev in events { @@ -1332,16 +1319,13 @@ pub(super) mod test { PutEvent::Request { key, .. } if key == for_key => { is_expected_key = true; } - PutEvent::PutSuccess { requester, value } - if requester == peer && value == expected_value => - { + PutEvent::PutSuccess { requester } if requester == peer => { is_expected_peer = true; - is_expected_value = true; } _ => {} } } - if is_expected_value && is_expected_peer && is_expected_key { + if is_expected_peer && is_expected_key { return true; } } @@ -1510,22 +1494,26 @@ pub(super) mod test { ]; let listener = TestEventListener::new().await; - locations.iter().for_each(|(other, location)| { - listener.register_events(Either::Left(NetEventLog { - tx: &tx, - peer_id: &peer_id, - kind: EventKind::Connect(ConnectEvent::Connected { - this: PeerKeyLocation { - peer: peer_id, - location: Some(loc), - }, - connected: PeerKeyLocation { - peer: *other, - location: Some(*location), - }, - }), - })); - }); + let futs = futures::stream::FuturesUnordered::from_iter(locations.iter().map( + |(other, location)| { + listener.register_events(Either::Left(NetEventLog { + tx: &tx, + peer_id: &peer_id, + kind: EventKind::Connect(ConnectEvent::Connected { + this: PeerKeyLocation { + peer: peer_id, + location: Some(loc), + }, + connected: PeerKeyLocation { + peer: *other, + location: Some(*location), + }, + }), + })) + }, + )); + + futures::future::join_all(futs).await; let distances: Vec<_> = listener.connections(peer_id).collect(); assert!(distances.len() == 3); diff --git a/crates/core/src/runtime.rs b/crates/core/src/wasm_runtime.rs similarity index 64% rename from crates/core/src/runtime.rs rename to crates/core/src/wasm_runtime.rs index 609bb803d..171a8afd8 100644 --- a/crates/core/src/runtime.rs +++ b/crates/core/src/wasm_runtime.rs @@ -4,20 +4,20 @@ mod delegate; mod delegate_store; mod error; mod native_api; +mod runtime; mod secrets_store; mod state_store; mod store; #[cfg(test)] mod tests; -mod wasm_runtime; pub(crate) use contract::ContractRuntimeInterface; pub use contract_store::ContractStore; -pub(crate) use delegate::{DelegateExecError, DelegateRuntimeInterface}; +pub(crate) use delegate::DelegateRuntimeInterface; pub use delegate_store::DelegateStore; -pub(crate) use error::ContractError; -pub(crate) use error::RuntimeResult; +pub(crate) use error::{ContractError, RuntimeInnerError, RuntimeResult}; +pub use runtime::{ContractExecError, Runtime}; +pub(crate) use secrets_store::SecretStoreError; pub use secrets_store::SecretsStore; pub use state_store::StateStore; pub(crate) use state_store::{StateStorage, StateStoreError}; -pub use wasm_runtime::{ContractExecError, Runtime}; diff --git a/crates/core/src/runtime/contract.rs b/crates/core/src/wasm_runtime/contract.rs similarity index 100% rename from crates/core/src/runtime/contract.rs rename to crates/core/src/wasm_runtime/contract.rs diff --git a/crates/core/src/runtime/contract_store.rs b/crates/core/src/wasm_runtime/contract_store.rs similarity index 100% rename from crates/core/src/runtime/contract_store.rs rename to crates/core/src/wasm_runtime/contract_store.rs diff --git a/crates/core/src/runtime/delegate.rs b/crates/core/src/wasm_runtime/delegate.rs similarity index 100% rename from crates/core/src/runtime/delegate.rs rename to crates/core/src/wasm_runtime/delegate.rs diff --git a/crates/core/src/runtime/delegate_store.rs b/crates/core/src/wasm_runtime/delegate_store.rs similarity index 99% rename from crates/core/src/runtime/delegate_store.rs rename to crates/core/src/wasm_runtime/delegate_store.rs index 5540ce428..03649c4b9 100644 --- a/crates/core/src/runtime/delegate_store.rs +++ b/crates/core/src/wasm_runtime/delegate_store.rs @@ -6,7 +6,7 @@ use freenet_stdlib::prelude::{ use std::{fs::File, io::Write, path::PathBuf, sync::Arc}; use stretto::Cache; -use crate::runtime::store::SafeWriter; +use crate::wasm_runtime::store::SafeWriter; use super::store::StoreFsManagement; use super::RuntimeResult; diff --git a/crates/core/src/runtime/error.rs b/crates/core/src/wasm_runtime/error.rs similarity index 60% rename from crates/core/src/runtime/error.rs rename to crates/core/src/wasm_runtime/error.rs index 7e855304b..933c12bdb 100644 --- a/crates/core/src/runtime/error.rs +++ b/crates/core/src/wasm_runtime/error.rs @@ -1,10 +1,10 @@ use std::fmt::Display; -use freenet_stdlib::prelude::{ContractKey, DelegateKey, SecretsId}; +use freenet_stdlib::prelude::{ContractKey, DelegateKey}; use crate::DynError; -use super::{delegate, secrets_store, wasm_runtime, DelegateExecError}; +use super::{delegate, runtime, secrets_store}; pub type RuntimeResult = std::result::Result; @@ -12,50 +12,8 @@ pub type RuntimeResult = std::result::Result; pub struct ContractError(Box); impl ContractError { - pub fn is_contract_exec_error(&self) -> bool { - matches!(&*self.0, RuntimeInnerError::ContractExecError(_)) - } - - pub fn is_delegate_exec_error(&self) -> bool { - matches!(&*self.0, RuntimeInnerError::DelegateExecError(_)) - } - - pub fn delegate_auth_access(&self) -> Option<&SecretsId> { - match &*self.0 { - RuntimeInnerError::DelegateExecError(DelegateExecError::UnauthorizedSecretAccess { - secret, - .. - }) => Some(secret), - _ => None, - } - } - - pub fn is_execution_error(&self) -> bool { - use RuntimeInnerError::*; - matches!( - &*self.0, - WasmInstantiationError(_) | WasmExportError(_) | WasmCompileError(_) - ) - } - - pub fn delegate_is_missing(&self) -> bool { - matches!(&*self.0, RuntimeInnerError::DelegateNotFound(_)) - } - - pub fn secret_is_missing(&self) -> bool { - matches!( - &*self.0, - RuntimeInnerError::SecretStoreError(secrets_store::SecretStoreError::MissingSecret(_)) - ) - } - - pub fn get_secret_id(&self) -> SecretsId { - match &*self.0 { - RuntimeInnerError::SecretStoreError( - secrets_store::SecretStoreError::MissingSecret(id), - ) => id.clone(), - _ => panic!(), - } + pub(crate) fn deref(&self) -> &RuntimeInnerError { + &self.0 } } @@ -87,7 +45,7 @@ impl_err!(std::io::Error); impl_err!(secrets_store::SecretStoreError); impl_err!(bincode::Error); impl_err!(delegate::DelegateExecError); -impl_err!(wasm_runtime::ContractExecError); +impl_err!(runtime::ContractExecError); impl_err!(wasmer::CompileError); impl_err!(wasmer::ExportError); impl_err!(wasmer::InstantiationError); @@ -123,7 +81,7 @@ pub(crate) enum RuntimeInnerError { ContractNotFound(ContractKey), #[error(transparent)] - ContractExecError(#[from] wasm_runtime::ContractExecError), + ContractExecError(#[from] runtime::ContractExecError), #[error("failed while unwrapping contract to raw bytes")] UnwrapContract, diff --git a/crates/core/src/runtime/native_api.rs b/crates/core/src/wasm_runtime/native_api.rs similarity index 98% rename from crates/core/src/runtime/native_api.rs rename to crates/core/src/wasm_runtime/native_api.rs index c061fd78d..0e903a463 100644 --- a/crates/core/src/runtime/native_api.rs +++ b/crates/core/src/wasm_runtime/native_api.rs @@ -4,7 +4,7 @@ use dashmap::DashMap; use once_cell::sync::Lazy; use wasmer::{Function, Imports}; -use super::wasm_runtime::InstanceInfo; +use super::runtime::InstanceInfo; /// This is a map of starting addresses of the instance memory space. /// diff --git a/crates/core/src/runtime/wasm_runtime.rs b/crates/core/src/wasm_runtime/runtime.rs similarity index 100% rename from crates/core/src/runtime/wasm_runtime.rs rename to crates/core/src/wasm_runtime/runtime.rs diff --git a/crates/core/src/runtime/secrets_store.rs b/crates/core/src/wasm_runtime/secrets_store.rs similarity index 100% rename from crates/core/src/runtime/secrets_store.rs rename to crates/core/src/wasm_runtime/secrets_store.rs diff --git a/crates/core/src/runtime/state_store.rs b/crates/core/src/wasm_runtime/state_store.rs similarity index 93% rename from crates/core/src/runtime/state_store.rs rename to crates/core/src/wasm_runtime/state_store.rs index 49a6f54a1..3019adf0e 100644 --- a/crates/core/src/runtime/state_store.rs +++ b/crates/core/src/wasm_runtime/state_store.rs @@ -11,12 +11,12 @@ pub enum StateStoreError { MissingContract(ContractKey), } -impl From for crate::runtime::ContractError { +impl From for crate::wasm_runtime::ContractError { fn from(value: StateStoreError) -> Self { match value { - StateStoreError::Any(err) => crate::runtime::ContractError::from(err), + StateStoreError::Any(err) => crate::wasm_runtime::ContractError::from(err), err @ StateStoreError::MissingContract(_) => { - crate::runtime::ContractError::from(Into::::into(format!("{err}"))) + crate::wasm_runtime::ContractError::from(Into::::into(format!("{err}"))) } } } diff --git a/crates/core/src/runtime/store.rs b/crates/core/src/wasm_runtime/store.rs similarity index 100% rename from crates/core/src/runtime/store.rs rename to crates/core/src/wasm_runtime/store.rs diff --git a/crates/core/src/runtime/tests/contract.rs b/crates/core/src/wasm_runtime/tests/contract.rs similarity index 98% rename from crates/core/src/runtime/tests/contract.rs rename to crates/core/src/wasm_runtime/tests/contract.rs index 914a20133..70ebaf898 100644 --- a/crates/core/src/runtime/tests/contract.rs +++ b/crates/core/src/wasm_runtime/tests/contract.rs @@ -1,6 +1,6 @@ use freenet_stdlib::prelude::*; -use crate::runtime::tests::TestSetup; +use crate::wasm_runtime::tests::TestSetup; use super::super::contract::*; use super::super::Runtime; diff --git a/crates/core/src/runtime/tests/mod.rs b/crates/core/src/wasm_runtime/tests/mod.rs similarity index 100% rename from crates/core/src/runtime/tests/mod.rs rename to crates/core/src/wasm_runtime/tests/mod.rs diff --git a/crates/core/src/runtime/tests/time.rs b/crates/core/src/wasm_runtime/tests/time.rs similarity index 100% rename from crates/core/src/runtime/tests/time.rs rename to crates/core/src/wasm_runtime/tests/time.rs diff --git a/crates/fdev/src/commands.rs b/crates/fdev/src/commands.rs index ac150d9a4..9057ed652 100644 --- a/crates/fdev/src/commands.rs +++ b/crates/fdev/src/commands.rs @@ -168,7 +168,7 @@ async fn execute_command( let state_store = StateStore::new(Storage::new(Some(&database_path)).await?, MAX_MEM_CACHE)?; let rt = freenet::dev_tool::Runtime::build(contract_store, delegate_store, secret_store, false)?; - let mut executor = Executor::new(state_store, || Ok(()), OperationMode::Local, rt) + let mut executor = Executor::new(state_store, || Ok(()), OperationMode::Local, rt, None) .await .map_err(|err| anyhow::anyhow!(err))?; diff --git a/crates/fdev/src/network_metrics_server.rs b/crates/fdev/src/network_metrics_server.rs index fbf7cebff..ebe5f0ab2 100644 --- a/crates/fdev/src/network_metrics_server.rs +++ b/crates/fdev/src/network_metrics_server.rs @@ -346,7 +346,8 @@ async fn record_saver( change: Change, } - // FIXME: this ain't flushing correctly after test ends + // FIXME: this ain't flushing correctly after test ends, + // for now flushing each single time we get a new record // let mut batch = Vec::with_capacity(1024); while let Ok(change) = incoming_rec.recv().await { let change = WithTimestamp { diff --git a/crates/fdev/src/testing.rs b/crates/fdev/src/testing.rs index c33ebb71d..85a09c9f3 100644 --- a/crates/fdev/src/testing.rs +++ b/crates/fdev/src/testing.rs @@ -45,13 +45,16 @@ pub struct TestConfig { /// Events are simulated get, puts and other operations. #[arg(long, default_value_t = u32::MAX)] events: u32, + /// Time in milliseconds to wait for the next event to be executed. + #[arg(long)] + event_wait_ms: Option, /// Time in milliseconds to wait for the network to be sufficiently connected to start sending events. /// (20% of the expected connections to be processed per gateway) #[arg(long)] - wait_duration: Option, - /// Time in milliseconds to wait for the next event to be executed. + connection_wait_ms: Option, + /// Time in milliseconds to wait for the next peer in the simulation to be started. #[arg(long)] - event_wait_time: Option, + peer_start_backoff_ms: Option, /// If provided, the execution data will be saved in this directory. #[arg(long)] execution_data: Option, @@ -67,11 +70,17 @@ impl TestConfig { fn get_connection_check_params(&self) -> (Duration, f64) { let conns_per_gw = (self.nodes / self.gateways) as f64; let conn_percent = (conns_per_gw / self.nodes as f64).min(0.99); - let connectivity_timeout = Duration::from_millis(self.wait_duration.unwrap_or_else(|| { - // expect a peer to take max 200ms to connect, this should happen in parallel - // but err on the side of safety - (conns_per_gw * 200.0).ceil() as u64 - })); + let connectivity_timeout = + Duration::from_millis(self.connection_wait_ms.unwrap_or_else(|| { + // expect a peer to take max 200ms to connect, this should happen in parallel + // but err on the side of safety + (conns_per_gw + * self + .peer_start_backoff_ms + .map(|ms| ms as f64) + .unwrap_or(200.0)) + .ceil() as u64 + })); (connectivity_timeout, conn_percent) } @@ -144,7 +153,7 @@ async fn config_sim_network(base_config: &TestConfig) -> anyhow::Result anyhow::Result anyhow::Result<(), Error> { .collect(); let mut events = EventChain::new(peers, user_ev_controller, config.events, true); let next_event_wait_time = config - .event_wait_time + .event_wait_ms .map(Duration::from_millis) .unwrap_or(Duration::from_millis(200)); let (connectivity_timeout, network_connection_percent) = config.get_connection_check_params(); @@ -376,7 +381,7 @@ impl SubProcess { } } -async fn child(config: &super::TestConfig, id: usize) -> anyhow::Result<()> { +async fn child(test_config: &super::TestConfig, id: usize) -> anyhow::Result<()> { // write logs to stderr so stdout and stdin are free of unexpected data std::env::set_var("FREENET_LOG_TO_STDERR", "1"); @@ -394,15 +399,22 @@ async fn child(config: &super::TestConfig, id: usize) -> anyhow::Result<()> { let mut event_generator = MemoryEventsGen::::new_with_seed( receiver_ch.clone(), node_config.peer_id, - config.seed.expect("seed should be set for child process"), + test_config + .seed + .expect("seed should be set for child process"), ); event_generator.rng_params( id, - config.gateways + config.nodes, - config.max_contract_number.unwrap_or(config.nodes * 10), - config.events as usize, + test_config.gateways + test_config.nodes, + test_config + .max_contract_number + .unwrap_or(test_config.nodes * 10), + test_config.events as usize, ); let config = SimPeer::from(node_config); + if let Some(backoff) = test_config.peer_start_backoff_ms { + tokio::time::sleep(Duration::from_millis(backoff)).await; + } tokio::task::spawn(this_child.event_loop()); config.start_child(event_generator).await?; Ok(()) diff --git a/crates/fdev/src/testing/single_process.rs b/crates/fdev/src/testing/single_process.rs index 287f11086..5c83d5221 100644 --- a/crates/fdev/src/testing/single_process.rs +++ b/crates/fdev/src/testing/single_process.rs @@ -16,7 +16,7 @@ pub(super) async fn run(config: &super::TestConfig) -> anyhow::Result<(), super: let events = config.events; let next_event_wait_time = config - .event_wait_time + .event_wait_ms .map(Duration::from_millis) .unwrap_or(Duration::from_millis(200)); let (connectivity_timeout, network_connection_percent) = config.get_connection_check_params(); diff --git a/crates/fdev/src/wasm_runtime/state.rs b/crates/fdev/src/wasm_runtime/state.rs index 22d45bf17..735ed823c 100644 --- a/crates/fdev/src/wasm_runtime/state.rs +++ b/crates/fdev/src/wasm_runtime/state.rs @@ -50,6 +50,7 @@ impl AppState { }, OperationMode::Local, rt, + None, ) .map_err(|err| anyhow::anyhow!(err)) .await?, diff --git a/crates/fdev/src/wasm_runtime/user_events.rs b/crates/fdev/src/wasm_runtime/user_events.rs index 35990b77d..bebea671c 100644 --- a/crates/fdev/src/wasm_runtime/user_events.rs +++ b/crates/fdev/src/wasm_runtime/user_events.rs @@ -102,19 +102,19 @@ impl StdInput { { self.input.rewind().map_err(|e| { Box::new(ClientError::from(ErrorKind::Unhandled { - cause: format!("{e}"), + cause: format!("{e}").into(), })) })?; match self.config.ser_format { Some(DeserializationFmt::Json) => { let state: serde_json::Value = self.read_input().map_err(|e| { Box::new(ClientError::from(ErrorKind::Unhandled { - cause: format!("deserialization error: {e}"), + cause: format!("deserialization error: {e}").into(), })) })?; let json_str = serde_json::to_string_pretty(&state).map_err(|e| { Box::new(ClientError::from(ErrorKind::Unhandled { - cause: format!("{e}"), + cause: format!("{e}").into(), })) })?; tracing::debug!("{cmd:?} value:\n{json_str}"); @@ -135,7 +135,7 @@ impl StdInput { _ => { let state: Vec = self.read_input().map_err(|e| { Box::new(ClientError::from(ErrorKind::Unhandled { - cause: format!("deserialization error: {e}"), + cause: format!("deserialization error: {e}").into(), })) })?; Ok(state.into()) @@ -242,7 +242,7 @@ impl From for OpenRequest<'static> { ContractRequest::Update { key, data }.into() } Command::Exit => ClientRequest::Disconnect { - cause: Some("shutdown".to_owned()), + cause: Some("shutdown".to_owned().into()), }, _ => unreachable!(), }; @@ -317,12 +317,13 @@ impl ClientEventsProxy for StdInput { .await .map_err(|e| { ClientError::from(ErrorKind::Unhandled { - cause: format!("{e}"), + cause: format!("{e}").into(), }) })? .ok_or_else(|| { ClientError::from(ErrorKind::Unhandled { - cause: format!("missing contract parameters: {key}",), + cause: format!("missing contract parameters: {key}",) + .into(), }) })?; if let Err(e) = self.app_state.printout_deser(&p) { diff --git a/network-monitor/dist/index.html b/network-monitor/dist/index.html index 1d060da73..fa02b2fdb 100644 --- a/network-monitor/dist/index.html +++ b/network-monitor/dist/index.html @@ -40,6 +40,14 @@

Connections history

>
+ diff --git a/network-monitor/src/topology.ts b/network-monitor/src/topology.ts index 2961ee5bf..42ff098f5 100644 --- a/network-monitor/src/topology.ts +++ b/network-monitor/src/topology.ts @@ -51,6 +51,7 @@ export class PeerId { } export function handleChange(peerChange: fbTopology.PeerChange) { + const previousPeers = Object.keys(peers).length; try { const unpacked = peerChange.unpack(); switch (unpacked.changeType) { @@ -74,11 +75,13 @@ export function handleChange(peerChange: fbTopology.PeerChange) { unpacked.currentState.forEach((connection) => { handleAddedConnection(connection, false); }); - ringHistogram(Object.values(peers)); } catch (e) { console.error(e); } finally { - updateTable(); + if (previousPeers !== Object.keys(peers).length) { + ringHistogram(Object.values(peers)); + updateTable(); + } } } @@ -382,8 +385,8 @@ function updateTable() { const rows = Array.from(tbody.querySelectorAll("tr")); const sortedRows = rows.sort((a, b) => { - const cellA = a.cells[1].textContent!; - const cellB = b.cells[1].textContent!; + const cellA = a.cells[2].textContent!; + const cellB = b.cells[2].textContent!; return cellA.localeCompare(cellB); }); @@ -395,6 +398,7 @@ const sortDirections: number[] = []; document.addEventListener("DOMContentLoaded", () => { ringHistogram(Object.values(peers)); + document .querySelector("#peers-table-h")! .querySelectorAll("th")! @@ -402,6 +406,12 @@ document.addEventListener("DOMContentLoaded", () => { sortDirections.push(1); tableSorting(header, index); }); + + (window as any).resetPeersTable = function () { + peers = {}; + updateTable(); + ringHistogram(Object.values(peers)); + }; }); function tableSorting(header: HTMLTableCellElement, index: number) { diff --git a/stdlib b/stdlib index 75d3d955b..73121030e 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit 75d3d955bdcc9c6afc419e1744efddb749868615 +Subproject commit 73121030e7dbc84379dfeed80a21c6188f520d34