diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae257aaf9..4c9589d5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,14 +10,14 @@ on: jobs: test_all: - name: Test features + name: Test runs-on: ubuntu-latest strategy: max-parallel: 1 matrix: - args: ["--no-default-features", "--features default"] + args: ["--no-default-features --features trace,websocket,sqlite"] env: FREENET_LOG: error CARGO_TARGET_DIR: ${{ github.workspace }}/target diff --git a/Cargo.lock b/Cargo.lock index 69f0e71db..9e82e978f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,7 +250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ "async-lock", - "autocfg", + "autocfg 1.1.0", "cfg-if", "concurrent-queue", "futures-lite", @@ -305,12 +305,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" - [[package]] name = "atomic" version = "0.6.0" @@ -337,6 +331,15 @@ dependencies = [ "url", ] +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -754,6 +757,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -841,7 +853,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" dependencies = [ - "autocfg", + "autocfg 1.1.0", "cfg-if", "libc", "scopeguard", @@ -996,7 +1008,7 @@ version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ - "autocfg", + "autocfg 1.1.0", "cfg-if", "crossbeam-utils", "memoffset 0.9.0", @@ -1029,7 +1041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -1332,7 +1344,7 @@ checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ "curve25519-dalek", "ed25519", - "rand_core", + "rand_core 0.6.4", "serde", "sha2", "zeroize", @@ -1588,11 +1600,11 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry-jaeger", - "ordered-float 3.9.2", + "ordered-float 4.1.1", "parking_lot", "pav_regression", "pico-args", - "rand", + "rand 0.8.5", "rocksdb", "serde", "serde_json", @@ -1606,8 +1618,8 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "ulid", "unsigned-varint", - "uuid", "wasmer", "xz2", ] @@ -1637,7 +1649,7 @@ dependencies = [ "futures", "js-sys", "once_cell", - "rand", + "rand 0.8.5", "semver", "serde", "serde-wasm-bindgen 0.6.0", @@ -1663,6 +1675,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "2.0.0" @@ -2162,7 +2180,7 @@ dependencies = [ "http", "hyper", "log", - "rand", + "rand 0.8.5", "tokio", "url", "xmltree", @@ -2174,7 +2192,7 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "autocfg", + "autocfg 1.1.0", "hashbrown 0.12.3", "serde", ] @@ -2457,7 +2475,7 @@ dependencies = [ "libp2p-swarm", "log", "quick-protobuf", - "rand", + "rand 0.8.5", ] [[package]] @@ -2492,7 +2510,7 @@ dependencies = [ "parking_lot", "pin-project", "quick-protobuf", - "rand", + "rand 0.8.5", "rw-stream-sink", "smallvec", "thiserror", @@ -2551,7 +2569,7 @@ dependencies = [ "log", "multihash", "quick-protobuf", - "rand", + "rand 0.8.5", "sha2", "thiserror", "zeroize", @@ -2570,7 +2588,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand", + "rand 0.8.5", "smallvec", "socket2 0.5.4", "tokio", @@ -2610,7 +2628,7 @@ dependencies = [ "multihash", "once_cell", "quick-protobuf", - "rand", + "rand 0.8.5", "sha2", "snow", "static_assertions", @@ -2633,7 +2651,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand", + "rand 0.8.5", "void", ] @@ -2653,7 +2671,7 @@ dependencies = [ "log", "parking_lot", "quinn", - "rand", + "rand 0.8.5", "ring", "rustls", "socket2 0.5.4", @@ -2674,7 +2692,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "log", - "rand", + "rand 0.8.5", "smallvec", "void", ] @@ -2696,7 +2714,7 @@ dependencies = [ "log", "multistream-select", "once_cell", - "rand", + "rand 0.8.5", "smallvec", "tokio", "void", @@ -2841,7 +2859,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ - "autocfg", + "autocfg 1.1.0", "scopeguard", ] @@ -2956,7 +2974,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -2965,7 +2983,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -3210,7 +3228,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-integer", "num-traits", ] @@ -3227,7 +3245,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -3238,7 +3256,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-traits", ] @@ -3248,7 +3266,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ - "autocfg", + "autocfg 1.1.0", "num-integer", "num-traits", ] @@ -3259,7 +3277,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ - "autocfg", + "autocfg 1.1.0", "libm", ] @@ -3401,7 +3419,7 @@ dependencies = [ "opentelemetry_api", "ordered-float 3.9.2", "percent-encoding", - "rand", + "rand 0.8.5", "regex", "thiserror", "tokio", @@ -3432,6 +3450,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -3651,7 +3678,7 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ - "autocfg", + "autocfg 1.1.0", "bitflags 1.3.2", "cfg-if", "concurrent-queue", @@ -3857,7 +3884,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c78e758510582acc40acb90458401172d41f1016f8c9dde89e49677afb7eec1" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring", "rustc-hash", "rustls", @@ -3895,6 +3922,25 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -3902,8 +3948,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", ] [[package]] @@ -3913,9 +3969,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", ] +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -3925,6 +3996,69 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rayon" version = "1.8.0" @@ -3957,6 +4091,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -4162,7 +4305,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", @@ -4535,7 +4678,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4550,7 +4693,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg", + "autocfg 1.1.0", ] [[package]] @@ -4586,7 +4729,7 @@ dependencies = [ "blake2", "chacha20poly1305 0.9.1", "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "ring", "rustc_version", "sha2", @@ -4772,7 +4915,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.5", "rsa", "serde", "sha1", @@ -4811,7 +4954,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha1", @@ -4866,12 +5009,12 @@ checksum = "51481a2abc8af47b7447b39b221ce806728e1dcbbb1354b0d5bacb1e5e1f72de" dependencies = [ "async-channel", "async-io", - "atomic 0.6.0", + "atomic", "crossbeam-channel", "futures", "getrandom", "parking_lot", - "rand", + "rand 0.8.5", "seahash", "thiserror", "tracing", @@ -5363,7 +5506,7 @@ dependencies = [ "idna 0.2.3", "ipnet", "lazy_static", - "rand", + "rand 0.8.5", "smallvec", "socket2 0.4.9", "thiserror", @@ -5389,7 +5532,7 @@ dependencies = [ "idna 0.4.0", "ipnet", "once_cell", - "rand", + "rand 0.8.5", "smallvec", "thiserror", "tinyvec", @@ -5410,7 +5553,7 @@ dependencies = [ "lru-cache", "once_cell", "parking_lot", - "rand", + "rand 0.8.5", "resolv-conf", "smallvec", "thiserror", @@ -5437,7 +5580,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -5456,6 +5599,18 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +[[package]] +name = "ulid" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e95a59b292ca0cf9b45be2e52294d1ca6cb24eb11b08ef4376f73f1a00c549" +dependencies = [ + "chrono", + "lazy_static", + "rand 0.6.5", + "serde", +] + [[package]] name = "unicase" version = "2.7.0" @@ -5580,11 +5735,6 @@ name = "uuid" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" -dependencies = [ - "atomic 0.5.3", - "getrandom", - "serde", -] [[package]] name = "valuable" @@ -6125,7 +6275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "serde", "zeroize", ] @@ -6206,7 +6356,7 @@ dependencies = [ "nohash-hasher", "parking_lot", "pin-project", - "rand", + "rand 0.8.5", "static_assertions", ] diff --git a/apps/freenet-email-app/web/src/api.rs b/apps/freenet-email-app/web/src/api.rs index 8a5e21b00..642ba9ab3 100644 --- a/apps/freenet-email-app/web/src/api.rs +++ b/apps/freenet-email-app/web/src/api.rs @@ -39,7 +39,7 @@ impl WebApi { #[cfg(all(not(target_family = "wasm"), feature = "use-node"))] fn new() -> Result { - todo!() + unimplemented!() } #[cfg(all(target_family = "wasm", feature = "use-node"))] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0a61fc330..52474ae34 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -59,13 +59,14 @@ thiserror = "1" tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "fs"] } tower-http = { version = "0.4", features = ["trace", "fs"] } unsigned-varint = "0.7" -uuid = { version = "1", features = ["serde", "v4", "v1"] } +# uuid = { version = "1", features = ["serde", "v4", "v1"] } +ulid = { version = "0.4", features = ["serde"] } sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio-rustls"], optional = true } # TODO(kakoc): clang should be installed for rocksdb; write about that in prerequisites/dev guide rocksdb = { version = "0.21.0", default-features = false, optional = true } pav_regression = "0.4.0" itertools = "0.11" -ordered-float = "3.9.1" +ordered-float = "4.1.1" notify = "6" wasmer = { workspace = true, features = [ "sys"] } chrono = { workspace = true } @@ -73,7 +74,7 @@ tar = { version = "0.4.38" } xz2 = { version = "0.1" } # Tracing deps -tracing = { version = "0.1", optional = true } +tracing = { version = "0.1" } opentelemetry = { version = "0.20.0", default-features = false, features = ["rt-tokio", "trace"], optional = true } opentelemetry-jaeger = { version = "0.19.0", features = ["rt-tokio","collector_client", "isahc"], optional = true } tracing-opentelemetry = { version = "0.21.0", optional = true } @@ -90,11 +91,11 @@ pico-args = "0.5" freenet-stdlib = { workspace = true, features = ["testing", "net"] } [features] -default = ["websocket", "rocks_db", "trace"] +default = ["trace", "websocket", "sqlite"] testing = ["arbitrary"] rocks_db = ["rocksdb"] sqlite = ["sqlx"] websocket = ["axum/ws"] -trace = ["tracing", "opentelemetry", "opentelemetry-jaeger", "tracing-opentelemetry", "tracing-subscriber"] +trace = ["opentelemetry", "opentelemetry-jaeger", "tracing-opentelemetry", "tracing-subscriber"] local-mode = [] network-mode = [] diff --git a/crates/core/src/bin/freenet.rs b/crates/core/src/bin/freenet.rs index aef4f7f60..da78c34eb 100644 --- a/crates/core/src/bin/freenet.rs +++ b/crates/core/src/bin/freenet.rs @@ -1,8 +1,6 @@ use clap::Parser; use freenet::local_node::{Executor, NodeConfig, OperationMode}; use std::net::SocketAddr; -use tracing::metadata::LevelFilter; -use tracing_subscriber::EnvFilter; type DynError = Box; @@ -23,16 +21,7 @@ async fn run_local(config: NodeConfig) -> Result<(), DynError> { } fn main() -> Result<(), DynError> { - tracing_subscriber::fmt() - .with_level(true) - .with_file(true) - .with_line_number(true) - .with_env_filter( - EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(), - ) - .init(); + freenet::config::set_logger(); let config = NodeConfig::parse(); let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(4) diff --git a/crates/core/src/client_events.rs b/crates/core/src/client_events.rs index b8574ed32..af1085741 100644 --- a/crates/core/src/client_events.rs +++ b/crates/core/src/client_events.rs @@ -99,6 +99,16 @@ pub struct OpenRequest<'a> { pub token: Option, } +impl Display for OpenRequest<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "client request {{ client: {}, req: {} }}", + &self.client_id, &*self.request + ) + } +} + impl<'a> OpenRequest<'a> { pub fn into_owned(self) -> OpenRequest<'static> { OpenRequest { @@ -145,8 +155,8 @@ pub(crate) mod test { // FIXME: remove unused #![allow(unused)] - use std::collections::HashMap; use std::sync::Arc; + use std::{collections::HashMap, time::Duration}; use freenet_stdlib::client_api::ContractRequest; use freenet_stdlib::prelude::*; @@ -245,7 +255,7 @@ pub(crate) mod test { } else { // let contract_no = rng.gen_range(0..self.non_owned_contracts.len()); // self.non_owned_contracts[contract_no] - todo!("fixme") + todo!() }; break ContractRequest::Subscribe { key, summary: None }.into(); } @@ -264,37 +274,38 @@ pub(crate) mod test { impl ClientEventsProxy for MemoryEventsGen { fn recv(&mut self) -> BoxFuture<'_, Result, ClientError>> { - // async move { - // loop { - // if self.signal.changed().await.is_ok() { - // let (ev_id, pk) = *self.signal.borrow(); - // if pk == self.id && !self.random { - // let res = OpenRequest { - // id: ClientId(1), - // request: self - // .generate_deterministic_event(&ev_id) - // .expect("event not found"), - // notification_channel: None, - // }; - // return Ok(res); - // } else if pk == self.id { - // let res = OpenRequest { - // id: ClientId(1), - // request: self.generate_rand_event(), - // notification_channel: None, - // }; - // return Ok(res); - // } - // } else { - // log::debug!("sender half of user event gen dropped"); - // // probably the process finished, wait for a bit and then kill the thread - // tokio::time::sleep(Duration::from_secs(1)).await; - // panic!("finished orphan background thread"); - // } - // } - // } - // .boxed() - todo!("fixme") + async { + loop { + if self.signal.changed().await.is_ok() { + let (ev_id, pk) = *self.signal.borrow(); + if pk == self.id && !self.random { + let res = OpenRequest { + client_id: ClientId::FIRST, + request: self + .generate_deterministic_event(&ev_id) + .expect("event not found") + .into(), + notification_channel: None, + token: None, + }; + return Ok(res.into_owned()); + } else if pk == self.id { + let res = OpenRequest { + client_id: ClientId::FIRST, + request: self.generate_rand_event().into(), + notification_channel: None, + token: None, + }; + return Ok(res.into_owned()); + } + } else { + // probably the process finished, wait for a bit and then kill the thread + tokio::time::sleep(Duration::from_secs(1)).await; + panic!("finished orphan background thread"); + } + } + } + .boxed() } fn send( diff --git a/crates/core/src/client_events/websocket.rs b/crates/core/src/client_events/websocket.rs index c2840438a..47f7b4b12 100644 --- a/crates/core/src/client_events/websocket.rs +++ b/crates/core/src/client_events/websocket.rs @@ -474,7 +474,7 @@ impl ClientEventsProxy for WebSocketProxy { break Ok(reply.into_owned()); } } else { - todo!() + break Err(ClientError::from(ErrorKind::ChannelClosed)); } } } diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 8a36a3b1e..b503edef2 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -52,6 +52,7 @@ pub(crate) struct WebSocketApiConfig { port: u16, } +#[cfg(feature = "websocket")] impl From for SocketAddr { fn from(val: WebSocketApiConfig) -> Self { (val.ip, val.port).into() @@ -143,10 +144,6 @@ impl ConfigPaths { } impl Config { - pub fn set_from_cli() -> Result<(), DynError> { - todo!() - } - pub fn set_op_mode(mode: OperationMode) { let local_mode = matches!(mode, OperationMode::Local); Self::conf() @@ -323,10 +320,48 @@ impl libp2p::swarm::Executor for GlobalExecutor { } } +pub fn set_logger() { + #[cfg(feature = "trace")] + { + static LOGGER_SET: AtomicBool = AtomicBool::new(false); + if LOGGER_SET + .compare_exchange( + false, + true, + std::sync::atomic::Ordering::Acquire, + std::sync::atomic::Ordering::SeqCst, + ) + .is_err() + { + return; + } + + let filter = if cfg!(any(test, debug_assertions)) { + tracing_subscriber::filter::LevelFilter::DEBUG.into() + } else { + tracing_subscriber::filter::LevelFilter::INFO.into() + }; + + let sub = tracing_subscriber::fmt().with_level(true).with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(filter) + .from_env_lossy() + .add_directive("stretto=off".parse().unwrap()) + .add_directive("sqlx=error".parse().unwrap()), + ); + + if cfg!(any(test, debug_assertions)) { + sub.with_file(true).with_line_number(true).init(); + } else { + sub.init(); + } + } +} + +#[cfg(feature = "trace")] pub(super) mod tracer { use super::*; - #[cfg(feature = "trace")] pub fn init_tracer() -> Result<(), opentelemetry::trace::TraceError> { use opentelemetry::{global, sdk::propagation::TraceContextPropagator}; use tracing_subscriber::layer::SubscriberExt; diff --git a/crates/core/src/contract.rs b/crates/core/src/contract.rs index ef3d77fdd..3a69540b4 100644 --- a/crates/core/src/contract.rs +++ b/crates/core/src/contract.rs @@ -1,4 +1,5 @@ use crate::runtime::ContractError as ContractRtError; +use either::Either; use freenet_stdlib::prelude::*; mod executor; @@ -12,7 +13,7 @@ pub(crate) use executor::{ }; pub(crate) use handler::{ contract_handler_channel, ClientResponses, ClientResponsesSender, ContractHandler, - ContractHandlerEvent, ContractHandlerToEventLoopChannel, EventId, NetEventListener, + ContractHandlerEvent, ContractHandlerToEventLoopChannel, EventId, NetEventListenerHalve, NetworkContractHandler, StoreResponse, }; #[cfg(test)] @@ -22,12 +23,14 @@ pub use executor::{Executor, ExecutorError, OperationMode}; use executor::ContractExecutor; +#[tracing::instrument(skip_all)] pub(crate) async fn contract_handling<'a, CH>(mut contract_handler: CH) -> Result<(), ContractError> where CH: ContractHandler + Send + 'static, { loop { let (id, event) = contract_handler.channel().recv_from_event_loop().await?; + tracing::debug!(%event, "Got contract handling event"); match event { ContractHandlerEvent::GetQuery { key, @@ -39,6 +42,7 @@ where .await { Ok((state, contract)) => { + tracing::debug!("Fetched contract {key}"); contract_handler .channel() .send_to_event_loop( @@ -54,7 +58,7 @@ where .await?; } Err(err) => { - tracing::warn!("error while executing get contract query: {err}"); + tracing::warn!("Error while executing get contract query: {err}"); contract_handler .channel() .send_to_event_loop( @@ -77,6 +81,7 @@ where .await?; } Err(err) => { + tracing::error!("Error while caching: {err}"); let err = ContractError::ContractRuntimeError(err); contract_handler .channel() @@ -86,29 +91,25 @@ where } } ContractHandlerEvent::PutQuery { - key: _key, - state: _state, + key, + state, + related_contracts, + parameters, } => { - // let _put_result = contract_handler - // .handle_request(ClientRequest::Put { - // contract: todo!(), - // state: _state, - // }.into()) - // .await - // .map(|r| { - // let _r = r.unwrap_put(); - // unimplemented!(); - // }); - // contract_handler - // .channel() - // .send_to_listener( - // _id, - // ContractHandlerEvent::PushResponse { - // new_value: put_result, - // }, - // ) - // .await?; - todo!("perform put request"); + let put_result = contract_handler + .executor() + .upsert_contract_state(key, Either::Left(state), related_contracts, parameters) + .await + .map_err(Into::into); + contract_handler + .channel() + .send_to_event_loop( + id, + ContractHandlerEvent::PutResponse { + new_value: put_result, + }, + ) + .await?; } _ => unreachable!(), } @@ -121,9 +122,9 @@ pub(crate) enum ContractError { ChannelDropped(Box), #[error("contract {0} not found in storage")] ContractNotFound(ContractKey), - #[error("")] - ContractRuntimeError(ContractRtError), #[error(transparent)] + ContractRuntimeError(ContractRtError), + #[error("{0}")] IOError(#[from] std::io::Error), #[error("no response received from handler")] NoEvHandlerResponse, diff --git a/crates/core/src/contract/executor.rs b/crates/core/src/contract/executor.rs index a708ca866..58319f760 100644 --- a/crates/core/src/contract/executor.rs +++ b/crates/core/src/contract/executor.rs @@ -124,14 +124,6 @@ pub(crate) fn executor_channel( (listener_halve, sender_halve) } -#[cfg(test)] -pub(crate) fn executor_channel_test() -> ( - ExecutorToEventLoopChannel, - ExecutorToEventLoopChannel, -) { - todo!() -} - impl ExecutorToEventLoopChannel { async fn send_to_event_loop(&mut self, message: T) -> Result<(), DynError> where @@ -197,8 +189,8 @@ struct GetContract { #[async_trait::async_trait] impl ComposeNetworkMessage for GetContract { - fn initiate_op(self, op_manager: &OpManager) -> operations::get::GetOp { - operations::get::start_op(self.key, self.fetch_contract, &op_manager.ring.peer_key) + fn initiate_op(self, _op_manager: &OpManager) -> operations::get::GetOp { + operations::get::start_op(self.key, self.fetch_contract) } async fn resume_op( @@ -237,6 +229,13 @@ pub(crate) trait ContractExecutor: Send + Sync + 'static { &mut self, contract: ContractContainer, ) -> Result<(), crate::runtime::ContractError>; + async fn upsert_contract_state( + &mut self, + key: ContractKey, + state: Either>, + related_contracts: RelatedContracts<'static>, + params: Option>, + ) -> Result; } /// A WASM executor which will run any contracts, delegates, etc. registered. @@ -338,8 +337,46 @@ impl Executor { } } -impl Executor { - pub async fn from_config(config: NodeConfig) -> Result { +impl Executor { + pub async fn new( + state_store: StateStore, + ctrl_handler: impl FnOnce(), + mode: OperationMode, + runtime: R, + ) -> Result { + ctrl_handler(); + + Ok(Self { + #[cfg(any( + all(feature = "local-mode", feature = "network-mode"), + all(not(feature = "local-mode"), not(feature = "network-mode")), + ))] + mode, + runtime, + state_store, + update_notifications: HashMap::default(), + subscriber_summaries: HashMap::default(), + delegate_attested_ids: HashMap::default(), + #[cfg(any( + not(feature = "local-mode"), + feature = "network-mode", + all(not(feature = "local-mode"), not(feature = "network-mode")) + ))] + event_loop_channel: None, + }) + } + + async fn get_stores( + config: &NodeConfig, + ) -> Result< + ( + ContractStore, + DelegateStore, + SecretsStore, + StateStore, + ), + DynError, + > { const MAX_SIZE: i64 = 10 * 1024 * 1024; const MAX_MEM_CACHE: u32 = 10_000_000; let static_conf = crate::config::Config::conf(); @@ -367,50 +404,26 @@ impl Executor { .unwrap_or_else(|| static_conf.secrets_dir()); let secret_store = SecretsStore::new(secrets_dir)?; + Ok((contract_store, delegate_store, secret_store, state_store)) + } +} + +impl Executor { + pub async fn from_config(config: NodeConfig) -> Result { + let (contract_store, delegate_store, secret_store, state_store) = + Self::get_stores(&config).await?; + let rt = Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(); Executor::new( - contract_store, - delegate_store, - secret_store, state_store, || { crate::util::set_cleanup_on_exit().unwrap(); }, OperationMode::Local, + rt, ) .await } - #[allow(unused_variables)] - pub async fn new( - contract_store: ContractStore, - delegate_store: DelegateStore, - secret_store: SecretsStore, - contract_state: StateStore, - ctrl_handler: impl FnOnce(), - mode: OperationMode, - ) -> Result { - ctrl_handler(); - - Ok(Self { - #[cfg(any( - all(feature = "local-mode", feature = "network-mode"), - all(not(feature = "local-mode"), not(feature = "network-mode")), - ))] - mode, - runtime: Runtime::build(contract_store, delegate_store, secret_store, false).unwrap(), - state_store: contract_state, - update_notifications: HashMap::default(), - subscriber_summaries: HashMap::default(), - delegate_attested_ids: HashMap::default(), - #[cfg(any( - not(feature = "local-mode"), - feature = "network-mode", - all(not(feature = "local-mode"), not(feature = "network-mode")) - ))] - event_loop_channel: None, - }) - } - pub fn register_contract_notifier( &mut self, key: ContractKey, @@ -1174,8 +1187,24 @@ impl Executor { #[cfg(test)] impl Executor { - pub async fn new_mock() -> Result { - todo!() + pub async fn new_mock(data_dir: &str) -> Result { + let tmp_path = std::env::temp_dir().join(format!("freenet-executor-{data_dir}")); + + let contracts_data_dir = tmp_path.join("contracts"); + std::fs::create_dir_all(&contracts_data_dir).expect("directory created"); + let contract_store = ContractStore::new(contracts_data_dir, u16::MAX as i64)?; + + // uses inmemory SQLite + let state_store = StateStore::new(Storage::new().await?, u16::MAX as u32).unwrap(); + + let executor = Executor::new( + state_store, + || {}, + OperationMode::Local, + super::MockRuntime { contract_store }, + ) + .await?; + Ok(executor) } pub async fn handle_request<'a>( @@ -1184,7 +1213,7 @@ impl Executor { _req: ClientRequest<'a>, _updates: Option>>, ) -> Response { - todo!() + unreachable!() } } @@ -1215,6 +1244,16 @@ impl ContractExecutor for Executor { ) -> Result<(), crate::runtime::ContractError> { self.runtime.contract_store.store_contract(contract) } + + async fn upsert_contract_state( + &mut self, + _key: ContractKey, + _state: Either>, + _related_contracts: RelatedContracts<'static>, + _params: Option>, + ) -> Result { + todo!() + } } #[cfg(test)] @@ -1254,12 +1293,35 @@ impl ContractExecutor for Executor { self.runtime.contract_store.store_contract(contract)?; Ok(()) } + + async fn upsert_contract_state( + &mut self, + key: ContractKey, + state: Either>, + _related_contracts: RelatedContracts<'static>, + params: Option>, + ) -> Result { + // todo: instead allow to perform mutations per contract based on incoming value so we can track + // state values over the network + match (state, params) { + (Either::Left(state), Some(params)) => { + self.state_store + .store(key, state.clone(), params.into_owned()) + .await?; + return Ok(state); + } + _ => unreachable!(), + } + } } #[cfg(test)] mod test { use super::*; - use crate::runtime::{ContractStore, StateStore}; + use crate::{ + contract::MockRuntime, + runtime::{ContractStore, StateStore}, + }; #[tokio::test(flavor = "multi_thread")] async fn local_node_handle() -> Result<(), Box> { @@ -1270,14 +1332,12 @@ mod test { let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE).unwrap(); let mut counter = 0; Executor::new( - contract_store, - DelegateStore::default(), - SecretsStore::default(), state_store, || { counter += 1; }, OperationMode::Local, + MockRuntime { contract_store }, ) .await .expect("local node with handle"); diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 311916723..e259404bb 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -1,6 +1,5 @@ #![allow(unused)] // FIXME: remove this - -use std::collections::{BTreeMap, VecDeque}; +use std::collections::BTreeMap; use std::hash::Hash; use std::marker::PhantomData; use std::sync::atomic::{AtomicU64, Ordering::SeqCst}; @@ -19,15 +18,7 @@ use super::{ }; use crate::client_events::HostResult; use crate::message::Transaction; -use crate::node::OpManager; -use crate::{ - client_events::ClientId, - node::NodeConfig, - runtime::{ContractStore, Runtime, StateStorage, StateStore}, - DynError, -}; - -pub const MAX_MEM_CACHE: i64 = 10_000_000; +use crate::{client_events::ClientId, node::NodeConfig, runtime::Runtime, DynError}; pub(crate) struct ClientResponses(UnboundedReceiver<(ClientId, HostResult)>); @@ -141,19 +132,19 @@ impl ContractHandler for NetworkContractHandler { #[cfg(test)] impl ContractHandler for NetworkContractHandler { - type Builder = (); + type Builder = String; type ContractExecutor = Executor; fn build( channel: ContractHandlerToEventLoopChannel, _executor_request_sender: ExecutorToEventLoopChannel, - _builder: Self::Builder, + data_dir: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { - async { - let executor = Executor::new_mock().await?; + async move { + let executor = Executor::new_mock(&data_dir).await?; Ok(Self { executor, channel }) } .boxed() @@ -188,12 +179,23 @@ impl ContractHandler for NetworkContractHandler { pub(crate) struct EventId { id: u64, client_id: Option, + transaction: Option, } impl EventId { pub fn client_id(&self) -> Option { self.client_id } + + pub fn transaction(&self) -> Option { + self.transaction + } + + // FIXME: this should be used somewhere to inform than an event is pending + // a transaction resolution + pub fn with_transaction(&mut self, transaction: Transaction) { + self.transaction = Some(transaction); + } } impl PartialEq for EventId { @@ -218,17 +220,17 @@ pub(crate) struct ContractHandlerToEventLoopChannel { } pub(crate) struct ContractHandlerHalve; -pub(crate) struct NetEventListener; +pub(crate) struct NetEventListenerHalve; mod sealed { - use super::{ContractHandlerHalve, NetEventListener}; + use super::{ContractHandlerHalve, NetEventListenerHalve}; pub(crate) trait ChannelHalve {} impl ChannelHalve for ContractHandlerHalve {} - impl ChannelHalve for NetEventListener {} + impl ChannelHalve for NetEventListenerHalve {} } pub(crate) fn contract_handler_channel() -> ( - ContractHandlerToEventLoopChannel, + ContractHandlerToEventLoopChannel, ContractHandlerToEventLoopChannel, ) { let (notification_tx, notification_channel) = mpsc::unbounded_channel(); @@ -258,7 +260,7 @@ static EV_ID: AtomicU64 = AtomicU64::new(0); // kind of event and can be optimized on a case basis const CH_EV_RESPONSE_TIME_OUT: Duration = Duration::from_secs(300); -impl ContractHandlerToEventLoopChannel { +impl ContractHandlerToEventLoopChannel { /// Send an event to the contract handler and receive a response event if successful. pub async fn send_to_handler( &mut self, @@ -289,8 +291,7 @@ impl ContractHandlerToEventLoopChannel { } } - // todo: use - pub async fn recv_from_handler(&mut self) -> (EventId, ContractHandlerEvent) { + pub async fn recv_from_handler(&mut self) -> EventId { todo!() } } @@ -318,6 +319,7 @@ impl ContractHandlerToEventLoopChannel { EventId { id: msg.id, client_id: msg.client_id, + transaction: None, }, msg.ev, )); @@ -344,6 +346,8 @@ pub(crate) enum ContractHandlerEvent { PutQuery { key: ContractKey, state: WrappedState, + related_contracts: RelatedContracts<'static>, + parameters: Option>, }, /// The response to a push query. PutResponse { @@ -365,9 +369,47 @@ pub(crate) enum ContractHandlerEvent { CacheResult(Result<(), ContractError>), } -impl ContractHandlerEvent { - pub async fn into_network_op(self, op_manager: &OpManager) -> Transaction { - todo!() +impl std::fmt::Display for ContractHandlerEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ContractHandlerEvent::PutQuery { + key, parameters, .. + } => { + if let Some(params) = parameters { + write!(f, "put query {{ {key}, params: {:?} }}", params.as_ref()) + } else { + write!(f, "put query {{ {key} }}") + } + } + ContractHandlerEvent::PutResponse { new_value } => match new_value { + Ok(v) => { + write!(f, "put query response {{ {v} }}",) + } + Err(e) => { + write!(f, "put query failed {{ {e} }}",) + } + }, + ContractHandlerEvent::GetQuery { + key, + fetch_contract, + } => { + write!(f, "get query {{ {key}, fetch contract: {fetch_contract} }}",) + } + ContractHandlerEvent::GetResponse { key, response } => match response { + Ok(_) => { + write!(f, "get query response {{ {key} }}",) + } + Err(_) => { + write!(f, "get query failed {{ {key} }}",) + } + }, + ContractHandlerEvent::Cache(container) => { + write!(f, "caching {{ {} }}", container.key()) + } + ContractHandlerEvent::CacheResult(r) => { + write!(f, "caching result {{ {} }}", r.is_ok()) + } + } } } @@ -375,56 +417,44 @@ impl ContractHandlerEvent { pub mod test { use std::sync::Arc; - use crate::runtime::ContractStore; - use freenet_stdlib::{ - client_api::{ClientRequest, HostResponse}, - prelude::*, - }; + use freenet_stdlib::prelude::*; use super::*; - use crate::{config::GlobalExecutor, contract::MockRuntime}; + use crate::config::GlobalExecutor; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn channel_test() -> Result<(), anyhow::Error> { let (mut send_halve, mut rcv_halve) = contract_handler_channel(); + let contract = ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( + Arc::new(ContractCode::from(vec![0, 1, 2, 3])), + Parameters::from(vec![4, 5]), + ))); + + let contract_cp = contract.clone(); let h = GlobalExecutor::spawn(async move { - let contract = - ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(vec![0, 1, 2, 3])), - Parameters::from(vec![]), - ))); send_halve - .send_to_handler(ContractHandlerEvent::Cache(contract), None) + .send_to_handler(ContractHandlerEvent::Cache(contract_cp), None) .await }); - let (id, ev) = tokio::time::timeout(Duration::from_millis(100), rcv_halve.recv_from_event_loop()) .await??; - if let ContractHandlerEvent::Cache(contract) = ev { - let data: Vec = contract.data(); - assert_eq!(data, vec![0, 1, 2, 3]); - let contract = ContractContainer::Wasm(ContractWasmAPIVersion::V1( - WrappedContract::new(Arc::new(ContractCode::from(data)), Parameters::from(vec![])), - )); - tokio::time::timeout( - Duration::from_millis(100), - rcv_halve.send_to_event_loop(id, ContractHandlerEvent::Cache(contract)), - ) - .await??; - } else { + let ContractHandlerEvent::Cache(contract) = ev else { anyhow::bail!("invalid event"); - } - - if let ContractHandlerEvent::Cache(contract) = h.await?? { - let data: Vec = contract.data(); - assert_eq!(data, vec![0, 1, 2, 3]); - } else { + }; + assert_eq!(contract.data(), vec![0, 1, 2, 3]); + + tokio::time::timeout( + Duration::from_millis(100), + rcv_halve.send_to_event_loop(id, ContractHandlerEvent::Cache(contract)), + ) + .await??; + let ContractHandlerEvent::Cache(contract) = h.await?? else { anyhow::bail!("invalid event!"); - } + }; + assert_eq!(contract.data(), vec![0, 1, 2, 3]); Ok(()) } diff --git a/crates/core/src/contract/in_memory.rs b/crates/core/src/contract/in_memory.rs index f390c7625..1fc7149ac 100644 --- a/crates/core/src/contract/in_memory.rs +++ b/crates/core/src/contract/in_memory.rs @@ -1,7 +1,4 @@ -use crate::{ - client_events::ClientId, - runtime::{ContractStore, StateStorage, StateStore}, -}; +use crate::{client_events::ClientId, runtime::ContractStore}; use freenet_stdlib::{ client_api::{ClientError, ClientRequest, HostResponse}, prelude::WrappedContract, @@ -12,63 +9,44 @@ use tokio::sync::mpsc::UnboundedSender; use super::{ executor::{ExecutorHalve, ExecutorToEventLoopChannel}, handler::{ContractHandler, ContractHandlerHalve, ContractHandlerToEventLoopChannel}, - storages::in_memory::MemKVStore, Executor, }; -use crate::{config::Config, DynError}; +use crate::DynError; pub(crate) struct MockRuntime { pub contract_store: ContractStore, } -pub(crate) struct MemoryContractHandler -where - KVStore: StateStorage, -{ +pub(crate) struct MemoryContractHandler { channel: ContractHandlerToEventLoopChannel, - _kv_store: StateStore, - _runtime: MockRuntime, + runtime: Executor, } -impl MemoryContractHandler -where - KVStore: StateStorage + Send + Sync + 'static, - ::Error: Into>, -{ - const MAX_MEM_CACHE: i64 = 10_000_000; - - pub fn new( +impl MemoryContractHandler { + pub async fn new( channel: ContractHandlerToEventLoopChannel, - kv_store: KVStore, + data_dir: &str, ) -> Self { MemoryContractHandler { channel, - _kv_store: StateStore::new(kv_store, 10_000_000).unwrap(), - _runtime: MockRuntime { - contract_store: ContractStore::new( - Config::conf().contracts_dir(), - Self::MAX_MEM_CACHE, - ) - .unwrap(), - }, + runtime: Executor::new_mock(data_dir).await.unwrap(), } } } impl ContractHandler for MemoryContractHandler { - type Builder = (); + type Builder = String; type ContractExecutor = Executor; fn build( channel: ContractHandlerToEventLoopChannel, _executor_request_sender: ExecutorToEventLoopChannel, - _config: Self::Builder, + config: Self::Builder, ) -> BoxFuture<'static, Result> where Self: Sized + 'static, { - let store = MemKVStore::new(); - async move { Ok(MemoryContractHandler::new(channel, store)) }.boxed() + async move { Ok(MemoryContractHandler::new(channel, &config).await) }.boxed() } fn channel(&mut self) -> &mut ContractHandlerToEventLoopChannel { @@ -81,15 +59,14 @@ impl ContractHandler for MemoryContractHandler { _client_id: ClientId, _updates: Option>>, ) -> BoxFuture<'static, Result> { - todo!() + unreachable!() } fn executor(&mut self) -> &mut Self::ContractExecutor { - todo!() + &mut self.runtime } } -#[ignore] #[test] fn serialization() -> Result<(), anyhow::Error> { let bytes = crate::util::test::random_bytes_1024(); diff --git a/crates/core/src/contract/storages/in_memory.rs b/crates/core/src/contract/storages/in_memory.rs deleted file mode 100644 index c9c66280e..000000000 --- a/crates/core/src/contract/storages/in_memory.rs +++ /dev/null @@ -1,41 +0,0 @@ -use dashmap::DashMap; -use freenet_stdlib::prelude::*; - -use crate::runtime::StateStorage; - -#[derive(Default, Clone)] -pub(crate) struct MemKVStore(DashMap); - -#[async_trait::async_trait] -impl StateStorage for MemKVStore { - type Error = String; - - async fn store(&mut self, _key: ContractKey, _state: WrappedState) -> Result<(), Self::Error> { - todo!() - } - - async fn get(&self, _key: &ContractKey) -> Result, Self::Error> { - todo!() - } - - async fn store_params( - &mut self, - _key: ContractKey, - _state: Parameters<'static>, - ) -> Result<(), Self::Error> { - todo!() - } - - async fn get_params<'a>( - &'a self, - _key: &'a ContractKey, - ) -> Result>, Self::Error> { - todo!() - } -} - -impl MemKVStore { - pub fn new() -> Self { - Self::default() - } -} diff --git a/crates/core/src/contract/storages/mod.rs b/crates/core/src/contract/storages/mod.rs index 48bc02864..e55e61c94 100644 --- a/crates/core/src/contract/storages/mod.rs +++ b/crates/core/src/contract/storages/mod.rs @@ -13,6 +13,3 @@ use self::rocks_db::RocksDb; #[cfg(all(feature = "rocks_db", not(feature = "sqlite")))] pub type Storage = RocksDb; - -#[cfg(test)] -pub(crate) mod in_memory; diff --git a/crates/core/src/contract/storages/rocks_db.rs b/crates/core/src/contract/storages/rocks_db.rs index 90850728f..5c77e20d6 100644 --- a/crates/core/src/contract/storages/rocks_db.rs +++ b/crates/core/src/contract/storages/rocks_db.rs @@ -92,94 +92,3 @@ impl StateStorage for RocksDb { } } } - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use freenet_stdlib::{client_api::ContractRequest, prelude::*}; - - use crate::{ - client_events::ClientId, - contract::{ - contract_handler_channel, executor::executor_channel_test, ContractHandler, - MockRuntime, NetworkContractHandler, - }, - DynError, - }; - - // Prepare and get handler for rocksdb - async fn get_handler() -> Result, DynError> { - let (_, ch_handler) = contract_handler_channel(); - let (_, executor_sender) = executor_channel_test(); - let handler = NetworkContractHandler::build(ch_handler, executor_sender, ()).await?; - Ok(handler) - } - - #[ignore] - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn contract_handler() -> Result<(), DynError> { - // Create a rocksdb handler and initialize the database - let mut handler = get_handler().await?; - - // Generate a contract - let contract_bytes = b"Test contract value".to_vec(); - let contract: ContractContainer = - ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(contract_bytes.clone())), - Parameters::from(vec![]), - ))); - - // Get contract parts - let state = WrappedState::new(contract_bytes.clone()); - handler - .handle_request( - ContractRequest::Put { - contract: contract.clone(), - state: state.clone(), - related_contracts: Default::default(), - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_put(); - let (get_result_value, _) = handler - .handle_request( - ContractRequest::Get { - key: contract.key().clone(), - fetch_contract: false, - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_get(); - assert_eq!(state, get_result_value); - - // Update the contract state with a new delta - let delta = StateDelta::from(b"New test contract value".to_vec()); - handler - .handle_request( - ContractRequest::Update { - key: contract.key().clone(), - data: delta.into(), - } - .into(), - ClientId::FIRST, - None, - ) - .await?; - // let (new_get_result_value, _) = handler - // .handle_request(ContractOps::Get { - // key: *contract.key(), - // contract: false, - // }) - // .await? - // .unwrap_summary(); - // assert_eq!(delta, new_get_result_value); - todo!("get summary and compare with delta"); - } -} diff --git a/crates/core/src/contract/storages/sqlite.rs b/crates/core/src/contract/storages/sqlite.rs index 253181b5f..9d25f6c31 100644 --- a/crates/core/src/contract/storages/sqlite.rs +++ b/crates/core/src/contract/storages/sqlite.rs @@ -129,100 +129,8 @@ pub enum SqlDbError { SqliteError(#[from] sqlx::Error), #[error(transparent)] RuntimeError(#[from] ContractError), - #[error(transparent)] + #[error("{0}")] IOError(#[from] std::io::Error), #[error(transparent)] StateStore(#[from] StateStoreError), } - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use freenet_stdlib::client_api::ContractRequest; - use freenet_stdlib::prelude::*; - - use crate::{ - client_events::ClientId, - contract::{ - contract_handler_channel, executor::executor_channel_test, ContractHandler, - MockRuntime, NetworkContractHandler, - }, - DynError, - }; - - // Prepare and get handler for an in-memory sqlite db - async fn get_handler() -> Result, DynError> { - let (_, ch_handler) = contract_handler_channel(); - let (_, executor_sender) = executor_channel_test(); - let handler = NetworkContractHandler::build(ch_handler, executor_sender, ()).await?; - Ok(handler) - } - - #[ignore] - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn contract_handler() -> Result<(), DynError> { - // Create a sqlite handler and initialize the database - let mut handler = get_handler().await?; - - // Generate a contract - let contract_bytes = b"test contract value".to_vec(); - let contract: ContractContainer = - ContractContainer::Wasm(ContractWasmAPIVersion::V1(WrappedContract::new( - Arc::new(ContractCode::from(contract_bytes.clone())), - Parameters::from(vec![]), - ))); - - // Get contract parts - let state = WrappedState::new(contract_bytes.clone()); - handler - .handle_request( - ContractRequest::Put { - contract: contract.clone(), - state: state.clone(), - related_contracts: Default::default(), - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_put(); - let (get_result_value, _) = handler - .handle_request( - ContractRequest::Get { - key: contract.key().clone(), - fetch_contract: false, - } - .into(), - ClientId::FIRST, - None, - ) - .await? - .unwrap_get(); - assert_eq!(state, get_result_value); - - // Update the contract state with a new delta - let delta = StateDelta::from(b"New test contract value".to_vec()); - handler - .handle_request( - ContractRequest::Update { - key: contract.key().clone(), - data: delta.into(), - } - .into(), - ClientId::FIRST, - None, - ) - .await?; - // let (new_get_result_value, _) = handler - // .handle_request(ContractOps::Get { - // key: *contract.key(), - // contract: false, - // }) - // .await? - // .unwrap_summary(); - // assert_eq!(delta, new_get_result_value); - todo!("get summary and compare with delta"); - } -} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 02ddb6b48..e26ed2240 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -24,7 +24,6 @@ pub mod local_node { /// Exports to build a running network simulation. pub mod network_sim { - // todo: streamline this use super::*; pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; pub use node::{InitPeerNode, NodeBuilder, NodeConfig}; @@ -37,5 +36,5 @@ pub mod dev_tool { pub use crate::config::Config; pub use client_events::{ClientEventsProxy, ClientId, OpenRequest}; pub use contract::{storages::Storage, Executor, OperationMode}; - pub use runtime::{ContractStore, DelegateStore, SecretsStore, StateStore}; + pub use runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 4f4bd3b7a..b35c821f3 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -3,23 +3,19 @@ use std::{fmt::Display, time::Duration}; use serde::{Deserialize, Serialize}; -use uuid::{ - v1::{Context, Timestamp}, - Uuid, -}; +use ulid::Ulid; use crate::{ node::{ConnectionError, PeerKey}, operations::{ - get::GetMsg, join_ring::JoinRingMsg, put::PutMsg, subscribe::SubscribeMsg, - update::UpdateMsg, + connect::ConnectMsg, get::GetMsg, put::PutMsg, subscribe::SubscribeMsg, update::UpdateMsg, }, ring::{Location, PeerKeyLocation}, }; pub(crate) use sealed_msg_type::{TransactionType, TransactionTypeId}; /// An transaction is a unique, universal and efficient identifier for any -/// roundtrip transaction as it is broadcasted around the F2 network. +/// roundtrip transaction as it is broadcasted around the Freenet network. /// /// The identifier conveys all necessary information to identify and classify the /// transaction: @@ -31,27 +27,14 @@ pub(crate) use sealed_msg_type::{TransactionType, TransactionTypeId}; /// A transaction may span different messages sent across the network. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy)] pub(crate) struct Transaction { - /// UUID V1, can retrieve timestamp information later to check for possible out of time - /// expired transactions which have been clean up already. - id: Uuid, + id: Ulid, ty: TransactionTypeId, } -static UUID_CONTEXT: Context = Context::new(14); - impl Transaction { - pub fn new(ty: TransactionTypeId, initial_peer: &PeerKey) -> Transaction { - // using v1 UUID to keep to keep track of the creation ts - let ts: Timestamp = uuid::timestamp::Timestamp::now(&UUID_CONTEXT); - - // event in the net this UUID should be unique since peer keys are unique - // however some id collision may be theoretically possible if two transactions - // are created at the same exact time and the first 6 bytes of the key coincide; - // in practice the chance of this happening is astronomically low - - let b = &mut [0; 6]; - b.copy_from_slice(&initial_peer.to_bytes()[0..6]); - let id = Uuid::new_v1(ts, b); + pub fn new() -> Transaction { + let ty = ::tx_type_id(); + let id = Ulid::new(); // 3 word size for 64-bits platforms Self { id, ty } @@ -103,7 +86,7 @@ mod sealed_msg_type { #[repr(u8)] #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] pub(crate) enum TransactionType { - JoinRing, + Connect, Put, Get, Subscribe, @@ -111,6 +94,19 @@ mod sealed_msg_type { Canceled, } + impl Display for TransactionType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TransactionType::Connect => write!(f, "join ring"), + TransactionType::Put => write!(f, "put"), + TransactionType::Get => write!(f, "get"), + TransactionType::Subscribe => write!(f, "subscribe"), + TransactionType::Update => write!(f, "update"), + TransactionType::Canceled => write!(f, "canceled"), + } + } + } + macro_rules! transaction_type_enumeration { (decl struct { $( $var:tt -> $ty:tt),+ }) => { $( @@ -130,7 +126,7 @@ mod sealed_msg_type { } transaction_type_enumeration!(decl struct { - JoinRing -> JoinRingMsg, + Connect -> ConnectMsg, Put -> PutMsg, Get -> GetMsg, Subscribe -> SubscribeMsg, @@ -138,19 +134,23 @@ mod sealed_msg_type { }); } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize)] pub(crate) enum Message { - JoinRing(JoinRingMsg), + Connect(ConnectMsg), Put(PutMsg), Get(GetMsg), Subscribe(SubscribeMsg), Update(UpdateMsg), - /// Failed a transaction, informing of cancellation. - Canceled(Transaction), + /// Failed a transaction, informing of abortion. + Aborted(Transaction), } pub(crate) trait InnerMessage: Into { fn id(&self) -> &Transaction; + + fn target(&self) -> Option<&PeerKeyLocation>; + + fn terminal(&self) -> bool; } /// Internal node events emitted to the event loop. @@ -189,24 +189,24 @@ impl Message { pub fn id(&self) -> &Transaction { use Message::*; match self { - JoinRing(op) => op.id(), + Connect(op) => op.id(), Put(op) => op.id(), Get(op) => op.id(), Subscribe(op) => op.id(), - Update(_op) => todo!(), - Canceled(tx) => tx, + Update(op) => op.id(), + Aborted(tx) => tx, } } pub fn target(&self) -> Option<&PeerKeyLocation> { use Message::*; match self { - JoinRing(op) => op.target(), + Connect(op) => op.target(), Put(op) => op.target(), Get(op) => op.target(), Subscribe(op) => op.target(), - Update(_op) => todo!(), - Canceled(_) => None, + Update(op) => op.target(), + Aborted(_) => None, } } @@ -214,18 +214,18 @@ impl Message { pub fn terminal(&self) -> bool { use Message::*; match self { - JoinRing(op) => op.terminal(), + Connect(op) => op.terminal(), Put(op) => op.terminal(), Get(op) => op.terminal(), Subscribe(op) => op.terminal(), - Update(_op) => todo!(), - Canceled(_) => true, + Update(op) => op.terminal(), + Aborted(_) => true, } } pub fn track_stats(&self) -> bool { use Message::*; - !matches!(self, JoinRing(_) | Subscribe(_) | Canceled(_)) + !matches!(self, Connect(_) | Subscribe(_) | Aborted(_)) } } @@ -234,12 +234,12 @@ impl Display for Message { use Message::*; write!(f, "Message {{")?; match self { - JoinRing(msg) => msg.fmt(f)?, + Connect(msg) => msg.fmt(f)?, Put(msg) => msg.fmt(f)?, Get(msg) => msg.fmt(f)?, Subscribe(msg) => msg.fmt(f)?, - Update(_op) => todo!(), - Canceled(msg) => msg.fmt(f)?, + Update(msg) => msg.fmt(f)?, + Aborted(msg) => msg.fmt(f)?, }; write!(f, "}}") } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index aa71a80ec..6dd9b3cb5 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -15,8 +15,10 @@ use std::{ time::Duration, }; +use either::Either; use freenet_stdlib::client_api::{ClientRequest, ContractRequest}; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; +use tracing::Instrument; #[cfg(test)] use self::in_memory_impl::NodeInMemory; @@ -29,29 +31,26 @@ use crate::{ ClientResponses, ClientResponsesSender, ContractError, ExecutorToEventLoopChannel, NetworkContractHandler, NetworkEventListenerHalve, OperationMode, }, - message::{InnerMessage, Message, Transaction, TransactionType, TxType}, + message::{Message, Transaction, TransactionType}, operations::{ - get, - join_ring::{self, JoinRingMsg, JoinRingOp}, - put, subscribe, OpEnum, OpError, OpOutcome, + connect::{self, ConnectMsg, ConnectOp}, + get, put, subscribe, OpEnum, OpError, OpOutcome, }, ring::{Location, PeerKeyLocation}, router::{RouteEvent, RouteOutcome}, - util::{ExponentialBackoff, IterExt}, + util::ExponentialBackoff, }; use crate::operations::handle_op_request; pub(crate) use conn_manager::{ConnectionBridge, ConnectionError}; -#[cfg(test)] -pub(crate) use event_log::test_utils::TestEventListener; pub(crate) use event_log::{EventLogRegister, EventRegister}; -pub(crate) use op_state::OpManager; +pub(crate) use op_state_manager::{OpManager, OpNotAvailable}; mod conn_manager; mod event_log; #[cfg(test)] mod in_memory_impl; -mod op_state; +mod op_state_manager; mod p2p_impl; #[cfg(test)] pub(crate) mod tests; @@ -77,7 +76,6 @@ pub struct Node(NodeP2P); impl Node { pub async fn run(self) -> Result<(), Box> { - //TODO: Initialize tracer self.0.run_node().await?; Ok(()) } @@ -101,6 +99,10 @@ pub struct NodeBuilder { pub(crate) local_ip: Option, /// socket port to bind to the listener pub(crate) local_port: Option, + /// IP dialers should connect to + pub(crate) public_ip: Option, + /// socket port dialers should connect to + pub(crate) public_port: Option, /// At least an other running listener node is required for joining the network. /// Not necessary if this is an initial node. pub(crate) remote_nodes: Vec, @@ -125,6 +127,8 @@ impl NodeBuilder { remote_nodes: Vec::with_capacity(1), local_ip: None, local_port: None, + public_ip: None, + public_port: None, location: None, max_hops_to_live: None, rnd_if_htl_above: None, @@ -284,29 +288,24 @@ async fn join_ring_request( where CM: ConnectionBridge + Send + Sync, { - let tx_id = Transaction::new(::tx_type_id(), &peer_key); + let tx_id = Transaction::new::(); let mut op = - join_ring::initial_request(peer_key, *gateway, op_storage.ring.max_hops_to_live, tx_id); + connect::initial_request(peer_key, *gateway, op_storage.ring.max_hops_to_live, tx_id); if let Some(mut backoff) = backoff { // backoff to retry later in case it failed - tracing::warn!( - "Performing a new join attempt, attempt number: {}", - backoff.retries() - ); + tracing::warn!("Performing a new join, attempt {}", backoff.retries() + 1); if backoff.sleep_async().await.is_none() { tracing::error!("Max number of retries reached"); - return Err(OpError::MaxRetriesExceeded( - tx_id, - format!("{:?}", tx_id.tx_type()), - )); + return Err(OpError::MaxRetriesExceeded(tx_id, tx_id.tx_type())); } op.backoff = Some(backoff); } - join_ring::join_ring_request(tx_id, op_storage, conn_manager, op).await?; + connect::connect_request(tx_id, op_storage, conn_manager, op).await?; Ok(()) } /// Process client events. +#[tracing::instrument(skip_all)] async fn client_event_handling( op_storage: Arc, mut client_events: ClientEv, @@ -318,7 +317,10 @@ async fn client_event_handling( tokio::select! { client_request = client_events.recv() => { let req = match client_request { - Ok(req) => req, + Ok(req) => { + tracing::debug!(%req, "got client request event"); + req + } Err(err) => { tracing::debug!(error = %err, "client error"); continue; @@ -332,6 +334,9 @@ async fn client_event_handling( } res = client_responses.recv() => { if let Some((cli_id, res)) = res { + if let Ok(res) = &res { + tracing::debug!(%res, "sending client response"); + } if let Err(err) = client_events.send(cli_id, res).await { tracing::error!("channel closed: {err}"); break; @@ -345,9 +350,9 @@ async fn client_event_handling( #[inline] async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc) { // this will indirectly start actions on the local contract executor - let op_storage_cp = op_storage.clone(); let fut = async move { let client_id = request.client_id; + let mut missing_contract = false; match *request.request { ClientRequest::ContractOp(ops) => match ops { ContractRequest::Put { @@ -358,18 +363,17 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc, op_storage: Arc { // Initialize a subscribe op. loop { - // FIXME: this will block the event loop until the subscribe op succeeds - // instead the op should be deferred for later execution - let op = subscribe::start_op(key.clone(), &op_storage_cp.ring.peer_key); - match subscribe::request_subscribe(&op_storage_cp, op, Some(client_id)) - .await - { - Err(OpError::ContractError(ContractError::ContractNotFound(key))) => { - tracing::warn!("Trying to subscribe to a contract not present: {}, requesting it first", key); - let get_op = - get::start_op(key.clone(), true, &op_storage_cp.ring.peer_key); + let op = subscribe::start_op(key.clone()); + match subscribe::request_subscribe(&op_storage, op, Some(client_id)).await { + Err(OpError::ContractError(ContractError::ContractNotFound(key))) + if !missing_contract => + { + tracing::info!("Trying to subscribe to a contract not present: {key}, requesting it first"); + missing_contract = true; + let get_op = get::start_op(key.clone(), true); if let Err(err) = - get::request_get(&op_storage_cp, get_op, Some(client_id)).await + get::request_get(&op_storage, get_op, Some(client_id)).await { - tracing::error!("Failed getting the contract `{}` while previously trying to subscribe; bailing: {}", key, err); - tokio::time::sleep(Duration::from_secs(5)).await; + tracing::error!("Failed getting the contract `{key}` while previously trying to subscribe; bailing: {err}"); + break; } } + Err(OpError::ContractError(ContractError::ContractNotFound(_))) => { + tracing::warn!("Still waiting for {key} contract"); + tokio::time::sleep(Duration::from_secs(2)).await + } Err(err) => { tracing::error!("{}", err); break; } - Ok(()) => break, + Ok(()) => { + if missing_contract { + tracing::debug!( + "Got back the missing contract ({key}) while subscribing" + ); + } + tracing::debug!("Starting subscribe request to {key}"); + break; + } } } - todo!() } _ => { tracing::error!("op not supported"); @@ -431,21 +444,25 @@ async fn process_open_request(request: OpenRequest<'static>, op_storage: Arc { tracing::debug!( - concat!("Handling ", $op, " get request @ {} (tx: {})"), + tx = %$id, + concat!("Handling ", $op, " request @ {}"), $op_storage.ring.peer_key, - $id ); }; } async fn report_result( + tx: Option, op_result: Result, OpError>, op_storage: &OpManager, executor_callback: Option>, @@ -477,12 +494,13 @@ async fn report_result( payload_transfer_time, }, }; - if let Err(err) = event_listener - .event_received(EventLog::route_event(op_res.id(), op_storage, &event)) - .await - { - tracing::warn!("failed logging event: {err}"); - } + event_listener + .register_events(Either::Left(EventLog::route_event( + op_res.id(), + op_storage, + &event, + ))) + .await; op_storage.ring.routing_finished(event); } // todo: handle failures, need to track timeouts and other potential failures @@ -504,11 +522,30 @@ async fn report_result( } Ok(None) => {} Err(err) => { - tracing::debug!("Finished tx w/ error: {}", err) + // just mark the operation as completed so no redundant messages are processed for this transaction anymore + if let Some(tx) = tx { + op_storage.completed(tx); + } + tracing::debug!("Finished transaction with error: {err}") } } } +macro_rules! handle_op_not_available { + ($op_result:ident) => { + if let Err(OpError::OpNotAvailable(state)) = &$op_result { + match state { + OpNotAvailable::Running => { + tokio::time::sleep(Duration::from_micros(1_000)).await; + continue; + } + OpNotAvailable::Completed => return, + } + } + }; +} + +#[tracing::instrument(name = "process_network_message", skip_all)] async fn process_message( msg: Result, op_storage: Arc, @@ -523,90 +560,99 @@ async fn process_message( let cli_req = client_id.zip(client_req_handler_callback); match msg { Ok(msg) => { - if let Err(err) = event_listener - .event_received(EventLog::from_msg(&msg, &op_storage)) - .await - { - tracing::warn!("failed logging event: {err}"); - } - match msg { - Message::JoinRing(op) => { - log_handling_msg!("join", op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; - } - Message::Put(op) => { - log_handling_msg!("put", *op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; - } - Message::Get(op) => { - log_handling_msg!("get", op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; - } - Message::Subscribe(op) => { - log_handling_msg!("subscribe", op.id(), op_storage); - let op_result = handle_op_request::( - &op_storage, - &mut conn_manager, - op, - client_id, - ) - .await; - report_result( - op_result, - &op_storage, - executor_callback, - cli_req, - &mut event_listener, - ) - .await; + let tx = Some(*msg.id()); + event_listener + .register_events(EventLog::from_inbound_msg(&msg, &op_storage)) + .await; + loop { + match &msg { + Message::Connect(op) => { + // log_handling_msg!("join", op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + Message::Put(op) => { + // log_handling_msg!("put", *op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + Message::Get(op) => { + // log_handling_msg!("get", op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + Message::Subscribe(op) => { + // log_handling_msg!("subscribe", op.id(), op_storage); + let op_result = handle_op_request::( + &op_storage, + &mut conn_manager, + op, + client_id, + ) + .await; + handle_op_not_available!(op_result); + break report_result( + tx, + op_result, + &op_storage, + executor_callback, + cli_req, + &mut event_listener, + ) + .await; + } + _ => break, } - _ => {} } } Err(err) => { report_result( + None, Err(err.into()), &op_storage, executor_callback, @@ -621,59 +667,30 @@ async fn process_message( async fn handle_cancelled_op( tx: Transaction, peer_key: PeerKey, - gateways: impl Iterator, op_storage: &OpManager, conn_manager: &mut CM, ) -> Result<(), OpError> where CM: ConnectionBridge + Send + Sync, { - tracing::warn!("Failed tx `{}`, potentially attempting a retry", tx); - match tx.tx_type() { - TransactionType::JoinRing => { - const MSG: &str = "Fatal error: unable to connect to the network"; - // the attempt to join the network failed, this could be a fatal error since the node - // is useless without connecting to the network, we will retry with exponential backoff - match op_storage.pop(&tx) { - Some(OpEnum::JoinRing(op)) if op.has_backoff() => { - if let JoinRingOp { - backoff: Some(backoff), - gateway, - .. - } = *op - { - if cfg!(test) { - join_ring_request(None, peer_key, &gateway, op_storage, conn_manager) - .await?; - } else { - join_ring_request( - Some(backoff), - peer_key, - &gateway, - op_storage, - conn_manager, - ) - .await?; - } - } - } - None | Some(OpEnum::JoinRing(_)) => { - let rand_gw = gateways - .shuffle() - .take(1) - .next() - .expect("at least one gateway"); - if !cfg!(test) { - tracing::error!("{}", MSG); - } else { - tracing::debug!("{}", MSG); - } - join_ring_request(None, peer_key, rand_gw, op_storage, conn_manager).await?; - } - _ => {} + if let TransactionType::Connect = tx.tx_type() { + // the attempt to join the network failed, this could be a fatal error since the node + // is useless without connecting to the network, we will retry with exponential backoff + match op_storage.pop(&tx) { + Ok(Some(OpEnum::Connect(op))) if op.has_backoff() => { + let ConnectOp { + gateway, backoff, .. + } = *op; + let backoff = backoff.expect("infallible"); + tracing::warn!("Retry connecting to gateway {}", gateway.peer); + join_ring_request(Some(backoff), peer_key, &gateway, op_storage, conn_manager) + .await?; } + Ok(Some(OpEnum::Connect(_))) => { + return Err(OpError::MaxRetriesExceeded(tx, tx.tx_type())); + } + _ => {} } - _ => unreachable!(), } Ok(()) } @@ -688,6 +705,7 @@ impl PeerKey { PeerKey::from(Keypair::generate_ed25519().public()) } + #[cfg(test)] pub fn to_bytes(self) -> Vec { self.0.to_bytes() } diff --git a/crates/core/src/node/conn_manager/in_memory.rs b/crates/core/src/node/conn_manager/in_memory.rs index ca62c9144..8953afc49 100644 --- a/crates/core/src/node/conn_manager/in_memory.rs +++ b/crates/core/src/node/conn_manager/in_memory.rs @@ -8,46 +8,54 @@ use std::{ use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; -use parking_lot::Mutex; -use rand::{prelude::StdRng, thread_rng, Rng, SeedableRng}; +use rand::{prelude::StdRng, seq::SliceRandom, Rng, SeedableRng}; +use tokio::sync::Mutex; use super::{ConnectionBridge, ConnectionError, PeerKey}; -use crate::{config::GlobalExecutor, message::Message}; +use crate::{ + config::GlobalExecutor, + message::Message, + node::{event_log::EventLog, EventLogRegister, OpManager}, +}; static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = OnceCell::new(); pub(in crate::node) struct MemoryConnManager { pub transport: InMemoryTransport, + log_register: Arc>>, + op_manager: Arc, msg_queue: Arc>>, peer: PeerKey, } impl MemoryConnManager { - pub fn new(peer: PeerKey) -> Self { - let transport = InMemoryTransport::new(peer); + pub fn new( + peer: PeerKey, + log_register: Box, + op_manager: Arc, + add_noise: bool, + ) -> Self { + let transport = InMemoryTransport::new(peer, add_noise); let msg_queue = Arc::new(Mutex::new(Vec::new())); let msg_queue_cp = msg_queue.clone(); - let tr_cp = transport.clone(); + let transport_cp = transport.clone(); GlobalExecutor::spawn(async move { // evaluate the messages as they arrive loop { - let msg = { tr_cp.msg_stack_queue.lock().pop() }; - if let Some(msg) = msg { - let msg_data: Message = - bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); - if let Some(mut queue) = msg_queue_cp.try_lock() { - queue.push(msg_data); - std::mem::drop(queue); - } - } - tokio::time::sleep(Duration::from_millis(10)).await; + let Some(msg) = transport_cp.msg_stack_queue.lock().await.pop() else { + continue; + }; + let msg_data: Message = bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); + msg_queue_cp.lock().await.push(msg_data); } }); Self { transport, + log_register: Arc::new(Mutex::new(log_register)), + op_manager, msg_queue, peer, } @@ -55,22 +63,29 @@ impl MemoryConnManager { pub async fn recv(&self) -> Result { loop { - if let Some(mut queue) = self.msg_queue.try_lock() { - let msg = queue.pop(); + let mut queue = self.msg_queue.lock().await; + let Some(msg) = queue.pop() else { std::mem::drop(queue); - if let Some(msg) = msg { - return Ok(msg); - } - } - tokio::time::sleep(Duration::from_millis(10)).await; + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + }; + return Ok(msg); } } } impl Clone for MemoryConnManager { fn clone(&self) -> Self { + let log_register = loop { + if let Ok(lr) = self.log_register.try_lock() { + break lr.trait_clone(); + } + std::thread::sleep(Duration::from_nanos(50)); + }; Self { transport: self.transport.clone(), + log_register: Arc::new(Mutex::new(log_register)), + op_manager: self.op_manager.clone(), msg_queue: self.msg_queue.clone(), peer: self.peer, } @@ -80,6 +95,11 @@ impl Clone for MemoryConnManager { #[async_trait::async_trait] impl ConnectionBridge for MemoryConnManager { async fn send(&self, target: &PeerKey, msg: Message) -> super::ConnResult<()> { + self.log_register + .try_lock() + .expect("unique lock") + .register_events(EventLog::from_outbound_msg(&msg, &self.op_manager)) + .await; let msg = bincode::serialize(&msg)?; self.transport.send(*target, msg); Ok(()) @@ -111,14 +131,13 @@ pub struct InMemoryTransport { } impl InMemoryTransport { - fn new(interface_peer: PeerKey) -> Self { + fn new(interface_peer: PeerKey, add_noise: bool) -> Self { let msg_stack_queue = Arc::new(Mutex::new(Vec::new())); - let (tx, rx) = NETWORK_WIRES.get_or_init(crossbeam::channel::unbounded); + let (network_tx, network_rx) = NETWORK_WIRES.get_or_init(crossbeam::channel::unbounded); // store messages incoming from the network in the msg stack - let rcv_msg_c = msg_stack_queue.clone(); - let rx = rx.clone(); - let tx_cp = tx.clone(); + let msg_stack_queue_cp = msg_stack_queue.clone(); + let network_tx_cp = network_tx.clone(); GlobalExecutor::spawn(async move { const MAX_DELAYED_MSG: usize = 10; let mut rng = StdRng::from_entropy(); @@ -126,25 +145,29 @@ impl InMemoryTransport { let mut delayed: HashMap<_, Vec<_>> = HashMap::with_capacity(MAX_DELAYED_MSG); let last_drain = Instant::now(); loop { - match rx.try_recv() { + match network_rx.try_recv() { Ok(msg) if msg.target == interface_peer => { tracing::trace!( "Inbound message received for peer {} from {}", interface_peer, msg.origin ); - if (rng.gen_bool(0.5) && delayed.len() < MAX_DELAYED_MSG) - || delayed.contains_key(&msg.target) - { + if rng.gen_bool(0.5) && delayed.len() < MAX_DELAYED_MSG && add_noise { delayed.entry(msg.target).or_default().push(msg); tokio::time::sleep(Duration::from_millis(10)).await; } else { - rcv_msg_c.lock().push(msg); + let mut queue = msg_stack_queue_cp.lock().await; + queue.push(msg); + if add_noise && rng.gen_bool(0.2) { + queue.shuffle(&mut rng); + } } } Ok(msg) => { // send back to the network since this msg belongs to other peer - tx_cp.send(msg).expect("failed to send msg back to network"); + network_tx_cp + .send(msg) + .expect("failed to send msg back to network"); tokio::time::sleep(Duration::from_nanos(1_000)).await } Err(channel::TryRecvError::Disconnected) => break, @@ -156,21 +179,21 @@ impl InMemoryTransport { && !delayed.is_empty()) || delayed.len() == MAX_DELAYED_MSG { - let mut queue = rcv_msg_c.lock(); + let mut queue = msg_stack_queue_cp.lock().await; for (_, msgs) in delayed.drain() { queue.extend(msgs); } - Self::shuffle(&mut queue); + let queue = &mut queue; + queue.shuffle(&mut rng); } } tracing::error!("Stopped receiving messages in {}", interface_peer); }); - let network = tx.clone(); Self { interface_peer, msg_stack_queue, - network, + network: network_tx.clone(), } } @@ -184,12 +207,4 @@ impl InMemoryTransport { tracing::error!("Network shutdown") } } - - fn shuffle(iter: &mut [T]) { - let mut rng = thread_rng(); - for i in (1..(iter.len() - 1)).rev() { - let idx = rng.gen_range(0..=i); - iter.swap(idx, i); - } - } } diff --git a/crates/core/src/node/conn_manager/p2p_protoc.rs b/crates/core/src/node/conn_manager/p2p_protoc.rs index bc43a77b6..d2c3f49ad 100644 --- a/crates/core/src/node/conn_manager/p2p_protoc.rs +++ b/crates/core/src/node/conn_manager/p2p_protoc.rs @@ -5,6 +5,7 @@ use std::{ pin::Pin, sync::Arc, task::Poll, + time::Duration, }; use asynchronous_codec::{BytesMut, Framed}; @@ -31,7 +32,10 @@ use libp2p::{ }, InboundUpgrade, Multiaddr, OutboundUpgrade, PeerId, Swarm, }; -use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::sync::{ + mpsc::{self, Receiver, Sender}, + Mutex, +}; use unsigned_varint::codec::UviBytes; use super::{ConnectionBridge, ConnectionError, EventLoopNotifications}; @@ -41,8 +45,8 @@ use crate::{ contract::{ClientResponsesSender, ExecutorToEventLoopChannel, NetworkEventListenerHalve}, message::{Message, NodeEvent, Transaction, TransactionType}, node::{ - handle_cancelled_op, join_ring_request, process_message, EventLogRegister, InitPeerNode, - NodeBuilder, OpManager, PeerKey, + event_log::EventLog, handle_cancelled_op, join_ring_request, process_message, + EventLogRegister, InitPeerNode, NodeBuilder, OpManager, PeerKey, }, operations::OpError, ring::PeerKeyLocation, @@ -60,7 +64,7 @@ const CURRENT_IDENTIFY_PROTOC_VER: &str = "/id/1.0.0"; fn config_behaviour( local_key: &Keypair, gateways: &[InitPeerNode], - _public_addr: &Option, + _private_addr: &Option, op_manager: Arc, ) -> NetBehaviour { let routing_table: HashMap<_, _> = gateways @@ -117,19 +121,73 @@ fn multiaddr_from_connection(conn: (IpAddr, u16)) -> Multiaddr { type P2pBridgeEvent = Either<(PeerKey, Box), NodeEvent>; -#[derive(Clone)] pub(crate) struct P2pBridge { active_net_connections: Arc>, accepted_peers: Arc>, ev_listener_tx: Sender, + op_manager: Arc, + log_register: Arc>>, } impl P2pBridge { - fn new(sender: Sender) -> Self { + fn new( + sender: Sender, + op_manager: Arc, + event_register: EL, + ) -> Self + where + EL: EventLogRegister, + { Self { active_net_connections: Arc::new(DashMap::new()), accepted_peers: Arc::new(DashSet::new()), ev_listener_tx: sender, + op_manager, + log_register: Arc::new(Mutex::new(Box::new(event_register))), + } + } +} + +#[cfg(any(debug_assertions, test))] +static CONTESTED_BRIDGE_CLONES: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +#[cfg(any(debug_assertions, test))] +static TOTAL_BRIDGE_CLONES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +impl Clone for P2pBridge { + fn clone(&self) -> Self { + let log_register = loop { + if let Ok(lr) = self.log_register.try_lock() { + #[cfg(any(debug_assertions, test))] + { + TOTAL_BRIDGE_CLONES.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } + break lr.trait_clone(); + } + #[cfg(any(debug_assertions, test))] + { + let contested = + CONTESTED_BRIDGE_CLONES.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + + if contested % 100 == 0 { + let total = TOTAL_BRIDGE_CLONES.load(std::sync::atomic::Ordering::Acquire); + if total > 0 { + let threshold = (total as f64 * 0.01) as usize; + if contested / total > threshold { + tracing::debug!("p2p bridge clone contested more than 1% of the time"); + } + } + } + } + std::thread::sleep(Duration::from_nanos(50)); + }; + Self { + active_net_connections: self.active_net_connections.clone(), + accepted_peers: self.accepted_peers.clone(), + ev_listener_tx: self.ev_listener_tx.clone(), + op_manager: self.op_manager.clone(), + log_register: Arc::new(Mutex::new(log_register)), } } } @@ -157,6 +215,10 @@ impl ConnectionBridge for P2pBridge { } async fn send(&self, target: &PeerKey, msg: Message) -> super::ConnResult<()> { + self.log_register + .lock() + .await + .register_events(EventLog::from_outbound_msg(&msg, &self.op_manager)); self.ev_listener_tx .send(Left((*target, Box::new(msg)))) .await @@ -172,6 +234,7 @@ pub(in crate::node) struct P2pConnManager { conn_bridge_rx: Receiver, /// last valid observed public address public_addr: Option, + listening_addr: Option, event_listener: Box, } @@ -180,13 +243,20 @@ impl P2pConnManager { transport: transport::Boxed<(PeerId, muxing::StreamMuxerBox)>, config: &NodeBuilder, op_manager: Arc, - event_listener: &dyn EventLogRegister, + event_listener: impl EventLogRegister + Clone, ) -> Result { // We set a global executor which is virtually the Tokio multi-threaded executor // to reuse it's thread pool and scheduler in order to drive futures. let global_executor = GlobalExecutor; - let public_addr = if let Some(conn) = config.local_ip.zip(config.local_port) { + let private_addr = if let Some(conn) = config.local_ip.zip(config.local_port) { + let public_addr = multiaddr_from_connection(conn); + Some(public_addr) + } else { + None + }; + + let public_addr = if let Some(conn) = config.public_ip.zip(config.public_port) { let public_addr = multiaddr_from_connection(conn); Some(public_addr) } else { @@ -196,8 +266,8 @@ impl P2pConnManager { let behaviour = config_behaviour( &config.local_key, &config.remote_nodes, - &public_addr, - op_manager, + &private_addr, + op_manager.clone(), ); let mut swarm = Swarm::new( transport, @@ -212,7 +282,7 @@ impl P2pConnManager { } let (tx_bridge_cmd, rx_bridge_cmd) = mpsc::channel(100); - let bridge = P2pBridge::new(tx_bridge_cmd); + let bridge = P2pBridge::new(tx_bridge_cmd, op_manager, event_listener.clone()); let gateways = config.get_gateways()?; Ok(P2pConnManager { @@ -221,17 +291,19 @@ impl P2pConnManager { bridge, conn_bridge_rx: rx_bridge_cmd, public_addr, - event_listener: event_listener.trait_clone(), + listening_addr: private_addr, + event_listener: Box::new(event_listener), }) } pub fn listen_on(&mut self) -> Result<(), anyhow::Error> { - if let Some(listening_addr) = &self.public_addr { + if let Some(listening_addr) = &self.listening_addr { self.swarm.listen_on(listening_addr.clone())?; } Ok(()) } + #[tracing::instrument(name = "network_event_listener", skip_all)] pub async fn run_event_listener( mut self, op_manager: Arc, @@ -336,9 +408,8 @@ impl P2pConnManager { msg = network_msg => { msg } msg = notification_msg => { msg } msg = bridge_msg => { msg } - (event_id, contract_handler_event) = op_manager.recv_from_handler() => { - if let Some(client_id) = event_id.client_id() { - let transaction = contract_handler_event.into_network_op(&op_manager).await; + event_id = op_manager.recv_from_handler() => { + if let Some((client_id, transaction)) = event_id.client_id().zip(event_id.transaction()) { tx_to_client.insert(transaction, client_id); } continue; @@ -353,19 +424,18 @@ impl P2pConnManager { Ok(Left((msg, client_id))) => { let cb = self.bridge.clone(); match msg { - Message::Canceled(tx) => { + Message::Aborted(tx) => { let tx_type = tx.tx_type(); let res = handle_cancelled_op( tx, op_manager.ring.peer_key, - self.gateways.iter(), &op_manager, &mut self.bridge, ) .await; match res { Err(OpError::MaxRetriesExceeded(_, _)) - if tx_type == TransactionType::JoinRing + if tx_type == TransactionType::Connect && self.public_addr.is_none() /* FIXME: this should be not a gateway instead */ => { tracing::warn!("Retrying joining the ring with an other peer"); @@ -449,7 +519,7 @@ impl P2pConnManager { self.public_addr = Some(address); } Ok(Right(IsPrivatePeer(_peer))) => { - todo!("attempt hole punching") + todo!("this peer is private, attempt hole punching") } Ok(Right(ClosedChannel)) => { tracing::info!("Notification channel closed"); @@ -500,7 +570,7 @@ enum ConnMngrActions { }, /// Update self own public address, useful when communicating for first time UpdatePublicAddr(Multiaddr), - /// A peer which we attempted connection to is private, attempt hole-punching + /// This is private, so when establishing connections hole-punching should be performed IsPrivatePeer(PeerId), NodeAction(NodeEvent), ClosedChannel, @@ -837,7 +907,7 @@ impl ConnectionHandler for Handler { Left(msg) => { let op_id = msg.id(); if msg.track_stats() { - if let Some(mut op) = self.op_manager.pop(op_id) { + if let Ok(Some(mut op)) = self.op_manager.pop(op_id) { op.record_transfer(); let _ = self.op_manager.push(*op_id, op); } @@ -882,7 +952,7 @@ impl ConnectionHandler for Handler { } => match Sink::poll_flush(Pin::new(&mut substream), cx) { Poll::Ready(Ok(())) => { if let Some(op_id) = op_id { - if let Some(mut op) = self.op_manager.pop(&op_id) { + if let Ok(Some(mut op)) = self.op_manager.pop(&op_id) { op.record_transfer(); let _ = self.op_manager.push(op_id, op); } @@ -911,7 +981,7 @@ impl ConnectionHandler for Handler { } => match Stream::poll_next(Pin::new(&mut substream), cx) { Poll::Ready(Some(Ok(msg))) => { let op_id = msg.id(); - if let Some(mut op) = self.op_manager.pop(op_id) { + if let Ok(Some(mut op)) = self.op_manager.pop(op_id) { op.record_transfer(); let _ = self.op_manager.push(*op_id, op); } diff --git a/crates/core/src/node/event_log.rs b/crates/core/src/node/event_log.rs index e7d6bcee7..b2bbd9b96 100644 --- a/crates/core/src/node/event_log.rs +++ b/crates/core/src/node/event_log.rs @@ -1,6 +1,7 @@ use std::{io, path::Path, time::SystemTime}; use chrono::{DateTime, Utc}; +use either::Either; use freenet_stdlib::prelude::*; use futures::{future::BoxFuture, FutureExt}; use serde::{Deserialize, Serialize}; @@ -14,8 +15,8 @@ use crate::{ config::GlobalExecutor, contract::StoreResponse, message::{Message, Transaction}, - operations::{get::GetMsg, join_ring::JoinRingMsg, put::PutMsg}, - ring::{Location, PeerKeyLocation}, + operations::{connect, get::GetMsg, put::PutMsg, subscribe::SubscribeMsg}, + ring::PeerKeyLocation, router::RouteEvent, DynError, }; @@ -34,7 +35,10 @@ struct ListenerLogId(usize); /// This type then can emit it's own information to adjacent systems /// or is a no-op. pub(crate) trait EventLogRegister: std::any::Any + Send + Sync + 'static { - fn event_received<'a>(&'a mut self, ev: EventLog) -> BoxFuture<'a, Result<(), DynError>>; + fn register_events<'a>( + &'a mut self, + events: Either, Vec>>, + ) -> BoxFuture<'a, ()>; fn trait_clone(&self) -> Box; fn as_any(&self) -> &dyn std::any::Any where @@ -44,7 +48,6 @@ pub(crate) trait EventLogRegister: std::any::Any + Send + Sync + 'static { } } -#[allow(dead_code)] // fixme: remove this pub(crate) struct EventLog<'a> { tx: &'a Transaction, peer_id: &'a PeerKey, @@ -64,15 +67,76 @@ impl<'a> EventLog<'a> { } } - pub fn from_msg(msg: &'a Message, op_storage: &'a OpManager) -> Self { + pub fn from_outbound_msg( + msg: &'a Message, + op_storage: &'a OpManager, + ) -> Either> { let kind = match msg { - Message::JoinRing(JoinRingMsg::Connected { sender, target, .. }) => { - EventKind::Connected { - loc: target.location.unwrap(), - from: target.peer, - to: *sender, + Message::Connect(connect::ConnectMsg::Response { + msg: + connect::ConnectResponse::AcceptedBy { + peers, + your_location, + your_peer_id, + }, + .. + }) => { + let this_peer = op_storage.ring.own_location(); + if peers.contains(&this_peer) { + EventKind::Connected { + this: this_peer, + connected: PeerKeyLocation { + peer: *your_peer_id, + location: Some(*your_location), + }, + } + } else { + EventKind::Ignored } } + _ => EventKind::Ignored, + }; + Either::Left(EventLog { + tx: msg.id(), + peer_id: &op_storage.ring.peer_key, + kind, + }) + } + + pub fn from_inbound_msg( + msg: &'a Message, + op_storage: &'a OpManager, + ) -> Either> { + let kind = match msg { + Message::Connect(connect::ConnectMsg::Response { + msg: + connect::ConnectResponse::AcceptedBy { + peers, + your_location, + your_peer_id, + }, + .. + }) => { + return Either::Right( + peers + .iter() + .map(|peer| { + let kind: EventKind = EventKind::Connected { + this: PeerKeyLocation { + peer: *your_peer_id, + location: Some(*your_location), + }, + connected: *peer, + }; + EventLog { + tx: msg.id(), + peer_id: &op_storage.ring.peer_key, + kind, + } + }) + .collect(), + ); + } Message::Put(PutMsg::RequestPut { contract, target, .. }) => { @@ -113,13 +177,22 @@ impl<'a> EventLog<'a> { value: StoreResponse { state: Some(_), .. }, .. }) => EventKind::Get { key: key.clone() }, - _ => EventKind::Unknown, + Message::Subscribe(SubscribeMsg::ReturnSub { + subscribed: true, + key, + sender, + .. + }) => EventKind::Subscribed { + key: key.clone(), + at: *sender, + }, + _ => EventKind::Ignored, }; - EventLog { + Either::Left(EventLog { tx: msg.id(), peer_id: &op_storage.ring.peer_key, kind, - } + }) } } @@ -339,14 +412,35 @@ impl EventRegister { } impl EventLogRegister for EventRegister { - fn event_received<'a>(&'a mut self, log: EventLog) -> BoxFuture<'a, Result<(), DynError>> { - let log_msg = LogMessage { - datetime: Utc::now(), - tx: *log.tx, - kind: log.kind, - peer_id: *log.peer_id, - }; - async { Ok(self.log_sender.send(log_msg).await?) }.boxed() + fn register_events<'a>( + &'a mut self, + logs: Either, Vec>>, + ) -> BoxFuture<'a, ()> { + async { + match logs { + Either::Left(log) => { + let log_msg = LogMessage { + datetime: Utc::now(), + tx: *log.tx, + kind: log.kind, + peer_id: *log.peer_id, + }; + let _ = self.log_sender.send(log_msg).await; + } + Either::Right(logs) => { + for log in logs { + let log_msg = LogMessage { + datetime: Utc::now(), + tx: *log.tx, + kind: log.kind, + peer_id: *log.peer_id, + }; + let _ = self.log_sender.send(log_msg).await; + } + } + } + } + .boxed() } fn trait_clone(&self) -> Box { @@ -355,18 +449,22 @@ impl EventLogRegister for EventRegister { } #[derive(Serialize, Deserialize)] +// todo: make this take by ref instead enum EventKind { Connected { - loc: Location, - from: PeerKey, - to: PeerKeyLocation, + this: PeerKeyLocation, + connected: PeerKeyLocation, }, Put(PutEvent), Get { key: ContractKey, }, Route(RouteEvent), - Unknown, + Subscribed { + key: ContractKey, + at: PeerKeyLocation, + }, + Ignored, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] @@ -408,18 +506,18 @@ pub(super) mod test_utils { }; use dashmap::DashMap; - use parking_lot::RwLock; + use parking_lot::Mutex; use super::*; - use crate::{message::TxType, ring::Distance}; + use crate::{node::tests::NodeLabel, ring::Distance}; static LOG_ID: AtomicUsize = AtomicUsize::new(0); #[derive(Clone)] pub(crate) struct TestEventListener { - node_labels: Arc>, + node_labels: Arc>, tx_log: Arc>>, - logs: Arc>>, + logs: Arc>>, } impl TestEventListener { @@ -427,16 +525,16 @@ pub(super) mod test_utils { TestEventListener { node_labels: Arc::new(DashMap::new()), tx_log: Arc::new(DashMap::new()), - logs: Arc::new(RwLock::new(Vec::new())), + logs: Arc::new(Mutex::new(Vec::new())), } } - pub fn add_node(&mut self, label: String, peer: PeerKey) { + pub fn add_node(&mut self, label: NodeLabel, peer: PeerKey) { self.node_labels.insert(label, peer); } pub fn is_connected(&self, peer: &PeerKey) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); logs.iter() .any(|log| &log.peer_id == peer && matches!(log.kind, EventKind::Connected { .. })) } @@ -447,7 +545,7 @@ pub(super) mod test_utils { for_key: &ContractKey, expected_value: &WrappedState, ) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); let put_ops = logs.iter().filter_map(|l| match &l.kind { EventKind::Put(ev) => Some((&l.tx, ev)), _ => None, @@ -485,7 +583,7 @@ pub(super) mod test_utils { /// The contract was broadcasted from one peer to an other successfully. pub fn contract_broadcasted(&self, for_key: &ContractKey) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); let put_broadcast_ops = logs.iter().filter_map(|l| match &l.kind { EventKind::Put(ev @ PutEvent::BroadcastEmitted { .. }) | EventKind::Put(ev @ PutEvent::BroadcastReceived { .. }) => Some((&l.tx, ev)), @@ -518,21 +616,39 @@ pub(super) mod test_utils { } pub fn has_got_contract(&self, peer: &PeerKey, expected_key: &ContractKey) -> bool { - let logs = self.logs.read(); + let logs = self.logs.lock(); logs.iter().any(|log| { &log.peer_id == peer && matches!(log.kind, EventKind::Get { ref key } if key == expected_key ) }) } + pub fn is_subscribed_to_contract( + &self, + peer: &PeerKey, + expected_key: &ContractKey, + ) -> bool { + let logs = self.logs.lock(); + logs.iter().any(|log| { + &log.peer_id == peer + && matches!(log.kind, EventKind::Subscribed { ref key, .. } if key == expected_key ) + }) + } + /// Unique connections for a given peer and their relative distance to other peers. pub fn connections(&self, peer: PeerKey) -> impl Iterator { - let logs = self.logs.read(); + let logs = self.logs.lock(); logs.iter() .filter_map(|l| { - if let EventKind::Connected { loc, from, to } = l.kind { - if from == peer { - return Some((to.peer, loc.distance(to.location.unwrap()))); + if let EventKind::Connected { this, connected } = l.kind { + if this.peer == peer { + return Some(( + connected.peer, + connected + .location + .expect("set location") + .distance(this.location.unwrap()), + )); } } None @@ -555,14 +671,28 @@ pub(super) mod test_utils { } impl super::EventLogRegister for TestEventListener { - fn event_received<'a>(&'a mut self, log: EventLog) -> BoxFuture<'a, Result<(), DynError>> { - let tx = log.tx; - let mut logs = self.logs.write(); - let (msg_log, log_id) = Self::create_log(log); - logs.push(msg_log); - std::mem::drop(logs); - self.tx_log.entry(*tx).or_default().push(log_id); - async { Ok(()) }.boxed() + fn register_events<'a>( + &'a mut self, + logs: Either, Vec>>, + ) -> BoxFuture<'a, ()> { + match logs { + Either::Left(log) => { + let tx = log.tx; + let (msg_log, log_id) = Self::create_log(log); + self.logs.lock().push(msg_log); + self.tx_log.entry(*tx).or_default().push(log_id); + } + Either::Right(logs) => { + let logs_list = &mut *self.logs.lock(); + for log in logs { + let tx = log.tx; + let (msg_log, log_id) = Self::create_log(log); + logs_list.push(msg_log); + self.tx_log.entry(*tx).or_default().push(log_id); + } + } + } + async {}.boxed() } fn trait_clone(&self) -> Box { @@ -570,12 +700,12 @@ pub(super) mod test_utils { } } - #[ignore] #[test] fn test_get_connections() -> Result<(), anyhow::Error> { + use crate::ring::Location; let peer_id = PeerKey::random(); let loc = Location::try_from(0.5)?; - let tx = Transaction::new(::tx_type_id(), &peer_id); + let tx = Transaction::new::(); let locations = [ (PeerKey::random(), Location::try_from(0.5)?), (PeerKey::random(), Location::try_from(0.75)?), @@ -584,18 +714,20 @@ pub(super) mod test_utils { let mut listener = TestEventListener::new(); locations.iter().for_each(|(other, location)| { - listener.event_received(EventLog { + listener.register_events(Either::Left(EventLog { tx: &tx, peer_id: &peer_id, kind: EventKind::Connected { - loc, - from: peer_id, - to: PeerKeyLocation { + this: PeerKeyLocation { + peer: peer_id, + location: Some(loc), + }, + connected: PeerKeyLocation { peer: *other, location: Some(*location), }, }, - }); + })); }); let distances: Vec<_> = listener.connections(peer_id).collect(); diff --git a/crates/core/src/node/in_memory_impl.rs b/crates/core/src/node/in_memory_impl.rs index 13a12491a..af7303320 100644 --- a/crates/core/src/node/in_memory_impl.rs +++ b/crates/core/src/node/in_memory_impl.rs @@ -7,7 +7,7 @@ use super::{ client_event_handling, conn_manager::{in_memory::MemoryConnManager, EventLoopNotifications}, handle_cancelled_op, join_ring_request, - op_state::OpManager, + op_state_manager::OpManager, process_message, EventLogRegister, PeerKey, }; use crate::{ @@ -15,7 +15,8 @@ use crate::{ config::GlobalExecutor, contract::{ self, executor_channel, ClientResponsesSender, ContractError, ContractHandler, - ContractHandlerEvent, ExecutorToEventLoopChannel, NetworkEventListenerHalve, + ContractHandlerEvent, ExecutorToEventLoopChannel, MemoryContractHandler, + NetworkEventListenerHalve, }, message::{Message, NodeEvent, TransactionType}, node::NodeBuilder, @@ -37,16 +38,14 @@ pub(super) struct NodeInMemory { impl NodeInMemory { /// Buils an in-memory node. Does nothing upon construction, - pub async fn build( + pub async fn build( builder: NodeBuilder<1>, event_listener: EL, - ch_builder: CH::Builder, - ) -> Result - where - CH: ContractHandler + Send + Sync + 'static, - { + ch_builder: String, + add_noise: bool, + ) -> Result { + let event_listener = Box::new(event_listener); let peer_key = PeerKey::from(builder.local_key.public()); - let conn_manager = MemoryConnManager::new(peer_key); let gateways = builder.get_gateways()?; let is_gateway = builder.local_ip.zip(builder.local_port).is_some(); @@ -55,9 +54,17 @@ impl NodeInMemory { let (ops_ch_channel, ch_channel) = contract::contract_handler_channel(); let op_storage = Arc::new(OpManager::new(ring, notification_tx, ops_ch_channel)); let (_executor_listener, executor_sender) = executor_channel(op_storage.clone()); - let contract_handler = CH::build(ch_channel, executor_sender, ch_builder) - .await - .map_err(|e| anyhow::anyhow!(e))?; + let contract_handler = + MemoryContractHandler::build(ch_channel, executor_sender, ch_builder) + .await + .map_err(|e| anyhow::anyhow!(e))?; + + let conn_manager = MemoryConnManager::new( + peer_key, + event_listener.trait_clone(), + op_storage.clone(), + add_noise, + ); GlobalExecutor::spawn(contract::contract_handling(contract_handler)); @@ -67,7 +74,7 @@ impl NodeInMemory { op_storage, gateways, notification_channel, - event_listener: Box::new(event_listener), + event_listener, is_gateway, _executor_listener, }) @@ -106,7 +113,8 @@ impl NodeInMemory { contract_subscribers: HashMap>, ) -> Result<(), ContractError> { for (contract, state) in contracts { - let key = contract.key(); + let key: ContractKey = contract.key(); + let parameters = contract.params(); self.op_storage .notify_contract_handler(ContractHandlerEvent::Cache(contract.clone()), None) .await?; @@ -115,6 +123,8 @@ impl NodeInMemory { ContractHandlerEvent::PutQuery { key: key.clone(), state, + related_contracts: RelatedContracts::default(), + parameters: Some(parameters), }, None, ) @@ -144,6 +154,7 @@ impl NodeInMemory { } /// Starts listening to incoming events. Will attempt to join the ring if any gateways have been provided. + #[tracing::instrument(name = "memory_event_listener", skip_all)] async fn run_event_listener( &mut self, _client_responses: ClientResponsesSender, // fixme: use this @@ -158,30 +169,32 @@ impl NodeInMemory { } }; - if let Ok(Either::Left(Message::Canceled(tx))) = msg { + if let Ok(Either::Left(Message::Aborted(tx))) = msg { let tx_type = tx.tx_type(); let res = handle_cancelled_op( tx, self.peer_key, - self.gateways.iter(), &self.op_storage, &mut self.conn_manager, ) .await; match res { Err(OpError::MaxRetriesExceeded(_, _)) - if tx_type == TransactionType::JoinRing && !self.is_gateway => + if tx_type == TransactionType::Connect && !self.is_gateway => { tracing::warn!("Retrying joining the ring with an other peer"); - let gateway = self.gateways.iter().shuffle().next().unwrap(); - join_ring_request( - None, - self.peer_key, - gateway, - &self.op_storage, - &mut self.conn_manager, - ) - .await? + if let Some(gateway) = self.gateways.iter().shuffle().next() { + join_ring_request( + None, + self.peer_key, + gateway, + &self.op_storage, + &mut self.conn_manager, + ) + .await? + } else { + anyhow::bail!("requires at least one gateway"); + } } Err(err) => return Err(anyhow::anyhow!(err)), Ok(_) => {} diff --git a/crates/core/src/node/op_state.rs b/crates/core/src/node/op_state_manager.rs similarity index 65% rename from crates/core/src/node/op_state.rs rename to crates/core/src/node/op_state_manager.rs index 047bcc1cc..24750589c 100644 --- a/crates/core/src/node/op_state.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -1,36 +1,50 @@ use std::{collections::BTreeMap, time::Instant}; -use dashmap::DashMap; +use dashmap::{DashMap, DashSet}; use either::Either; use parking_lot::RwLock; use tokio::sync::{mpsc::error::SendError, Mutex}; use crate::{ contract::{ - ContractError, ContractHandlerEvent, ContractHandlerToEventLoopChannel, NetEventListener, + ContractError, ContractHandlerEvent, ContractHandlerToEventLoopChannel, + NetEventListenerHalve, }, dev_tool::ClientId, message::{Message, Transaction, TransactionType}, operations::{ - get::GetOp, join_ring::JoinRingOp, put::PutOp, subscribe::SubscribeOp, OpEnum, OpError, + connect::ConnectOp, get::GetOp, put::PutOp, subscribe::SubscribeOp, update::UpdateOp, + OpEnum, OpError, }, ring::Ring, }; use super::{conn_manager::EventLoopNotificationsSender, PeerKey}; +#[derive(Debug, thiserror::Error)] +pub(crate) enum OpNotAvailable { + #[error("operation running")] + Running, + #[error("operation completed")] + Completed, +} + /// Thread safe and friendly data structure to maintain state of the different operations /// and enable their execution. pub(crate) struct OpManager { - join_ring: DashMap, + join_ring: DashMap, put: DashMap, get: DashMap, subscribe: DashMap, + update: DashMap, to_event_listener: EventLoopNotificationsSender, // todo: remove the need for a mutex here - ch_outbound: Mutex>, + ch_outbound: Mutex>, // FIXME: think of an optimal strategy to check for timeouts and clean up garbage _ops_ttl: RwLock>>, + // todo: improve this when the anti-write amplification functionality is added + completed: DashSet, + in_progress: DashSet, pub ring: Ring, } @@ -47,23 +61,25 @@ impl OpManager { pub(super) fn new( ring: Ring, notification_channel: EventLoopNotificationsSender, - contract_handler: ContractHandlerToEventLoopChannel, + contract_handler: ContractHandlerToEventLoopChannel, ) -> Self { Self { join_ring: DashMap::default(), put: DashMap::default(), get: DashMap::default(), subscribe: DashMap::default(), + update: DashMap::default(), ring, to_event_listener: notification_channel, ch_outbound: Mutex::new(contract_handler), + completed: DashSet::new(), + in_progress: DashSet::new(), _ops_ttl: RwLock::new(BTreeMap::new()), } } /// An early, fast path, return for communicating back changes of on-going operations - /// in the node to the main message handler receiving loop, without any transmission in - /// the network whatsoever. + /// in the node to the main message handler, without any transmission in the network whatsoever. /// /// Useful when transitioning between states that do not require any network communication /// with other nodes, like intermediate states before returning. @@ -81,14 +97,6 @@ impl OpManager { .map_err(|err| SendError(err.0.unwrap_left())) } - // /// Send an internal message to this node event loop. - // pub async fn notify_internal_op(&self, msg: NodeEvent) -> Result<(), SendError> { - // self.to_event_listener - // .send(Either::Right(msg)) - // .await - // .map_err(|err| SendError(err.0.unwrap_right())) - // } - /// Send an event to the contract handler and await a response event from it if successful. pub async fn notify_contract_handler( &self, @@ -102,43 +110,55 @@ impl OpManager { .await } - pub async fn recv_from_handler(&self) -> (crate::contract::EventId, ContractHandlerEvent) { + pub async fn recv_from_handler(&self) -> crate::contract::EventId { todo!() } pub fn push(&self, id: Transaction, op: OpEnum) -> Result<(), OpError> { + self.in_progress.remove(&id); match op { - OpEnum::JoinRing(tx) => { + OpEnum::Connect(op) => { #[cfg(debug_assertions)] - check_id_op!(id.tx_type(), TransactionType::JoinRing); - self.join_ring.insert(id, *tx); + check_id_op!(id.tx_type(), TransactionType::Connect); + self.join_ring.insert(id, *op); } - OpEnum::Put(tx) => { + OpEnum::Put(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::Put); - self.put.insert(id, tx); + self.put.insert(id, op); } - OpEnum::Get(tx) => { + OpEnum::Get(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::Get); - self.get.insert(id, tx); + self.get.insert(id, op); } - OpEnum::Subscribe(tx) => { + OpEnum::Subscribe(op) => { #[cfg(debug_assertions)] check_id_op!(id.tx_type(), TransactionType::Subscribe); - self.subscribe.insert(id, tx); + self.subscribe.insert(id, op); + } + OpEnum::Update(op) => { + #[cfg(debug_assertions)] + check_id_op!(id.tx_type(), TransactionType::Update); + self.update.insert(id, op); } } Ok(()) } - pub fn pop(&self, id: &Transaction) -> Option { - match id.tx_type() { - TransactionType::JoinRing => self + pub fn pop(&self, id: &Transaction) -> Result, OpNotAvailable> { + if self.completed.contains(id) { + return Err(OpNotAvailable::Completed); + } + if self.in_progress.contains(id) { + return Err(OpNotAvailable::Running); + } + let op = match id.tx_type() { + TransactionType::Connect => self .join_ring .remove(id) .map(|(_k, v)| v) - .map(|op| OpEnum::JoinRing(Box::new(op))), + .map(|op| OpEnum::Connect(Box::new(op))), TransactionType::Put => self.put.remove(id).map(|(_k, v)| v).map(OpEnum::Put), TransactionType::Get => self.get.remove(id).map(|(_k, v)| v).map(OpEnum::Get), TransactionType::Subscribe => self @@ -146,9 +166,15 @@ impl OpManager { .remove(id) .map(|(_k, v)| v) .map(OpEnum::Subscribe), - TransactionType::Update => todo!(), + TransactionType::Update => self.update.remove(id).map(|(_k, v)| v).map(OpEnum::Update), TransactionType::Canceled => unreachable!(), - } + }; + self.in_progress.insert(*id); + Ok(op) + } + + pub fn completed(&self, id: Transaction) { + self.completed.insert(id); } pub fn prune_connection(&self, peer: PeerKey) { diff --git a/crates/core/src/node/p2p_impl.rs b/crates/core/src/node/p2p_impl.rs index a1fc3684d..2ea78a3bb 100644 --- a/crates/core/src/node/p2p_impl.rs +++ b/crates/core/src/node/p2p_impl.rs @@ -62,7 +62,6 @@ impl NodeP2P { } // start the p2p event loop - // todo: pass `cli_response_sender` self.conn_manager .run_event_listener( self.op_manager.clone(), @@ -73,18 +72,18 @@ impl NodeP2P { .await } - pub(crate) async fn build( + pub(crate) async fn build( builder: NodeBuilder, event_listener: EL, ch_builder: CH::Builder, ) -> Result where CH: ContractHandler + Send + Sync + 'static, + EL: EventLogRegister + Clone, { let peer_key = PeerKey::from(builder.local_key.public()); let gateways = builder.get_gateways()?; - let event_listener: Box = Box::new(event_listener); let ring = Ring::new::(&builder, &gateways)?; let (notification_channel, notification_tx) = EventLoopNotifications::channel(); let (ch_outbound, ch_inbound) = contract::contract_handler_channel(); @@ -97,7 +96,7 @@ impl NodeP2P { let conn_manager = { let transport = Self::config_transport(&builder.local_key)?; - P2pConnManager::build(transport, &builder, op_storage.clone(), &*event_listener)? + P2pConnManager::build(transport, &builder, op_storage.clone(), event_listener)? }; GlobalExecutor::spawn(contract::contract_handling(contract_handler)); @@ -211,7 +210,7 @@ mod test { NodeP2P::build::( config, event_log::TestEventListener::new(), - (), + "ping-listener".into(), ) .await?, ); @@ -228,7 +227,7 @@ mod test { let mut peer2 = NodeP2P::build::( config, event_log::TestEventListener::new(), - (), + "ping-dialer".into(), ) .await .unwrap(); diff --git a/crates/core/src/node/tests.rs b/crates/core/src/node/tests.rs index 88b9d1d68..8508fa580 100644 --- a/crates/core/src/node/tests.rs +++ b/crates/core/src/node/tests.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, fmt::Write, net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener}, + sync::Arc, time::{Duration, Instant}, }; @@ -16,7 +17,6 @@ use tracing::{info, instrument}; use crate::{ client_events::test::MemoryEventsGen, config::GlobalExecutor, - contract::MemoryContractHandler, node::{event_log::TestEventListener, InitPeerNode, NodeBuilder, NodeInMemory}, ring::{Distance, Location, PeerKeyLocation}, }; @@ -42,21 +42,58 @@ pub fn get_dynamic_port() -> u16 { rand::thread_rng().gen_range(FIRST_DYNAMIC_PORT..LAST_DYNAMIC_PORT) } -/// A simulated in-memory network topology. -pub(crate) struct SimNetwork { - pub labels: HashMap, - pub event_listener: TestEventListener, - usr_ev_controller: Sender<(EventId, PeerKey)>, - receiver_ch: Receiver<(EventId, PeerKey)>, - gateways: Vec<(NodeInMemory, GatewayConfig)>, - nodes: Vec<(NodeInMemory, String)>, - ring_max_htl: usize, - rnd_if_htl_above: usize, - max_connections: usize, - min_connections: usize, +pub(crate) type EventId = usize; + +#[derive(PartialEq, Eq, Hash, Clone)] +pub(crate) struct NodeLabel(Arc); + +impl NodeLabel { + fn gateway(id: usize) -> Self { + Self(format!("gateway-{id}").into()) + } + + fn node(id: usize) -> Self { + Self(format!("node-{id}").into()) + } + + fn is_gateway(&self) -> bool { + self.0.starts_with("gateway") + } + + pub fn is_node(&self) -> bool { + self.0.starts_with("node") + } } -pub(crate) type EventId = usize; +impl std::fmt::Display for NodeLabel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::ops::Deref for NodeLabel { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a> From<&'a str> for NodeLabel { + fn from(value: &'a str) -> Self { + assert!(value.starts_with("gateway-") || value.starts_with("node-")); + let mut parts = value.split('-'); + assert!(parts.next().is_some()); + assert!(parts + .next() + .map(|s| s.parse::()) + .transpose() + .expect("should be an u16") + .is_some()); + assert!(parts.next().is_none()); + Self(value.to_string().into()) + } +} #[derive(Clone)] pub(crate) struct NodeSpecification { @@ -69,14 +106,33 @@ pub(crate) struct NodeSpecification { #[derive(Clone)] struct GatewayConfig { - label: String, + label: NodeLabel, port: u16, id: PeerId, location: Location, } +/// A simulated in-memory network topology. +pub(crate) struct SimNetwork { + name: String, + debug: bool, + pub labels: HashMap, + pub event_listener: TestEventListener, + user_ev_controller: Sender<(EventId, PeerKey)>, + receiver_ch: Receiver<(EventId, PeerKey)>, + gateways: Vec<(NodeInMemory, GatewayConfig)>, + nodes: Vec<(NodeInMemory, NodeLabel)>, + ring_max_htl: usize, + rnd_if_htl_above: usize, + max_connections: usize, + min_connections: usize, + init_backoff: Duration, + add_noise: bool, +} + impl SimNetwork { pub async fn new( + name: &str, gateways: usize, nodes: usize, ring_max_htl: usize, @@ -85,30 +141,49 @@ impl SimNetwork { min_connections: usize, ) -> Self { assert!(gateways > 0 && nodes > 0); - let (usr_ev_controller, _rcv_copy) = channel((0, PeerKey::random())); + let (user_ev_controller, receiver_ch) = channel((0, PeerKey::random())); let mut net = Self { + name: name.into(), + debug: false, event_listener: TestEventListener::new(), labels: HashMap::new(), - usr_ev_controller, - receiver_ch: _rcv_copy, + user_ev_controller, + receiver_ch, gateways: Vec::with_capacity(gateways), nodes: Vec::with_capacity(nodes), ring_max_htl, rnd_if_htl_above, max_connections, min_connections, + init_backoff: Duration::from_millis(1), + add_noise: false, }; - net.build_gateways(gateways); + net.build_gateways(gateways).await; net.build_nodes(nodes).await; net } + pub fn with_start_backoff(&mut self, value: Duration) { + self.init_backoff = value; + } + + /// Simulates network random behaviour, like messages arriving delayed or out of order, throttling etc. + #[allow(unused)] + pub fn with_noise(&mut self) { + self.add_noise = true; + } + + #[allow(unused)] + pub fn debug(&mut self) { + self.debug = true; + } + #[instrument(skip(self))] - fn build_gateways(&mut self, num: usize) { + async fn build_gateways(&mut self, num: usize) { info!("Building {} gateways", num); let mut configs = Vec::with_capacity(num); for node_no in 0..num { - let label = format!("gateway-{}", node_no); + let label = NodeLabel::gateway(node_no); let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); let port = get_free_port().unwrap(); @@ -141,35 +216,35 @@ impl SimNetwork { )); } - // FIXME: - // for (mut this_node, this_config) in configs { - // for GatewayConfig { - // port, id, location, .. - // } in configs.iter().filter_map(|(_, config)| { - // if this_config.label != config.label { - // Some(config) - // } else { - // None - // } - // }) { - // this_node.add_gateway( - // InitPeerNode::new(*id, *location) - // .listening_ip(Ipv6Addr::LOCALHOST) - // .listening_port(*port), - // ); - // } - - // let gateway = NodeInMemory::::build::( - // this_node, - // Some(Box::new(self.event_listener.clone())), - // ) - // .unwrap(); - // self.gateways.push((gateway, this_config)); - // } + let gateways: Vec<_> = configs.iter().map(|(_, gw)| gw.clone()).collect(); + for (mut this_node, this_config) in configs { + for GatewayConfig { + port, id, location, .. + } in gateways + .iter() + .filter(|config| this_config.label != config.label) + { + this_node.add_gateway( + InitPeerNode::new(*id, *location) + .listening_ip(Ipv6Addr::LOCALHOST) + .listening_port(*port), + ); + } + let gateway = NodeInMemory::build( + this_node, + self.event_listener.clone(), + format!("{}-{label}", self.name, label = this_config.label), + self.add_noise, + ) + .await + .unwrap(); + self.gateways.push((gateway, this_config)); + } } #[instrument(skip(self))] async fn build_nodes(&mut self, num: usize) { + info!("Building {} regular nodes", num); let gateways: Vec<_> = self .gateways .iter() @@ -178,7 +253,7 @@ impl SimNetwork { .collect(); for node_no in 0..num { - let label = format!("node-{}", node_no); + let label = NodeLabel::node(node_no); let pair = identity::Keypair::generate_ed25519(); let id = pair.public().to_peer_id(); @@ -205,10 +280,11 @@ impl SimNetwork { self.event_listener .add_node(label.clone(), PeerKey::from(id)); - let node = NodeInMemory::build::( + let node = NodeInMemory::build( config, self.event_listener.clone(), - (), + format!("{}-{label}", self.name), + self.add_noise, ) .await .unwrap(); @@ -216,28 +292,25 @@ impl SimNetwork { } } - pub async fn build(&mut self) { - self.build_with_specs(HashMap::new()).await + pub async fn start(&mut self) { + self.start_with_spec(HashMap::new()).await } - pub async fn build_with_specs(&mut self, mut specs: HashMap) { + pub async fn start_with_spec(&mut self, mut specs: HashMap) { let mut gw_not_init = self.gateways.len(); let gw = self.gateways.drain(..).map(|(n, c)| (n, c.label)); for (node, label) in gw.chain(self.nodes.drain(..)).collect::>() { let node_spec = specs.remove(&label); self.initialize_peer(node, label, node_spec); - if gw_not_init != 0 { - gw_not_init -= 1; - } else { - tokio::time::sleep(Duration::from_millis(1)).await; - } + gw_not_init = gw_not_init.saturating_sub(1); + tokio::time::sleep(self.init_backoff).await; } } fn initialize_peer( &mut self, mut peer: NodeInMemory, - label: String, + label: NodeLabel, node_specs: Option, ) { let mut user_events = MemoryEventsGen::new(self.receiver_ch.clone(), peer.peer_key); @@ -246,6 +319,7 @@ impl SimNetwork { user_events.request_contracts(specs.non_owned_contracts); user_events.generate_events(specs.events_to_generate); } + tracing::debug!(peer = %label, "initializing"); self.labels.insert(label, peer.peer_key); GlobalExecutor::spawn(async move { if let Some(specs) = node_specs { @@ -257,23 +331,20 @@ impl SimNetwork { }); } - pub fn get_locations_by_node(&self) -> HashMap { - let mut locations_by_node: HashMap = HashMap::new(); + pub fn get_locations_by_node(&self) -> HashMap { + let mut locations_by_node: HashMap = HashMap::new(); // Get node and gateways location by label for (node, label) in &self.nodes { - locations_by_node.insert(label.to_string(), node.op_storage.ring.own_location()); + locations_by_node.insert(label.clone(), node.op_storage.ring.own_location()); } for (node, config) in &self.gateways { - locations_by_node.insert( - config.label.to_string(), - node.op_storage.ring.own_location(), - ); + locations_by_node.insert(config.label.clone(), node.op_storage.ring.own_location()); } locations_by_node } - pub fn connected(&self, peer: &str) -> bool { + pub fn connected(&self, peer: &NodeLabel) -> bool { if let Some(key) = self.labels.get(peer) { self.event_listener.is_connected(key) } else { @@ -281,62 +352,83 @@ impl SimNetwork { } } - pub fn has_put_contract(&self, peer: &str, key: &ContractKey, value: &WrappedState) -> bool { - if let Some(pk) = self.labels.get(peer) { + pub fn has_put_contract( + &self, + peer: impl Into, + key: &ContractKey, + value: &WrappedState, + ) -> bool { + if let Some(pk) = self.labels.get(&peer.into()) { self.event_listener.has_put_contract(pk, key, value) } else { panic!("peer not found"); } } - pub fn has_got_contract(&self, peer: &str, key: &ContractKey) -> bool { - if let Some(pk) = self.labels.get(peer) { + pub fn has_got_contract(&self, peer: impl Into, key: &ContractKey) -> bool { + if let Some(pk) = self.labels.get(&peer.into()) { self.event_listener.has_got_contract(pk, key) } else { panic!("peer not found"); } } + pub fn is_subscribed_to_contract(&self, peer: impl Into, key: &ContractKey) -> bool { + if let Some(pk) = self.labels.get(&peer.into()) { + self.event_listener.is_subscribed_to_contract(pk, key) + } else { + panic!("peer not found"); + } + } + /// Builds an histogram of the distribution in the ring of each node relative to each other. - pub fn ring_distribution(&self, scale: i32) -> impl Iterator { + pub fn ring_distribution(&self, scale: i32) -> Vec<(f64, usize)> { let mut all_dists = Vec::with_capacity(self.labels.len()); for (.., key) in &self.labels { all_dists.push(self.event_listener.connections(*key)); } - group_locations_in_buckets( + let mut dist_buckets = group_locations_in_buckets( all_dists.into_iter().flatten().map(|(_, l)| l.as_f64()), scale, ) + .collect::>(); + dist_buckets + .sort_by(|(d0, _), (d1, _)| d0.partial_cmp(d1).unwrap_or(std::cmp::Ordering::Equal)); + dist_buckets } /// Returns the connectivity in the network per peer (that is all the connections /// this peers has registered). - pub fn node_connectivity(&self) -> HashMap> { + pub fn node_connectivity(&self) -> HashMap)> { let mut peers_connections = HashMap::with_capacity(self.labels.len()); let key_to_label: HashMap<_, _> = self.labels.iter().map(|(k, v)| (v, k)).collect(); for (label, key) in &self.labels { - peers_connections.insert( - label.clone(), - self.event_listener - .connections(*key) - .map(|(k, d)| (key_to_label[&k].clone(), d)) - .collect::>(), - ); + let conns = self + .event_listener + .connections(*key) + .map(|(k, d)| (key_to_label[&k].clone(), d)) + .collect::>(); + peers_connections.insert(label.clone(), (*key, conns)); } peers_connections } + /// # Arguments + /// + /// - label: node for which to trigger the + /// - event_id: which event to trigger + /// - await_for: if set, wait for the duration before returning pub async fn trigger_event( &self, - label: &str, + label: impl Into, event_id: EventId, await_for: Option, ) -> Result<(), anyhow::Error> { let peer = self .labels - .get(label) + .get(&label.into()) .ok_or_else(|| anyhow::anyhow!("node not found"))?; - self.usr_ev_controller + self.user_ev_controller .send((event_id, *peer)) .expect("node listeners disconnected"); if let Some(sleep_time) = await_for { @@ -344,6 +436,114 @@ impl SimNetwork { } Ok(()) } + + /// Checks that all peers in the network have acquired at least one connection to any + /// other peers. + pub async fn check_connectivity(&self, time_out: Duration) -> Result<(), anyhow::Error> { + let num_nodes = self.nodes.capacity(); + let mut connected = HashSet::new(); + let elapsed = Instant::now(); + while elapsed.elapsed() < time_out && connected.len() < num_nodes { + for node in 0..num_nodes { + if !connected.contains(&node) && self.connected(&NodeLabel::node(node)) { + connected.insert(node); + } + } + } + + let expected = HashSet::from_iter(0..num_nodes); + let mut missing: Vec<_> = expected + .difference(&connected) + .map(|n| format!("node-{}", n)) + .collect(); + + let node_connectivity = self.node_connectivity(); + let connections = pretty_print_connections(&node_connectivity); + tracing::info!("Number of simulated nodes: {num_nodes}"); + tracing::info!("{connections}"); + + if !missing.is_empty() { + missing.sort(); + tracing::error!("Nodes without connection: {:?}", missing); + tracing::error!("Total nodes without connection: {:?}", missing.len()); + anyhow::bail!("found disconnected nodes"); + } + + tracing::info!( + "Required time for connecting all peers: {} secs", + elapsed.elapsed().as_secs() + ); + + let hist = self.ring_distribution(1); + tracing::info!("Ring distribution: {:?}", hist); + + Ok(()) + } + + /// Recommended to calling after `check_connectivity` to ensure enough time + /// elapsed for all peers to become connected. + /// + /// Checks that there is a good connectivity over the simulated network, + /// meaning that: + /// + /// - at least 50% of the peers have more than the minimum connections + /// - + pub fn network_connectivity_quality(&self) -> Result<(), anyhow::Error> { + const HIGHER_THAN_MIN_THRESHOLD: f64 = 0.5; + let num_nodes = self.nodes.capacity(); + let min_connections_threshold = (num_nodes as f64 * HIGHER_THAN_MIN_THRESHOLD) as usize; + let node_connectivity = self.node_connectivity(); + + let mut connections_per_peer: Vec<_> = node_connectivity + .iter() + .map(|(k, v)| (k, v.1.len())) + .filter(|&(k, _)| !k.is_gateway()) + .map(|(_, v)| v) + .collect(); + + // ensure at least "most" normal nodes have more than one connection + connections_per_peer.sort_unstable_by_key(|num_conn| *num_conn); + if connections_per_peer[min_connections_threshold] < self.min_connections { + tracing::error!( + "Low connectivity; more than {:.0}% of the nodes don't have more than minimum connections", + HIGHER_THAN_MIN_THRESHOLD * 100.0 + ); + anyhow::bail!("low connectivity"); + } else { + let idx = connections_per_peer[min_connections_threshold..] + .iter() + .position(|num_conn| *num_conn < self.min_connections) + .unwrap_or_else(|| connections_per_peer[min_connections_threshold..].len() - 1) + + (min_connections_threshold - 1); + let percentile = idx as f64 / connections_per_peer.len() as f64 * 100.0; + tracing::info!("{percentile:.0}% nodes have higher than required minimum connections"); + } + + // ensure the average number of connections per peer is above the mean between max and min connections + let expected_avg_connections = + ((self.max_connections - self.min_connections) / 2) + self.min_connections; + let avg_connections: usize = connections_per_peer.iter().sum::() / num_nodes; + tracing::info!( + "Average connections: {avg_connections} (expected > {expected_avg_connections})" + ); + if avg_connections < expected_avg_connections { + tracing::error!("Average number of connections ({avg_connections}) is low (< {expected_avg_connections})"); + anyhow::bail!("low average number of connections"); + } + Ok(()) + } +} + +impl Drop for SimNetwork { + fn drop(&mut self) { + if !self.debug { + for label in self.labels.keys() { + let p = std::env::temp_dir() + .join(format!("freenet-executor-{sim}-{label}", sim = self.name)); + let _ = std::fs::remove_dir_all(p); + } + } + } } fn group_locations_in_buckets( @@ -366,82 +566,14 @@ fn group_locations_in_buckets( .map(move |(k, v)| ((k as f64 / (10.0f64).powi(scale)), v)) } -pub(crate) async fn check_connectivity( - sim_nodes: &SimNetwork, - num_nodes: usize, - time_out: Duration, -) -> Result<(), anyhow::Error> { - let mut connected = HashSet::new(); - let elapsed = Instant::now(); - while elapsed.elapsed() < time_out && connected.len() < num_nodes { - for node in 0..num_nodes { - if !connected.contains(&node) && sim_nodes.connected(&format!("node-{}", node)) { - connected.insert(node); - } - } - } - tokio::time::sleep(Duration::from_millis(1_000)).await; - let expected = HashSet::from_iter(0..num_nodes); - let mut missing: Vec<_> = expected - .difference(&connected) - .map(|n| format!("node-{}", n)) - .collect(); - - let node_connectivity = sim_nodes.node_connectivity(); - let connections = pretty_print_connections(&node_connectivity); - tracing::info!("{connections}"); - - if !missing.is_empty() { - missing.sort(); - tracing::error!("Nodes without connection: {:?}", missing); - tracing::error!("Total nodes without connection: {:?}", missing.len()); - anyhow::bail!("found disconnected nodes"); - } - - tracing::info!( - "Required time for connecting all peers: {} secs", - elapsed.elapsed().as_secs() - ); - - let hist: Vec<_> = sim_nodes.ring_distribution(1).collect(); - tracing::info!("Ring distribution: {:?}", hist); - - let mut connections_per_peer: Vec<_> = node_connectivity - .iter() - .map(|(k, v)| (k, v.len())) - .filter_map(|(k, v)| { - if !k.starts_with("gateway") { - Some(v) - } else { - None - } - }) - .collect(); - - // ensure at least some normal nodes have more than one connection - connections_per_peer.sort_unstable_by_key(|num_conn| *num_conn); - if *connections_per_peer.iter().last().unwrap() < 1 { - anyhow::bail!("low connectivy; nodes didn't connect beyond the gateway"); - } - - // ensure the average number of connections per peer is above N - let avg_connections: usize = connections_per_peer.iter().sum::() / num_nodes; - tracing::info!("Average connections: {}", avg_connections); - if avg_connections < 1 { - anyhow::bail!("average number of connections is low"); - } - Ok(()) -} - -fn pretty_print_connections(conns: &HashMap>) -> String { +fn pretty_print_connections( + conns: &HashMap)>, +) -> String { let mut connections = String::from("Node connections:\n"); let mut conns = conns.iter().collect::>(); - conns.sort_by(|a, b| a.0.cmp(b.0)); - for (peer, conns) in conns { - if peer.starts_with("gateway") { - continue; - } - writeln!(&mut connections, "{peer}:").unwrap(); + conns.sort_by(|(a, _), (b, _)| a.cmp(b)); + for (peer, (key, conns)) in conns { + writeln!(&mut connections, "{peer} ({key}):").unwrap(); for (conn, dist) in conns { let dist = dist.as_f64(); writeln!(&mut connections, " {conn} (dist: {dist:.3})").unwrap(); @@ -450,7 +582,6 @@ fn pretty_print_connections(conns: &HashMap>) connections } -#[ignore] #[test] fn group_locations_test() -> Result<(), anyhow::Error> { let locations = vec![0.5356, 0.5435, 0.5468, 0.5597, 0.6745, 0.7309, 0.7412]; diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index 53e953442..786af2f0e 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -7,12 +7,13 @@ use crate::{ client_events::{ClientId, HostResult}, contract::ContractError, message::{InnerMessage, Message, Transaction, TransactionType}, - node::{ConnectionBridge, ConnectionError, OpManager, PeerKey}, + node::{ConnectionBridge, ConnectionError, OpManager, OpNotAvailable, PeerKey}, ring::{Location, PeerKeyLocation, RingError}, + DynError, }; +pub(crate) mod connect; pub(crate) mod get; -pub(crate) mod join_ring; pub(crate) mod op_trait; pub(crate) mod put; pub(crate) mod subscribe; @@ -33,7 +34,7 @@ pub(crate) struct OpInitialization { pub(crate) async fn handle_op_request( op_storage: &OpManager, conn_manager: &mut CB, - msg: Op::Message, + msg: &Op::Message, client_id: Option, ) -> Result, OpError> where @@ -43,7 +44,7 @@ where let sender; let tx = *msg.id(); let result = { - let OpInitialization { sender: s, op } = Op::load_or_init(op_storage, &msg)?; + let OpInitialization { sender: s, op } = Op::load_or_init(op_storage, msg)?; sender = s; op.process_message(conn_manager, op_storage, msg, client_id) .await @@ -74,7 +75,7 @@ where } Err((err, tx_id)) => { if let Some(sender) = sender { - conn_manager.send(&sender, Message::Canceled(tx_id)).await?; + conn_manager.send(&sender, Message::Aborted(tx_id)).await?; } return Err(err); } @@ -125,19 +126,21 @@ where } pub(crate) enum OpEnum { - JoinRing(Box), + Connect(Box), Put(put::PutOp), Get(get::GetOp), Subscribe(subscribe::SubscribeOp), + Update(update::UpdateOp), } impl OpEnum { delegate::delegate! { to match self { - OpEnum::JoinRing(op) => op, + OpEnum::Connect(op) => op, OpEnum::Put(op) => op, OpEnum::Get(op) => op, OpEnum::Subscribe(op) => op, + OpEnum::Update(op) => op, } { pub fn id(&self) -> &Transaction; pub fn outcome(&self) -> OpOutcome; @@ -183,6 +186,8 @@ pub(crate) enum OpError { RingError(#[from] RingError), #[error(transparent)] ContractError(#[from] ContractError), + #[error(transparent)] + ExecutorError(DynError), #[error("unexpected operation state")] UnexpectedOpState, @@ -194,8 +199,10 @@ pub(crate) enum OpError { IncorrectTxType(TransactionType, TransactionType), #[error("op not present: {0}")] OpNotPresent(Transaction), - #[error("max number of retries for tx {0} of op type {1} reached")] - MaxRetriesExceeded(Transaction, String), + #[error("max number of retries for tx {0} of op type `{1}` reached")] + MaxRetriesExceeded(Transaction, TransactionType), + #[error("op not available")] + OpNotAvailable(#[from] OpNotAvailable), // user for control flow /// This is used as an early interrumpt of an op update when an op diff --git a/crates/core/src/operations/join_ring.rs b/crates/core/src/operations/connect.rs similarity index 58% rename from crates/core/src/operations/join_ring.rs rename to crates/core/src/operations/connect.rs index 32415cade..5eb8eb7f2 100644 --- a/crates/core/src/operations/join_ring.rs +++ b/crates/core/src/operations/connect.rs @@ -1,3 +1,4 @@ +//! Operation which seeks new connections in the ring. use futures::Future; use std::pin::Pin; use std::{collections::HashSet, time::Duration}; @@ -15,13 +16,11 @@ use crate::{ util::ExponentialBackoff, }; -pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; +pub(crate) use self::messages::{ConnectMsg, ConnectRequest, ConnectResponse}; -const MAX_JOIN_RETRIES: usize = 3; - -pub(crate) struct JoinRingOp { +pub(crate) struct ConnectOp { id: Transaction, - state: Option, + state: Option, pub gateway: Box, /// keeps track of the number of retries and applies an exponential backoff cooldown period pub backoff: Option, @@ -29,7 +28,7 @@ pub(crate) struct JoinRingOp { _ttl: Duration, } -impl JoinRingOp { +impl ConnectOp { pub fn has_backoff(&self) -> bool { self.backoff.is_some() } @@ -39,25 +38,46 @@ impl JoinRingOp { } pub(super) fn finalized(&self) -> bool { - matches!(self.state, Some(JRState::Connected)) + matches!(self.state, Some(ConnectState::Connected)) } pub(super) fn record_transfer(&mut self) {} } -pub(crate) struct JoinRingResult {} +/// Not really used since client requests will never interact with this directly. +pub(crate) struct ConnectResult {} -impl TryFrom for JoinRingResult { +impl TryFrom for ConnectResult { type Error = OpError; - fn try_from(_value: JoinRingOp) -> Result { - todo!() + fn try_from(_value: ConnectOp) -> Result { + Ok(Self {}) } } -impl Operation for JoinRingOp { - type Message = JoinRingMsg; - type Result = JoinRingResult; +/* +Will need to do some changes for when we perform parallel joins. + +t0: (#1 join attempt) +joiner: + 1. dont knows location + 3. knows location + n. connected to the ring +gateway: + 2. assigned_location: None; assigned location to `joiner` based on IP and communicate to joiner + 4. forward to N peers + ... + +(2 join subsequently to acquire more/better connections) +join: + 1. knows location +gateway: + 2. assigned_location: Some(loc) +*/ + +impl Operation for ConnectOp { + type Message = ConnectMsg; + type Result = ConnectResult; fn load_or_init( op_storage: &OpManager, @@ -66,21 +86,24 @@ impl Operation for JoinRingOp { let sender; let tx = *msg.id(); match op_storage.pop(msg.id()) { - Some(OpEnum::JoinRing(join_op)) => { + Ok(Some(OpEnum::Connect(connect_op))) => { sender = msg.sender().cloned(); // was an existing operation, the other peer messaged back Ok(OpInitialization { - op: *join_op, + op: *connect_op, sender, }) } - Some(_) => Err(OpError::OpNotPresent(tx)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(tx, op); + Err(OpError::OpNotPresent(tx)) + } + Ok(None) => { // new request to join this node, initialize the machine Ok(OpInitialization { op: Self { id: tx, - state: Some(JRState::Initializing), + state: Some(ConnectState::Initializing), backoff: None, gateway: Box::new(op_storage.ring.own_location()), _ttl: PEER_TIMEOUT, @@ -88,6 +111,7 @@ impl Operation for JoinRingOp { sender: None, }) } + Err(err) => Err(err.into()), } } @@ -99,61 +123,64 @@ impl Operation for JoinRingOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, _client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let mut return_msg = None; + let return_msg; let mut new_state = None; match input { - JoinRingMsg::Request { + ConnectMsg::Request { id, msg: - JoinRequest::StartReq { + ConnectRequest::StartReq { target: this_node_loc, - req_peer, + joiner, hops_to_live, + assigned_location, .. }, } => { // likely a gateway which accepts connections tracing::debug!( - "Initial join request received from {} with HTL {} @ {}", - req_peer, + tx = %id, + "Connection request received from {} with HTL {} @ {}", + joiner, hops_to_live, this_node_loc.peer ); - let new_location = Location::random(); + // todo: location should be based on your public IP + let new_location = assigned_location.unwrap_or_else(Location::random); // FIXME: don't try to forward to peers which have already been tried (add a rejected_by list) let accepted_by = if op_storage.ring.should_accept(&new_location) { - tracing::debug!("Accepting connection from {}", req_peer,); - HashSet::from_iter([this_node_loc]) + tracing::debug!(tx = %id, "Accepting connection from {}", joiner,); + HashSet::from_iter([*this_node_loc]) } else { - tracing::debug!("Rejecting connection from peer {}", req_peer); + tracing::debug!(tx = %id, at_peer = %this_node_loc.peer, "Rejecting connection from peer {}", joiner); HashSet::new() }; let new_peer_loc = PeerKeyLocation { location: Some(new_location), - peer: req_peer, + peer: *joiner, }; if let Some(mut updated_state) = forward_conn( - id, + *id, &op_storage.ring, conn_manager, new_peer_loc, new_peer_loc, - hops_to_live, + *hops_to_live, accepted_by.len(), ) .await? { tracing::debug!( - "Awaiting proxy response from @ {} (tx: {})", + tx = %id, + "Awaiting proxy response from @ {}", this_node_loc.peer, - id ); updated_state.add_new_proxy(accepted_by)?; // awaiting responses from proxies @@ -162,33 +189,35 @@ impl Operation for JoinRingOp { } else { if !accepted_by.is_empty() { tracing::debug!( + tx = %id, "OC received at gateway {} from requesting peer {}", this_node_loc.peer, - req_peer + joiner ); - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); } else { + op_storage.completed(*id); new_state = None } - return_msg = Some(JoinRingMsg::Response { - id, - sender: this_node_loc, - msg: JoinResponse::AcceptedBy { + return_msg = Some(ConnectMsg::Response { + id: *id, + sender: *this_node_loc, + msg: ConnectResponse::AcceptedBy { peers: accepted_by, your_location: new_location, - your_peer_id: req_peer, + your_peer_id: *joiner, }, target: PeerKeyLocation { - peer: req_peer, + peer: *joiner, location: Some(new_location), }, }); } } - JoinRingMsg::Request { + ConnectMsg::Request { id, msg: - JoinRequest::Proxy { + ConnectRequest::Proxy { sender, joiner, hops_to_live, @@ -196,7 +225,8 @@ impl Operation for JoinRingOp { } => { let own_loc = op_storage.ring.own_location(); tracing::debug!( - "Proxy join request received from {} to join new peer {} with HTL {} @ {}", + tx = %id, + "Proxy connect request received from {} to ing new peer {} with HTL {} @ {}", sender.peer, joiner.peer, hops_to_live, @@ -206,10 +236,11 @@ impl Operation for JoinRingOp { .ring .should_accept(&joiner.location.ok_or(ConnectionError::LocationUnknown)?) { - tracing::debug!("Accepting proxy connection from {}", joiner.peer); + tracing::debug!(tx = %id, "Accepting proxy connection from {}", joiner.peer); HashSet::from_iter([own_loc]) } else { tracing::debug!( + tx = %id, "Not accepting new proxy connection for sender {}", joiner.peer ); @@ -217,12 +248,12 @@ impl Operation for JoinRingOp { }; if let Some(mut updated_state) = forward_conn( - id, + *id, &op_storage.ring, conn_manager, - sender, - joiner, - hops_to_live, + *sender, + *joiner, + *hops_to_live, accepted_by.len(), ) .await? @@ -233,17 +264,13 @@ impl Operation for JoinRingOp { return_msg = None; } else { match self.state { - Some(JRState::Initializing) => { - let (state, msg) = try_proxy_connection( - &id, - &sender, - &own_loc, - accepted_by.clone(), - ); + Some(ConnectState::Initializing) => { + let (state, msg) = + try_proxy_connection(id, sender, &own_loc, accepted_by.clone()); new_state = state; return_msg = msg; } - Some(JRState::AwaitingProxyResponse { + Some(ConnectState::AwaitingProxyResponse { accepted_by: mut previously_accepted, new_peer_id, target, @@ -259,18 +286,19 @@ impl Operation for JoinRingOp { } if match_target { - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); tracing::debug!( + tx = %id, "Sending response to join request with all the peers that accepted \ connection from gateway {} to peer {}", sender.peer, target.peer ); - return_msg = Some(JoinRingMsg::Response { - id, + return_msg = Some(ConnectMsg::Response { + id: *id, target, - sender, - msg: JoinResponse::AcceptedBy { + sender: *sender, + msg: ConnectResponse::AcceptedBy { peers: accepted_by, your_location: new_location, your_peer_id: new_peer_id, @@ -282,18 +310,19 @@ impl Operation for JoinRingOp { // is that we would end up with a dead connection; // this then must be dealed with by the normal mechanisms that keep // connections alive and prune any dead connections - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); tracing::debug!( - "Sending response to join request with all the peers that accepted \ + tx = %id, + "Sending response to connect request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", sender.peer, own_loc.peer ); - return_msg = Some(JoinRingMsg::Response { - id, + return_msg = Some(ConnectMsg::Response { + id: *id, target, - sender, - msg: JoinResponse::Proxy { accepted_by }, + sender: *sender, + msg: ConnectResponse::Proxy { accepted_by }, }); } } @@ -302,155 +331,192 @@ impl Operation for JoinRingOp { if let Some(state) = new_state { if state.is_connected() { new_state = None; + op_storage.completed(*id); } else { new_state = Some(state); } }; } } - JoinRingMsg::Response { + ConnectMsg::Response { id, sender, msg: - JoinResponse::AcceptedBy { + ConnectResponse::AcceptedBy { peers: accepted_by, your_location, your_peer_id, }, .. } => { - tracing::debug!("Join response received from {}", sender.peer); + tracing::debug!(tx = %id, "Connect response received from {}", sender.peer); // Set the given location let pk_loc = PeerKeyLocation { - location: Some(your_location), - peer: your_peer_id, + location: Some(*your_location), + peer: *your_peer_id, }; - match self.state { - Some(JRState::Connecting(ConnectionInfo { gateway, .. })) => { - if !accepted_by.clone().is_empty() { - tracing::debug!( - "OC received and acknowledged at requesting peer {} from gateway {}", - your_peer_id, - gateway.peer - ); - new_state = Some(JRState::OCReceived); - return_msg = Some(JoinRingMsg::Response { - id, - msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, + // fixme: remove + tracing::debug!("accepted by state: {:?} ", self.state,); + let Some(ConnectState::Connecting(ConnectionInfo { gateway, .. })) = self.state + else { + return Err(OpError::InvalidStateTransition(self.id)); + }; + if !accepted_by.is_empty() { + tracing::debug!("accepted by list: {:?} ", accepted_by); + tracing::debug!( + tx = %id, + "OC received and acknowledged at requesting peer {} from gateway {}", + your_peer_id, + gateway.peer + ); + new_state = Some(ConnectState::OCReceived); + return_msg = Some(ConnectMsg::Response { + id: *id, + msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, + sender: pk_loc, + target: *sender, + }); + tracing::debug!( + tx = %id, + this_peer = %your_peer_id, + location = %your_location, + "Updating assigned location" + ); + op_storage.ring.update_location(Some(*your_location)); + + for other_peer in accepted_by { + let _ = propagate_oc_to_accepted_peers( + conn_manager, + op_storage, + *sender, + other_peer, + ConnectMsg::Response { + id: *id, + target: *other_peer, sender: pk_loc, - target: sender, - }); - } + msg: ConnectResponse::ReceivedOC { by_peer: pk_loc }, + }, + ) + .await; } - _ => return Err(OpError::InvalidStateTransition(self.id)), - }; - - op_storage.ring.update_location(Some(your_location)); - - for other_peer in accepted_by { - let _ = propagate_oc_to_accepted_peers( - conn_manager, - op_storage, - sender, - &other_peer, - JoinRingMsg::Response { - id, - target: other_peer, - sender: pk_loc, - msg: JoinResponse::ReceivedOC { by_peer: pk_loc }, - }, - ) - .await; + } else { + // no connections accepted, failed + tracing::debug!( + tx = %id, + peer = %your_peer_id, + "Failed to establish any connections, aborting" + ); + let op = ConnectOp { + id: *id, + state: None, + gateway: self.gateway, + backoff: self.backoff, + _ttl: self._ttl, + }; + op_storage + .notify_op_change( + Message::Aborted(*id), + OpEnum::Connect(op.into()), + None, + ) + .await?; + return Err(OpError::StatePushed); } - op_storage.ring.update_location(Some(your_location)); } - JoinRingMsg::Response { + ConnectMsg::Response { id, sender, target, - msg: JoinResponse::Proxy { mut accepted_by }, + msg: ConnectResponse::Proxy { accepted_by }, } => { - tracing::debug!("Received proxy join at @ {}", target.peer); + tracing::debug!(tx = %id, "Received proxy connect at @ {}", target.peer); match self.state { - Some(JRState::Initializing) => { + Some(ConnectState::Initializing) => { // the sender of the response is the target of the request and // is only a completed tx if it accepted the connection - if accepted_by.contains(&sender) { + if accepted_by.contains(sender) { tracing::debug!( - "Return to {}, connected at proxy {} (tx: {})", + tx = %id, + "Return to {}, connected at proxy {}", target.peer, sender.peer, - id ); - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); } else { tracing::debug!("Failed to connect at proxy {}", sender.peer); new_state = None; + op_storage.completed(*id); } - return_msg = Some(JoinRingMsg::Response { - msg: JoinResponse::Proxy { accepted_by }, - sender, - id, - target, + return_msg = Some(ConnectMsg::Response { + msg: ConnectResponse::Proxy { + accepted_by: accepted_by.clone(), + }, + sender: *sender, + id: *id, + target: *target, }); } - Some(JRState::AwaitingProxyResponse { + Some(ConnectState::AwaitingProxyResponse { accepted_by: mut previously_accepted, new_peer_id, - target: state_target, + target: original_target, new_location, }) => { // Check if the response reached the target node and if the request // has been accepted by any node let is_accepted = !accepted_by.is_empty(); - let is_target_peer = new_peer_id == state_target.peer; + let is_target_peer = new_peer_id == original_target.peer; if is_accepted { - previously_accepted.extend(accepted_by.drain()); + previously_accepted.extend(accepted_by.iter().copied()); if is_target_peer { - new_state = Some(JRState::OCReceived); + new_state = Some(ConnectState::OCReceived); } else { // for proxies just consider the connection open directly // what would happen in case that the connection is not confirmed end-to-end // is that we would end up with a dead connection; // this then must be dealed with by the normal mechanisms that keep // connections alive and prune any dead connections - new_state = Some(JRState::Connected); + new_state = Some(ConnectState::Connected); } } if is_target_peer { tracing::debug!( - "Sending response to join request with all the peers that accepted \ + tx = %id, + "Sending response to connect request with all the peers that accepted \ connection from gateway {} to peer {}", target.peer, - state_target.peer + original_target.peer ); - return_msg = Some(JoinRingMsg::Response { - id, - target: state_target, - sender: target, - msg: JoinResponse::AcceptedBy { - peers: accepted_by, + return_msg = Some(ConnectMsg::Response { + id: *id, + target: original_target, + sender: *target, + msg: ConnectResponse::AcceptedBy { + peers: previously_accepted, your_location: new_location, your_peer_id: new_peer_id, }, }); } else { tracing::debug!( - "Sending response to join request with all the peers that accepted \ + tx = %id, + "Sending response to connect request with all the peers that accepted \ connection from proxy peer {} to proxy peer {}", target.peer, - state_target.peer + original_target.peer ); - return_msg = Some(JoinRingMsg::Response { - id, - target: state_target, - sender: target, - msg: JoinResponse::Proxy { accepted_by }, + return_msg = Some(ConnectMsg::Response { + id: *id, + target: original_target, + sender: *target, + msg: ConnectResponse::Proxy { + accepted_by: previously_accepted, + }, }); } } @@ -459,60 +525,63 @@ impl Operation for JoinRingOp { if let Some(state) = new_state { if state.is_connected() { new_state = None; + op_storage.completed(*id); } else { new_state = Some(state) } }; } - JoinRingMsg::Response { + ConnectMsg::Response { id, sender, - msg: JoinResponse::ReceivedOC { by_peer }, + msg: ConnectResponse::ReceivedOC { by_peer }, target, } => { match self.state { - Some(JRState::OCReceived) => { - tracing::debug!("Acknowledge connected at gateway"); - new_state = Some(JRState::Connected); - return_msg = Some(JoinRingMsg::Connected { - id, - sender: target, - target: sender, + Some(ConnectState::OCReceived) => { + tracing::debug!(tx = %id, "Acknowledge connected at gateway"); + new_state = Some(ConnectState::Connected); + return_msg = Some(ConnectMsg::Connected { + id: *id, + sender: *target, + target: *sender, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), } if let Some(state) = new_state { if !state.is_connected() { - return Err(OpError::InvalidStateTransition(id)); + return Err(OpError::InvalidStateTransition(*id)); } else { conn_manager.add_connection(sender.peer).await?; op_storage.ring.add_connection( sender.location.ok_or(ConnectionError::LocationUnknown)?, sender.peer, ); - tracing::debug!("Opened connection with peer {}", by_peer.peer); + tracing::debug!(tx = %id, "Opened connection with peer {}", by_peer.peer); new_state = None; + op_storage.completed(*id); } }; } - JoinRingMsg::Connected { target, sender, id } => { + ConnectMsg::Connected { target, sender, id } => { match self.state { - Some(JRState::OCReceived) => { - tracing::debug!("Acknowledge connected at peer"); - new_state = Some(JRState::Connected); + Some(ConnectState::OCReceived) => { + tracing::debug!(tx = %id, "Acknowledge connected at peer {}", target.peer); + new_state = Some(ConnectState::Connected); return_msg = None; } _ => return Err(OpError::InvalidStateTransition(self.id)), }; if let Some(state) = new_state { if !state.is_connected() { - return Err(OpError::InvalidStateTransition(id)); + return Err(OpError::InvalidStateTransition(*id)); } else { tracing::info!( - "Successfully completed connection @ {}, new location = {:?}", + tx = %id, + assigned_location = ?op_storage.ring.own_location().location, + "Successfully completed connection @ {}", target.peer, - op_storage.ring.own_location().location ); conn_manager.add_connection(sender.peer).await?; op_storage.ring.add_connection( @@ -520,6 +589,7 @@ impl Operation for JoinRingOp { sender.peer, ); new_state = None; + op_storage.completed(*id); } }; } @@ -540,13 +610,13 @@ impl Operation for JoinRingOp { fn build_op_result( id: Transaction, - state: Option, - msg: Option, + state: Option, + msg: Option, gateway: Box, backoff: Option, ttl: Duration, ) -> Result { - let output_op = Some(JoinRingOp { + let output_op = Some(ConnectOp { id, state, gateway, @@ -555,7 +625,7 @@ fn build_op_result( }); Ok(OperationResult { return_msg: msg.map(Message::from), - state: output_op.map(|op| OpEnum::JoinRing(Box::new(op))), + state: output_op.map(|op| OpEnum::Connect(Box::new(op))), }) } @@ -564,21 +634,21 @@ fn try_proxy_connection( sender: &PeerKeyLocation, own_loc: &PeerKeyLocation, accepted_by: HashSet, -) -> (Option, Option) { +) -> (Option, Option) { let new_state = if accepted_by.contains(own_loc) { tracing::debug!( - "Return to {}, connected at proxy {} (tx: {})", + tx = %id, + "Return to {}, connected at proxy {}", sender.peer, own_loc.peer, - id ); - Some(JRState::Connected) + Some(ConnectState::Connected) } else { - tracing::debug!("Failed to connect at proxy {}", sender.peer); + tracing::debug!(tx = %id, "Failed to connect at proxy {}", sender.peer); None }; - let return_msg = Some(JoinRingMsg::Response { - msg: JoinResponse::Proxy { accepted_by }, + let return_msg = Some(ConnectMsg::Response { + msg: ConnectResponse::Proxy { accepted_by }, sender: *own_loc, id: *id, target: *sender, @@ -591,14 +661,15 @@ async fn propagate_oc_to_accepted_peers( op_storage: &OpManager, sender: PeerKeyLocation, other_peer: &PeerKeyLocation, - msg: JoinRingMsg, + msg: ConnectMsg, ) -> Result<(), OpError> { + let id = msg.id(); if op_storage.ring.should_accept( &other_peer .location .ok_or(ConnectionError::LocationUnknown)?, ) { - tracing::info!("Established connection to {}", other_peer.peer); + tracing::info!(tx = %id, "Establishing connection to {}", other_peer.peer); conn_manager.add_connection(other_peer.peer).await?; op_storage.ring.add_connection( other_peer @@ -612,7 +683,7 @@ async fn propagate_oc_to_accepted_peers( let _ = conn_manager.send(&other_peer.peer, msg.into()).await; } } else { - tracing::debug!("Not accepting connection to {}", other_peer.peer); + tracing::debug!(tx = %id, "Not accepting connection to {}", other_peer.peer); } Ok(()) @@ -622,7 +693,7 @@ mod states { use super::*; use std::fmt::Display; - impl Display for JRState { + impl Display for ConnectState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Initializing => write!(f, "Initializing"), @@ -637,11 +708,12 @@ mod states { } } -enum JRState { +#[derive(Debug)] +enum ConnectState { Initializing, Connecting(ConnectionInfo), AwaitingProxyResponse { - /// Could be either the requester or nodes which have been previously forwarded to + /// Could be either the joiner or nodes which have been previously forwarded to target: PeerKeyLocation, accepted_by: HashSet, new_location: Location, @@ -658,7 +730,7 @@ struct ConnectionInfo { max_hops_to_live: usize, } -impl JRState { +impl ConnectState { fn try_unwrap_connecting(self) -> Result { if let Self::Connecting(conn_info) = self { Ok(conn_info) @@ -668,7 +740,7 @@ impl JRState { } fn is_connected(&self) -> bool { - matches!(self, JRState::Connected { .. }) + matches!(self, ConnectState::Connected { .. }) } fn add_new_proxy( @@ -689,20 +761,26 @@ pub(crate) fn initial_request( gateway: PeerKeyLocation, max_hops_to_live: usize, id: Transaction, -) -> JoinRingOp { - tracing::debug!("Connecting to gw {} from {}", gateway.peer, this_peer); - let state = JRState::Connecting(ConnectionInfo { +) -> ConnectOp { + const MAX_JOIN_RETRIES: usize = 3; + tracing::debug!(tx = %id, "Connecting to gateway {} from {}", gateway.peer, this_peer); + let state = ConnectState::Connecting(ConnectionInfo { gateway, this_peer, max_hops_to_live, }); - JoinRingOp { + let ceiling = if cfg!(test) { + Duration::from_secs(1) + } else { + Duration::from_secs(120) + }; + ConnectOp { id, state: Some(state), gateway: Box::new(gateway), backoff: Some(ExponentialBackoff::new( Duration::from_secs(1), - Duration::from_secs(120), + ceiling, MAX_JOIN_RETRIES, )), _ttl: PEER_TIMEOUT, @@ -710,16 +788,16 @@ pub(crate) fn initial_request( } /// Join ring routine, called upon performing a join operation for this node. -pub(crate) async fn join_ring_request( +pub(crate) async fn connect_request( tx: Transaction, op_storage: &OpManager, conn_manager: &mut CB, - join_op: JoinRingOp, + join_op: ConnectOp, ) -> Result<(), OpError> where CB: ConnectionBridge, { - let JoinRingOp { + let ConnectOp { id, state, backoff, @@ -733,18 +811,20 @@ where } = state.expect("infallible").try_unwrap_connecting()?; tracing::info!( - "Joining ring via {} (at {}) (tx: {})", + tx = %id, + "Connecting to peer {} (at {})", gateway.peer, gateway.location.ok_or(ConnectionError::LocationUnknown)?, - tx ); conn_manager.add_connection(gateway.peer).await?; - let join_req = Message::from(messages::JoinRingMsg::Request { + let assigned_location = op_storage.ring.own_location().location; + let join_req = Message::from(messages::ConnectMsg::Request { id: tx, - msg: messages::JoinRequest::StartReq { + msg: messages::ConnectRequest::StartReq { target: gateway, - req_peer: this_peer, + joiner: this_peer, + assigned_location, hops_to_live: max_hops_to_live, max_hops_to_live, }, @@ -752,9 +832,9 @@ where conn_manager.send(&gateway.peer, join_req).await?; op_storage.push( tx, - OpEnum::JoinRing(Box::new(JoinRingOp { + OpEnum::Connect(Box::new(ConnectOp { id, - state: Some(JRState::Connecting(ConnectionInfo { + state: Some(ConnectState::Connecting(ConnectionInfo { gateway, this_peer, max_hops_to_live, @@ -773,63 +853,81 @@ async fn forward_conn( ring: &Ring, conn_manager: &mut CM, req_peer: PeerKeyLocation, - new_peer_loc: PeerKeyLocation, + joiner: PeerKeyLocation, left_htl: usize, num_accepted: usize, -) -> Result, OpError> +) -> Result, OpError> where CM: ConnectionBridge, { - if left_htl == 0 || (ring.num_connections() == 0 && num_accepted == 0) { + if left_htl == 0 { + tracing::debug!( + tx = %id, + joiner = %joiner.peer, + "Couldn't forward connect petition, no hops left or enough connections", + ); + return Ok(None); + } + + if ring.num_connections() == 0 { + tracing::warn!( + tx = %id, + joiner = %joiner.peer, + "Couldn't forward connect petition, not enough connections", + ); return Ok(None); } let forward_to = if left_htl >= ring.rnd_if_htl_above { tracing::debug!( - "Randomly selecting peer to forward JoinRequest (requester: {})", - req_peer.peer + tx = %id, + joiner = %joiner.peer, + "Randomly selecting peer to forward connect request", ); - ring.random_peer(|p| p.peer != req_peer.peer) + ring.random_peer(|p| p != &req_peer.peer) } else { tracing::debug!( - "Selecting close peer to forward request (requester: {})", - req_peer.peer + tx = %id, + joiner = %joiner.peer, + "Selecting close peer to forward request", ); - ring.routing(&new_peer_loc.location.unwrap(), Some(&req_peer.peer), &[]) - .and_then(|pkl| (pkl.peer != new_peer_loc.peer).then_some(pkl)) + // FIXME: target the `desired_location` + ring.routing(&joiner.location.unwrap(), Some(&req_peer.peer), &[]) + .and_then(|pkl| (pkl.peer != joiner.peer).then_some(pkl)) }; if let Some(forward_to) = forward_to { - let forwarded = Message::from(JoinRingMsg::Request { + let forwarded = Message::from(ConnectMsg::Request { id, - msg: JoinRequest::Proxy { - joiner: new_peer_loc, + msg: ConnectRequest::Proxy { + joiner, hops_to_live: left_htl.min(ring.max_hops_to_live) - 1, sender: ring.own_location(), }, }); tracing::debug!( - "Forwarding JoinRequest from sender {} to {}", + tx = %id, + "Forwarding connect request from sender {} to {}", req_peer.peer, forward_to.peer ); conn_manager.send(&forward_to.peer, forwarded).await?; // awaiting for responses from forward nodes - let new_state = JRState::AwaitingProxyResponse { + let new_state = ConnectState::AwaitingProxyResponse { target: req_peer, accepted_by: HashSet::new(), - new_location: new_peer_loc.location.unwrap(), - new_peer_id: new_peer_loc.peer, + new_location: joiner.location.unwrap(), + new_peer_id: joiner.peer, }; Ok(Some(new_state)) } else { if num_accepted != 0 { tracing::warn!( - "Unable to forward, will only be connected to one peer (tx: {})", - id + tx = %id, + "Unable to forward, will only be connecting to one peer", ); } else { - tracing::warn!("Unable to forward or accept any connections (tx: {})", id); + tracing::warn!(tx = %id, "Unable to forward or accept any connections"); } Ok(None) } @@ -844,17 +942,17 @@ mod messages { use crate::message::InnerMessage; use serde::{Deserialize, Serialize}; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum JoinRingMsg { + #[derive(Debug, Serialize, Deserialize)] + pub(crate) enum ConnectMsg { Request { id: Transaction, - msg: JoinRequest, + msg: ConnectRequest, }, Response { id: Transaction, sender: PeerKeyLocation, target: PeerKeyLocation, - msg: JoinResponse, + msg: ConnectResponse, }, Connected { id: Transaction, @@ -863,7 +961,7 @@ mod messages { }, } - impl InnerMessage for JoinRingMsg { + impl InnerMessage for ConnectMsg { fn id(&self) -> &Transaction { match self { Self::Request { id, .. } => id, @@ -871,28 +969,13 @@ mod messages { Self::Connected { id, .. } => id, } } - } - - impl JoinRingMsg { - pub fn sender(&self) -> Option<&PeerKey> { - use JoinRingMsg::*; - match self { - Response { sender, .. } => Some(&sender.peer), - Connected { sender, .. } => Some(&sender.peer), - Request { - msg: JoinRequest::StartReq { req_peer, .. }, - .. - } => Some(req_peer), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { - use JoinRingMsg::*; + fn target(&self) -> Option<&PeerKeyLocation> { + use ConnectMsg::*; match self { Response { target, .. } => Some(target), Request { - msg: JoinRequest::StartReq { target, .. }, + msg: ConnectRequest::StartReq { target, .. }, .. } => Some(target), Connected { target, .. } => Some(target), @@ -900,66 +983,76 @@ mod messages { } } - pub fn terminal(&self) -> bool { - use JoinRingMsg::*; + fn terminal(&self) -> bool { + use ConnectMsg::*; matches!( self, Response { - msg: JoinResponse::Proxy { .. }, + msg: ConnectResponse::Proxy { .. }, .. } | Connected { .. } ) } } - impl Display for JoinRingMsg { + impl ConnectMsg { + pub fn sender(&self) -> Option<&PeerKey> { + use ConnectMsg::*; + match self { + Response { sender, .. } => Some(&sender.peer), + Connected { sender, .. } => Some(&sender.peer), + Request { + msg: + ConnectRequest::StartReq { + joiner: req_peer, .. + }, + .. + } => Some(req_peer), + _ => None, + } + } + } + + impl Display for ConnectMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); match self { Self::Request { - msg: JoinRequest::StartReq { .. }, + msg: ConnectRequest::StartReq { .. }, .. } => write!(f, "StartRequest(id: {id})"), Self::Request { - msg: JoinRequest::Accepted { .. }, + msg: ConnectRequest::Accepted { .. }, .. } => write!(f, "RequestAccepted(id: {id})"), Self::Request { - msg: JoinRequest::Proxy { .. }, + msg: ConnectRequest::Proxy { .. }, .. } => write!(f, "ProxyRequest(id: {id})"), Self::Response { - msg: JoinResponse::AcceptedBy { .. }, + msg: ConnectResponse::AcceptedBy { .. }, .. } => write!(f, "RouteValue(id: {id})"), Self::Response { - msg: JoinResponse::ReceivedOC { .. }, + msg: ConnectResponse::ReceivedOC { .. }, .. } => write!(f, "RouteValue(id: {id})"), Self::Response { - msg: JoinResponse::Proxy { .. }, + msg: ConnectResponse::Proxy { .. }, .. } => write!(f, "RouteValue(id: {id})"), Self::Connected { .. } => write!(f, "Connected(id: {id})"), - _ => todo!(), + _ => unimplemented!(), } } } - /* - - Peer A ---> Peer B (forward) ----> Peer C - |----- (forward) ---------> Peer D - - - Peer A ---> Peer B (forward) ----> Peer C ----> Peer D - - */ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum JoinRequest { + pub(crate) enum ConnectRequest { StartReq { target: PeerKeyLocation, - req_peer: PeerKey, + joiner: PeerKey, + assigned_location: Option, hops_to_live: usize, max_hops_to_live: usize, }, @@ -978,7 +1071,7 @@ mod messages { } #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum JoinResponse { + pub(crate) enum ConnectResponse { AcceptedBy { peers: HashSet, your_location: Location, @@ -997,37 +1090,67 @@ mod messages { mod test { use std::time::Duration; - use crate::node::tests::{check_connectivity, SimNetwork}; + use crate::node::tests::SimNetwork; /// Given a network of one node and one gateway test that both are connected. - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn one_node_connects_to_gw() { - let mut sim_nodes = SimNetwork::new(1, 1, 1, 1, 2, 2).await; - sim_nodes.build().await; + let mut sim_nodes = SimNetwork::new("join_one_node_connects_to_gw", 1, 1, 1, 1, 2, 2).await; + sim_nodes.start().await; tokio::time::sleep(Duration::from_secs(3)).await; - assert!(sim_nodes.connected("node-0")); + assert!(sim_nodes.connected(&"node-0".into())); } /// Once a gateway is left without remaining open slots, ensure forwarding connects - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn forward_connection_to_node() -> Result<(), anyhow::Error> { - const NUM_NODES: usize = 10usize; + const NUM_NODES: usize = 3usize; const NUM_GW: usize = 1usize; - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - sim_nodes.build().await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await + let mut sim_nw = SimNetwork::new( + "join_forward_connection_to_node", + NUM_GW, + NUM_NODES, + 2, + 1, + 2, + 1, + ) + .await; + // sim_nw.with_start_backoff(Duration::from_millis(100)); + sim_nw.start().await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; + let some_forwarded = sim_nw + .node_connectivity() + .into_iter() + .flat_map(|(_this, (_, conns))| conns.into_keys()) + .any(|c| c.is_node()); + assert!( + some_forwarded, + "didn't find any connection succesfully forwarded" + ); + Ok(()) } - /// Given a network of N peers all nodes should have connections. - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + /// Given a network of N peers all good connectivity #[ignore] - async fn all_nodes_should_connect() -> Result<(), anyhow::Error> { + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn network_should_achieve_good_connectivity() -> Result<(), anyhow::Error> { + crate::config::set_logger(); const NUM_NODES: usize = 10usize; - const NUM_GW: usize = 1usize; - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 1000, 2).await; - sim_nodes.build().await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(10)).await + const NUM_GW: usize = 2usize; + let mut sim_nw = SimNetwork::new( + "join_all_nodes_should_connect", + NUM_GW, + NUM_NODES, + 5, + 3, + 6, + 2, + ) + .await; + sim_nw.with_start_backoff(Duration::from_millis(200)); + sim_nw.start().await; + sim_nw.check_connectivity(Duration::from_secs(10)).await?; + sim_nw.network_connectivity_quality() } } diff --git a/crates/core/src/operations/get.rs b/crates/core/src/operations/get.rs index 56d03bc11..97dff1e17 100644 --- a/crates/core/src/operations/get.rs +++ b/crates/core/src/operations/get.rs @@ -8,7 +8,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::{ContractError, ContractHandlerEvent, StoreResponse}, - message::{InnerMessage, Message, Transaction, TxType}, + message::{InnerMessage, Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{Location, PeerKeyLocation, RingError}, @@ -29,7 +29,7 @@ const MAX_GET_RETRY_HOPS: usize = 1; pub(crate) struct GetOp { id: Transaction, state: Option, - result: Option, + pub(super) result: Option, stats: Option, _ttl: Duration, } @@ -86,10 +86,7 @@ impl GetOp { } pub(super) fn finalized(&self) -> bool { - self.stats - .as_ref() - .map(|s| s.transfer_time.is_some()) - .unwrap_or(false) + self.result.is_some() } pub(super) fn record_transfer(&mut self) { @@ -132,7 +129,7 @@ impl TryFrom for GetResult { fn try_from(value: GetOp) -> Result { match value.result { Some(r) => Ok(r), - _ => todo!(), + _ => Err(OpError::UnexpectedOpState), } } } @@ -150,13 +147,16 @@ impl Operation for GetOp { sender = Some(peer_key_loc.peer); }; let tx = *msg.id(); - let result = match op_storage.pop(msg.id()) { - Some(OpEnum::Get(get_op)) => { + match op_storage.pop(msg.id()) { + Ok(Some(OpEnum::Get(get_op))) => { Ok(OpInitialization { op: get_op, sender }) // was an existing operation, other peer messaged back } - Some(_) => return Err(OpError::OpNotPresent(tx)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(tx, op); + Err(OpError::OpNotPresent(tx)) + } + Ok(None) => { // new request to get a value for a contract, initialize the machine Ok(OpInitialization { op: Self { @@ -169,9 +169,8 @@ impl Operation for GetOp { sender, }) } - }; - - result + Err(err) => Err(err.into()), + } } fn id(&self) -> &Transaction { @@ -182,7 +181,7 @@ impl Operation for GetOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -203,21 +202,23 @@ impl Operation for GetOp { self.state, Some(GetState::AwaitingResponse { .. }) )); - tracing::debug!("Seek contract {} @ {} (tx: {})", key, target.peer, id); + tracing::debug!(tx = %id, "Seek contract {} @ {}", key, target.peer); new_state = self.state; stats = Some(GetStats { - contract_location: Location::from(&key), + contract_location: Location::from(key), caching_peer: None, transfer_time: None, first_response_time: None, step: Default::default(), }); + let own_loc = op_storage.ring.own_location(); return_msg = Some(GetMsg::SeekNode { - key, - id, - target, - sender: op_storage.ring.own_location(), - fetch_contract, + key: key.clone(), + id: *id, + target: *target, + skip_list: vec![own_loc.peer], + sender: own_loc, + fetch_contract: *fetch_contract, htl: MAX_GET_RETRY_HOPS, }); } @@ -227,15 +228,24 @@ impl Operation for GetOp { fetch_contract, sender, target, + skip_list, htl, } => { + let htl = *htl; + let id = *id; + let key: ContractKey = key.clone(); + let fetch_contract = *fetch_contract; + let is_cached_contract = op_storage.ring.is_contract_cached(&key); if let Some(s) = stats.as_mut() { - s.caching_peer = Some(target); + s.caching_peer = Some(*target); } + let mut skip_list = skip_list.clone(); + skip_list.push(target.peer); if !is_cached_contract { tracing::warn!( + tx = %id, "Contract `{}` not found while processing a get request at node @ {}", key, target.peer @@ -243,6 +253,7 @@ impl Operation for GetOp { if htl == 0 { tracing::warn!( + tx = %id, "The maximum HOPS number has been exceeded, sending the error \ back to the node @ {}", sender.peer @@ -259,7 +270,8 @@ impl Operation for GetOp { contract: None, }, sender: op_storage.ring.own_location(), - target: sender, // return to requester + updated_skip_list: skip_list, + target: *sender, // return to requester }), None, stats, @@ -268,13 +280,11 @@ impl Operation for GetOp { } let new_htl = htl - 1; - let Some(new_target) = - op_storage.ring.closest_caching(&key, &[sender.peer]) + let Some(new_target) = op_storage.ring.closest_caching(&key, &skip_list) else { - tracing::warn!("no peer found while trying getting contract {key}"); + tracing::warn!(tx = %id, "No other peers found while trying getting contract {key} @ {}", target.peer); return Err(OpError::RingError(RingError::NoCachingPeers(key))); }; - continue_seeking( conn_manager, &new_target, @@ -282,8 +292,9 @@ impl Operation for GetOp { id, key, fetch_contract, - sender, + sender: *sender, target: new_target, + skip_list, htl: new_htl, }) .into(), @@ -293,8 +304,8 @@ impl Operation for GetOp { return_msg = None; new_state = None; } else if let ContractHandlerEvent::GetResponse { - response: value, key: returned_key, + response: value, } = op_storage .notify_contract_handler( ContractHandlerEvent::GetQuery { @@ -316,26 +327,35 @@ impl Operation for GetOp { Err(err) => return Err(err), } - tracing::debug!("Contract {returned_key} found @ peer {}", target.peer); + tracing::debug!(tx = %id, "Contract {returned_key} found @ peer {}", target.peer); match self.state { Some(GetState::AwaitingResponse { .. }) => { tracing::debug!( - "Completed operation, Get response received for contract {key}" + tx = %id, + "Completed operation, get response received for contract {key}" ); // Completed op new_state = None; return_msg = None; } Some(GetState::ReceivedRequest) => { - tracing::debug!("Returning contract {} to {}", key, sender.peer); + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; + let value = match value { + Ok(res) => res, + Err(err) => { + tracing::error!(tx = %id, "error: {err}"); + return Err(OpError::ExecutorError(err)); + } + }; return_msg = Some(GetMsg::ReturnGet { id, key, - value: value.unwrap(), - sender: target, - target: sender, + value, + updated_skip_list: vec![], + sender: *target, + target: *sender, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -354,10 +374,11 @@ impl Operation for GetOp { }, sender, target, - .. + updated_skip_list, } => { let this_loc = target; tracing::warn!( + tx = %id, "Neither contract or contract value for contract `{}` found at peer {}, \ retrying with other peers", key, @@ -377,20 +398,21 @@ impl Operation for GetOp { skip_list.push(target.peer); if let Some(target) = op_storage .ring - .closest_caching(&key, skip_list.as_slice()) + .closest_caching(key, skip_list.as_slice()) .into_iter() .next() { return_msg = Some(GetMsg::SeekNode { - id, - key, + id: *id, + key: key.clone(), target, - sender: this_loc, + sender: *this_loc, fetch_contract, htl: MAX_GET_RETRY_HOPS, + skip_list: updated_skip_list.clone(), }); } else { - return Err(RingError::NoCachingPeers(key).into()); + return Err(RingError::NoCachingPeers(key.clone()).into()); } new_state = Some(GetState::AwaitingResponse { skip_list, @@ -399,24 +421,26 @@ impl Operation for GetOp { }); } else { tracing::error!( + tx = %id, "Failed getting a value for contract {}, reached max retries", key ); - return Err(OpError::MaxRetriesExceeded(id, "get".to_owned())); + return Err(OpError::MaxRetriesExceeded(*id, id.tx_type())); } } Some(GetState::ReceivedRequest) => { - tracing::debug!("Returning contract {} to {}", key, sender.peer); + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; return_msg = Some(GetMsg::ReturnGet { - id, - key, + id: *id, + key: key.clone(), value: StoreResponse { state: None, contract: None, }, - sender, - target, + updated_skip_list: updated_skip_list.clone(), + sender: *sender, + target: *target, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -432,7 +456,12 @@ impl Operation for GetOp { }, sender, target, + updated_skip_list, } => { + let id = *id; + let key = key.clone(); + let mut updated_skip_list = updated_skip_list.clone(); + updated_skip_list.push(sender.peer); let require_contract = matches!( self.state, Some(GetState::AwaitingResponse { @@ -445,18 +474,28 @@ impl Operation for GetOp { if require_contract { if let Some(contract) = &contract { // store contract first - op_storage + let res = op_storage .notify_contract_handler( ContractHandlerEvent::Cache(contract.clone()), client_id, ) .await?; + match res { + ContractHandlerEvent::CacheResult(Ok(_)) => { + op_storage.ring.contract_cached(&key); + } + ContractHandlerEvent::CacheResult(Err(err)) => { + return Err(OpError::ContractError(err)); + } + _ => unreachable!(), + } let key = contract.key(); - tracing::debug!("Contract `{}` successfully cached", key); + tracing::debug!(tx = %id, "Contract `{}` successfully cached", key); } else { // no contract, consider this like an error ignoring the incoming update value tracing::warn!( - "Contract not received from peer {} while requested", + tx = %id, + "Contract not received from peer {} while required", sender.peer ); @@ -477,8 +516,9 @@ impl Operation for GetOp { state: None, contract: None, }, - sender, - target, + sender: *sender, + target: *target, + updated_skip_list, }), OpEnum::Get(op), None, @@ -488,40 +528,54 @@ impl Operation for GetOp { } } - op_storage + let parameters = contract.as_ref().map(|c| c.params()); + let res = op_storage .notify_contract_handler( ContractHandlerEvent::PutQuery { key: key.clone(), state: value.clone(), + related_contracts: RelatedContracts::default(), + parameters, }, client_id, ) .await?; + match res { + ContractHandlerEvent::PutResponse { new_value: Ok(_) } => {} + ContractHandlerEvent::PutResponse { + new_value: Err(err), + } => { + tracing::debug!(tx = %id, error = %err, "Failed put at executor"); + return Err(OpError::ExecutorError(err)); + } + _ => unreachable!(), + } match self.state { Some(GetState::AwaitingResponse { fetch_contract, .. }) => { if fetch_contract && contract.is_none() { tracing::error!( + tx = %id, "Get response received for contract {key}, but the contract wasn't returned" ); new_state = None; return_msg = None; result = Some(GetResult { state: value.clone(), - contract, + contract: contract.clone(), }); } else { - tracing::debug!("Get response received for contract {}", key); + tracing::debug!(tx = %id, "Get response received for contract {}", key); new_state = None; return_msg = None; result = Some(GetResult { state: value.clone(), - contract, + contract: contract.clone(), }); } } Some(GetState::ReceivedRequest) => { - tracing::debug!("Returning contract {} to {}", key, sender.peer); + tracing::debug!(tx = %id, "Returning contract {} to {}", key, sender.peer); new_state = None; return_msg = Some(GetMsg::ReturnGet { id, @@ -530,8 +584,9 @@ impl Operation for GetOp { state: None, contract: None, }, - sender, - target, + sender: *sender, + target: *target, + updated_skip_list, }); } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -571,13 +626,12 @@ async fn continue_seeking( new_target: &PeerKeyLocation, retry_msg: Message, ) -> Result<(), OpError> { - tracing::info!( - "Retrying to get the contract from node @ {}", + tracing::debug!( + tx = %retry_msg.id(), + "Forwarding get request to {}", new_target.peer ); - conn_manager.send(&new_target.peer, retry_msg).await?; - Ok(()) } @@ -591,6 +645,7 @@ fn check_contract_found( if returned_key != key { // shouldn't be a reachable path tracing::error!( + tx = %id, "contract retrieved ({}) and asked ({}) are not the same", returned_key, key @@ -611,11 +666,10 @@ fn check_contract_found( } } -pub(crate) fn start_op(key: ContractKey, fetch_contract: bool, this_peer: &PeerKey) -> GetOp { +pub(crate) fn start_op(key: ContractKey, fetch_contract: bool) -> GetOp { let contract_location = Location::from(&key); - tracing::debug!("Requesting get contract {} @ loc({contract_location})", key,); - - let id = Transaction::new(::tx_type_id(), this_peer); + let id = Transaction::new::(); + tracing::debug!(tx = %id, "Requesting get contract {key} @ loc({contract_location})"); let state = Some(GetState::PrepareRequest { key, id, @@ -676,9 +730,9 @@ pub(crate) async fn request_get( return Err(OpError::UnexpectedOpState); }; tracing::debug!( - "Preparing get contract request to {} (tx: {})", + tx = %id, + "Preparing get contract request to {}", target.peer, - id ); match get_op.state { @@ -730,7 +784,7 @@ mod messages { use super::*; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum GetMsg { RequestGet { id: Transaction, @@ -745,6 +799,8 @@ mod messages { target: PeerKeyLocation, sender: PeerKeyLocation, htl: usize, + // FIXME: remove skip list once we deduplicate at top msg handling level + skip_list: Vec, }, ReturnGet { id: Transaction, @@ -752,6 +808,7 @@ mod messages { value: StoreResponse, sender: PeerKeyLocation, target: PeerKeyLocation, + updated_skip_list: Vec, }, } @@ -763,17 +820,8 @@ mod messages { Self::ReturnGet { id, .. } => id, } } - } - - impl GetMsg { - pub fn sender(&self) -> Option<&PeerKeyLocation> { - match self { - Self::SeekNode { target, .. } => Some(target), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { + fn target(&self) -> Option<&PeerKeyLocation> { match self { Self::SeekNode { target, .. } => Some(target), Self::RequestGet { target, .. } => Some(target), @@ -781,12 +829,21 @@ mod messages { } } - pub fn terminal(&self) -> bool { + fn terminal(&self) -> bool { use GetMsg::*; matches!(self, ReturnGet { .. }) } } + impl GetMsg { + pub fn sender(&self) -> Option<&PeerKeyLocation> { + match self { + Self::SeekNode { target, .. } => Some(target), + _ => None, + } + } + } + impl Display for GetMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); @@ -805,9 +862,8 @@ mod test { use std::collections::HashMap; use super::*; - use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; + use crate::node::tests::{NodeSpecification, SimNetwork}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_get_op_between_nodes() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 1usize; @@ -840,26 +896,30 @@ mod test { contract_subscribers: HashMap::new(), }; - let get_specs = HashMap::from_iter([ - ("node-0".to_string(), node_0), - ("gateway-0".to_string(), gw_0), - ]); + let get_specs = HashMap::from_iter([("node-0".into(), node_0), ("gateway-0".into(), gw_0)]); // establish network - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - sim_nodes.build_with_specs(get_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + let mut sim_nw = SimNetwork::new( + "successful_get_op_between_nodes", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 2, + ) + .await; + sim_nw.start_with_spec(get_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; // trigger get @ node-0, which does not own the contract - sim_nodes - .trigger_event("node-0", 1, Some(Duration::from_millis(100))) + sim_nw + .trigger_event("node-0", 1, Some(Duration::from_millis(50))) .await?; - tokio::time::sleep(Duration::from_millis(100)).await; - assert!(sim_nodes.has_got_contract("node-0", &key)); + assert!(sim_nw.has_got_contract("node-0", &key)); Ok(()) } - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn contract_not_found() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 2usize; @@ -882,22 +942,22 @@ mod test { contract_subscribers: HashMap::new(), }; - let get_specs = HashMap::from_iter([("node-1".to_string(), node_1)]); + let get_specs = HashMap::from_iter([("node-1".into(), node_1)]); // establish network - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - sim_nodes.build_with_specs(get_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + let mut sim_nw = + SimNetwork::new("get_contract_not_found", NUM_GW, NUM_NODES, 3, 2, 4, 2).await; + sim_nw.start_with_spec(get_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; // trigger get @ node-1, which does not own the contract - sim_nodes - .trigger_event("node-1", 1, Some(Duration::from_millis(100))) + sim_nw + .trigger_event("node-1", 1, Some(Duration::from_millis(50))) .await?; - assert!(!sim_nodes.has_got_contract("node-1", &key)); + assert!(!sim_nw.has_got_contract("node-1", &key)); Ok(()) } - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn contract_found_after_retry() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 2usize; @@ -940,20 +1000,29 @@ mod test { }; let get_specs = HashMap::from_iter([ - ("node-0".to_string(), node_0), - ("node-1".to_string(), node_1), - ("gateway-0".to_string(), gw_0), + ("node-0".into(), node_0), + ("node-1".into(), node_1), + ("gateway-0".into(), gw_0), ]); // establish network - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 3).await; - sim_nodes.build_with_specs(get_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + let mut sim_nw = SimNetwork::new( + "get_contract_found_after_retry", + NUM_GW, + NUM_NODES, + 3, + 2, + 4, + 2, + ) + .await; + sim_nw.start_with_spec(get_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; - sim_nodes - .trigger_event("node-0", 1, Some(Duration::from_millis(500))) + sim_nw + .trigger_event("node-0", 1, Some(Duration::from_millis(200))) .await?; - assert!(sim_nodes.has_got_contract("node-0", &key)); + assert!(sim_nw.has_got_contract("node-0", &key)); Ok(()) } } diff --git a/crates/core/src/operations/op_trait.rs b/crates/core/src/operations/op_trait.rs index 646fec42b..3c906106f 100644 --- a/crates/core/src/operations/op_trait.rs +++ b/crates/core/src/operations/op_trait.rs @@ -31,7 +31,7 @@ where self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>>; } diff --git a/crates/core/src/operations/put.rs b/crates/core/src/operations/put.rs index 181a1ffc6..c4480836d 100644 --- a/crates/core/src/operations/put.rs +++ b/crates/core/src/operations/put.rs @@ -15,7 +15,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::ContractHandlerEvent, - message::{InnerMessage, Message, Transaction, TxType}, + message::{InnerMessage, Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{Location, PeerKeyLocation, RingError}, @@ -136,13 +136,16 @@ impl Operation for PutOp { }; let tx = *msg.id(); - let result = match op_storage.pop(msg.id()) { - Some(OpEnum::Put(put_op)) => { + match op_storage.pop(msg.id()) { + Ok(Some(OpEnum::Put(put_op))) => { // was an existing operation, the other peer messaged back Ok(OpInitialization { op: put_op, sender }) } - Some(_) => return Err(OpError::OpNotPresent(tx)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(tx, op); + Err(OpError::OpNotPresent(tx)) + } + Ok(None) => { // new request to put a new value for a contract, initialize the machine Ok(OpInitialization { op: Self { @@ -154,8 +157,8 @@ impl Operation for PutOp { sender, }) } - }; - result + Err(err) => Err(err.into()), + } } fn id(&self) -> &Transaction { @@ -166,7 +169,7 @@ impl Operation for PutOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -178,6 +181,7 @@ impl Operation for PutOp { PutMsg::RequestPut { id, contract, + related_contracts, value, htl, target, @@ -186,19 +190,20 @@ impl Operation for PutOp { let key = contract.key(); tracing::debug!( - "Performing a RequestPut for contract {} from {} to {}", + "Rquesting put for contract {} from {} to {}", key, sender.peer, target.peer ); return_msg = Some(PutMsg::SeekNode { - id, + id: *id, sender, - target, - value, - contract, - htl, + target: *target, + value: value.clone(), + contract: contract.clone(), + related_contracts: related_contracts.clone(), + htl: *htl, skip_list: vec![sender.peer], }); @@ -210,17 +215,19 @@ impl Operation for PutOp { sender, value, contract, + related_contracts, htl, target, - mut skip_list, + skip_list, } => { let key = contract.key(); let is_cached_contract = op_storage.ring.is_contract_cached(&key); tracing::debug!( - "Performing a SeekNode at {}, trying put the contract {}", + tx = %id, + "Puttting contract {} at target peer {}", + key, target.peer, - key ); if !is_cached_contract @@ -228,36 +235,48 @@ impl Operation for PutOp { .ring .within_caching_distance(&Location::from(&key)) { - tracing::debug!("Contract `{}` not cached @ peer {}", key, target.peer); - match try_to_cache_contract(op_storage, &contract, &key, client_id).await { + tracing::debug!(tx = %id, "Contract `{}` not cached @ peer {}", key, target.peer); + match try_to_cache_contract(op_storage, contract, &key, client_id).await { Ok(_) => {} Err(err) => return Err(err), } } else if !is_cached_contract { + // FIXME // in this case forward to a closer node to the target location and just wait for a response // to give back to requesting peer - // FIXME tracing::warn!( + tx = %id, "Contract {} not found while processing info, forwarding", key ); } // after the contract has been cached, push the update query - tracing::debug!("Attempting contract value update"); - let new_value = put_contract(op_storage, key.clone(), value, client_id).await?; - tracing::debug!("Contract successfully updated"); + tracing::debug!(tx = %id, "Attempting contract value update"); + let parameters = contract.params(); + let new_value = put_contract( + op_storage, + key.clone(), + value.clone(), + related_contracts.clone(), + parameters, + client_id, + ) + .await?; + tracing::debug!(tx = %id, "Contract successfully updated"); // if the change was successful, communicate this back to the requestor and broadcast the change conn_manager .send( &sender.peer, (PutMsg::SuccessfulUpdate { - id, + id: *id, new_value: new_value.clone(), }) .into(), ) .await?; + + let mut skip_list = skip_list.clone(); skip_list.push(target.peer); if let Some(new_htl) = htl.checked_sub(1) { @@ -265,9 +284,9 @@ impl Operation for PutOp { forward_changes( op_storage, conn_manager, - &contract, + contract, new_value.clone(), - id, + *id, new_htl, skip_list.as_slice(), ) @@ -280,18 +299,19 @@ impl Operation for PutOp { .map(|i| i.value().to_vec()) .unwrap_or_default(); tracing::debug!( + tx = %id, "Successfully updated a value for contract {} @ {:?}", key, target.location ); match try_to_broadcast( - (id, client_id), + (*id, client_id), op_storage, self.state, broadcast_to, key.clone(), - new_value, + (contract.params(), new_value), self._ttl, ) .await @@ -307,24 +327,32 @@ impl Operation for PutOp { id, key, new_value, + parameters, sender, sender_subscribers, } => { let target = op_storage.ring.own_location(); tracing::debug!("Attempting contract value update"); - let new_value = - put_contract(op_storage, key.clone(), new_value, client_id).await?; + let new_value = put_contract( + op_storage, + key.clone(), + new_value.clone(), + RelatedContracts::default(), + parameters.clone(), + client_id, + ) + .await?; tracing::debug!("Contract successfully updated"); let broadcast_to = op_storage .ring - .subscribers_of(&key) + .subscribers_of(key) .map(|i| { // Avoid already broadcast nodes and sender from broadcasting let mut subscribers: Vec = i.value().to_vec(); let mut avoid_list: HashSet = - sender_subscribers.into_iter().map(|pl| pl.peer).collect(); + sender_subscribers.iter().map(|pl| pl.peer).collect(); avoid_list.insert(sender.peer); subscribers.retain(|s| !avoid_list.contains(&s.peer)); subscribers @@ -337,12 +365,12 @@ impl Operation for PutOp { ); match try_to_broadcast( - (id, client_id), + (*id, client_id), op_storage, self.state, broadcast_to, - key, - new_value, + key.clone(), + (parameters.clone(), new_value), self._ttl, ) .await @@ -356,23 +384,30 @@ impl Operation for PutOp { } PutMsg::Broadcasting { id, - mut broadcast_to, - mut broadcasted_to, + broadcast_to, + broadcasted_to, key, new_value, + parameters, } => { let sender = op_storage.ring.own_location(); - let msg = PutMsg::BroadcastTo { - id, - key: key.clone(), - new_value: new_value.clone(), - sender, - sender_subscribers: broadcast_to.clone(), - }; + let mut broadcasted_to = *broadcasted_to; let mut broadcasting = Vec::with_capacity(broadcast_to.len()); - for peer in &broadcast_to { - let f = conn_manager.send(&peer.peer, msg.clone().into()); + let mut filtered_broadcast = broadcast_to + .iter() + .filter(|pk| pk.peer != sender.peer) + .collect::>(); + for peer in filtered_broadcast.iter() { + let msg = PutMsg::BroadcastTo { + id: *id, + key: key.clone(), + new_value: new_value.clone(), + sender, + sender_subscribers: broadcast_to.clone(), + parameters: parameters.clone(), + }; + let f = conn_manager.send(&peer.peer, msg.into()); broadcasting.push(f); } let error_futures = futures::future::join_all(broadcasting) @@ -391,7 +426,7 @@ impl Operation for PutOp { let mut incorrect_results = 0; for (peer_num, err) in error_futures { // remove the failed peers in reverse order - let peer = broadcast_to.remove(peer_num); + let peer = filtered_broadcast.remove(peer_num); tracing::warn!( "failed broadcasting put change to {} with error {}; dropping connection", peer.peer, @@ -407,13 +442,15 @@ impl Operation for PutOp { ); // Subscriber nodes have been notified of the change, the operation is completed + op_storage.completed(*id); return_msg = None; new_state = None; } - PutMsg::SuccessfulUpdate { .. } => { + PutMsg::SuccessfulUpdate { id, .. } => { match self.state { Some(PutState::AwaitingResponse { contract, .. }) => { tracing::debug!("Successfully updated value for {}", contract,); + op_storage.completed(*id); new_state = None; return_msg = None; } @@ -429,7 +466,7 @@ impl Operation for PutOp { contract, new_value, htl, - mut skip_list, + skip_list, } => { let key = contract.key(); let peer_loc = op_storage.ring.own_location(); @@ -445,7 +482,7 @@ impl Operation for PutOp { .ring .within_caching_distance(&Location::from(&key)); if !cached_contract && within_caching_dist { - match try_to_cache_contract(op_storage, &contract, &key, client_id).await { + match try_to_cache_contract(op_storage, contract, &key, client_id).await { Ok(_) => {} Err(err) => return Err(err), } @@ -457,9 +494,18 @@ impl Operation for PutOp { }); } // after the contract has been cached, push the update query - let new_value = put_contract(op_storage, key, new_value, client_id).await?; + let new_value = put_contract( + op_storage, + key, + new_value.clone(), + RelatedContracts::default(), + contract.params(), + client_id, + ) + .await?; - //update skip list + // update skip list + let mut skip_list = skip_list.clone(); skip_list.push(peer_loc.peer); // if successful, forward to the next closest peers (if any) @@ -467,14 +513,15 @@ impl Operation for PutOp { forward_changes( op_storage, conn_manager, - &contract, + contract, new_value, - id, + *id, new_htl, skip_list.as_slice(), ) .await; } + op_storage.completed(*id); return_msg = None; new_state = None; } @@ -505,7 +552,7 @@ fn build_op_result( }) } -async fn try_to_cache_contract<'a>( +pub(super) async fn try_to_cache_contract<'a>( op_storage: &'a OpManager, contract: &ContractContainer, key: &ContractKey, @@ -533,7 +580,7 @@ async fn try_to_broadcast( state: Option, broadcast_to: Vec, key: ContractKey, - new_value: WrappedState, + (parameters, new_value): (Parameters<'static>, WrappedState), ttl: Duration, ) -> Result<(Option, Option), OpError> { let new_state; @@ -556,6 +603,7 @@ async fn try_to_broadcast( return_msg = Some(PutMsg::Broadcasting { id, new_value, + parameters, broadcasted_to: 0, broadcast_to, key, @@ -585,9 +633,9 @@ async fn try_to_broadcast( pub(crate) fn start_op( contract: ContractContainer, + related_contracts: RelatedContracts<'static>, value: WrappedState, htl: usize, - peer: &PeerKey, ) -> PutOp { let key = contract.key(); let contract_location = Location::from(&key); @@ -596,10 +644,11 @@ pub(crate) fn start_op( key, ); - let id = Transaction::new(::tx_type_id(), peer); + let id = Transaction::new::(); // let payload_size = contract.data().len(); let state = Some(PutState::PrepareRequest { contract, + related_contracts, value, htl, }); @@ -623,6 +672,7 @@ enum PutState { ReceivedRequest, PrepareRequest { contract: ContractContainer, + related_contracts: RelatedContracts<'static>, value: WrappedState, htl: usize, }, @@ -666,13 +716,14 @@ pub(crate) async fn request_put( contract, value, htl, - .. + related_contracts, }) => { let key = contract.key(); let new_state = Some(PutState::AwaitingResponse { contract: key }); let msg = PutMsg::RequestPut { id, contract, + related_contracts, value, htl, target, @@ -699,11 +750,21 @@ async fn put_contract( op_storage: &OpManager, key: ContractKey, state: WrappedState, + related_contracts: RelatedContracts<'static>, + parameters: Parameters<'static>, client_id: Option, ) -> Result { // after the contract has been cached, push the update query match op_storage - .notify_contract_handler(ContractHandlerEvent::PutQuery { key, state }, client_id) + .notify_contract_handler( + ContractHandlerEvent::PutQuery { + key, + state, + related_contracts, + parameters: Some(parameters), + }, + client_id, + ) .await { Ok(ContractHandlerEvent::PutResponse { @@ -771,7 +832,7 @@ mod messages { use crate::message::InnerMessage; use serde::{Deserialize, Serialize}; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum PutMsg { /// Initialize the put operation by routing the value RouteValue { @@ -783,6 +844,8 @@ mod messages { RequestPut { id: Transaction, contract: ContractContainer, + #[serde(deserialize_with = "RelatedContracts::deser_related_contracts")] + related_contracts: RelatedContracts<'static>, value: WrappedState, /// max hops to live htl: usize, @@ -811,6 +874,8 @@ mod messages { target: PeerKeyLocation, value: WrappedState, contract: ContractContainer, + #[serde(deserialize_with = "RelatedContracts::deser_related_contracts")] + related_contracts: RelatedContracts<'static>, /// max hops to live htl: usize, // FIXME: remove skip list once we deduplicate at top msg handling level @@ -824,6 +889,8 @@ mod messages { broadcast_to: Vec, key: ContractKey, new_value: WrappedState, + #[serde(deserialize_with = "Parameters::deser_params")] + parameters: Parameters<'static>, }, /// Broadcasting a change to a peer, which then will relay the changes to other peers. BroadcastTo { @@ -831,6 +898,8 @@ mod messages { sender: PeerKeyLocation, key: ContractKey, new_value: WrappedState, + #[serde(deserialize_with = "Parameters::deser_params")] + parameters: Parameters<'static>, sender_subscribers: Vec, }, } @@ -848,18 +917,8 @@ mod messages { Self::BroadcastTo { id, .. } => id, } } - } - - impl PutMsg { - pub fn sender(&self) -> Option<&PeerKeyLocation> { - match self { - Self::SeekNode { sender, .. } => Some(sender), - Self::BroadcastTo { sender, .. } => Some(sender), - _ => None, - } - } - pub fn target(&self) -> Option<&PeerKeyLocation> { + fn target(&self) -> Option<&PeerKeyLocation> { match self { Self::SeekNode { target, .. } => Some(target), Self::RequestPut { target, .. } => Some(target), @@ -867,7 +926,7 @@ mod messages { } } - pub fn terminal(&self) -> bool { + fn terminal(&self) -> bool { use PutMsg::*; matches!( self, @@ -876,6 +935,16 @@ mod messages { } } + impl PutMsg { + pub fn sender(&self) -> Option<&PeerKeyLocation> { + match self { + Self::SeekNode { sender, .. } => Some(sender), + Self::BroadcastTo { sender, .. } => Some(sender), + _ => None, + } + } + } + impl Display for PutMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); @@ -901,9 +970,8 @@ mod test { use freenet_stdlib::prelude::*; use super::*; - use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; + use crate::node::tests::{NodeSpecification, SimNetwork}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn successful_put_op_between_nodes() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 2usize; @@ -916,10 +984,19 @@ mod test { let contract_val: WrappedState = gen.arbitrary()?; let new_value = WrappedState::new(Vec::from_iter(gen.arbitrary::<[u8; 20]>().unwrap())); - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - let mut locations = sim_nodes.get_locations_by_node(); - let node0_loc = locations.remove("node-0").unwrap(); - let node1_loc = locations.remove("node-1").unwrap(); + let mut sim_nw = SimNetwork::new( + "successful_put_op_between_nodes", + NUM_GW, + NUM_NODES, + 2, + 1, + 3, + 2, + ) + .await; + let mut locations = sim_nw.get_locations_by_node(); + let node0_loc = locations.remove(&"node-0".into()).unwrap(); + let node1_loc = locations.remove(&"node-1".into()).unwrap(); // both own the contract, and one triggers an update let node_0 = NodeSpecification { @@ -961,21 +1038,20 @@ mod test { // establish network let put_specs = HashMap::from_iter([ - ("node-0".to_string(), node_0), - ("node-1".to_string(), node_1), - ("gateway-0".to_string(), gw_0), + ("node-0".into(), node_0), + ("node-1".into(), node_1), + ("gateway-0".into(), gw_0), ]); - sim_nodes.build_with_specs(put_specs).await; - tokio::time::sleep(Duration::from_secs(5)).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; + sim_nw.start_with_spec(put_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; - // trigger the put op @ gw-0, this - sim_nodes - .trigger_event("gateway-0", 1, Some(Duration::from_secs(3))) + // trigger the put op @ gw-0 + sim_nw + .trigger_event("gateway-0", 1, Some(Duration::from_millis(200))) .await?; - assert!(sim_nodes.has_put_contract("gateway-0", &key, &new_value)); - assert!(sim_nodes.event_listener.contract_broadcasted(&key)); + assert!(sim_nw.has_put_contract("gateway-0", &key, &new_value)); + assert!(sim_nw.event_listener.contract_broadcasted(&key)); Ok(()) } } diff --git a/crates/core/src/operations/subscribe.rs b/crates/core/src/operations/subscribe.rs index 84b03563d..38d6b08e5 100644 --- a/crates/core/src/operations/subscribe.rs +++ b/crates/core/src/operations/subscribe.rs @@ -9,7 +9,7 @@ use crate::{ client_events::ClientId, config::PEER_TIMEOUT, contract::ContractError, - message::{Message, Transaction, TxType}, + message::{InnerMessage, Message, Transaction}, node::{ConnectionBridge, OpManager, PeerKey}, operations::{op_trait::Operation, OpInitialization}, ring::{PeerKeyLocation, RingError}, @@ -39,13 +39,16 @@ impl SubscribeOp { pub(super) fn record_transfer(&mut self) {} } -pub(crate) enum SubscribeResult {} +pub(crate) struct SubscribeResult {} impl TryFrom for SubscribeResult { type Error = OpError; - fn try_from(_value: SubscribeOp) -> Result { - todo!() + fn try_from(value: SubscribeOp) -> Result { + value + .finalized() + .then_some(SubscribeResult {}) + .ok_or(OpError::UnexpectedOpState) } } @@ -63,16 +66,19 @@ impl Operation for SubscribeOp { }; let id = *msg.id(); - let result = match op_storage.pop(msg.id()) { - Some(OpEnum::Subscribe(subscribe_op)) => { + match op_storage.pop(msg.id()) { + Ok(Some(OpEnum::Subscribe(subscribe_op))) => { // was an existing operation, the other peer messaged back Ok(OpInitialization { op: subscribe_op, sender, }) } - Some(_) => return Err(OpError::OpNotPresent(id)), - None => { + Ok(Some(op)) => { + let _ = op_storage.push(id, op); + Err(OpError::OpNotPresent(id)) + } + Ok(None) => { // new request to subcribe to a contract, initialize the machine Ok(OpInitialization { op: Self { @@ -83,8 +89,8 @@ impl Operation for SubscribeOp { sender, }) } - }; - result + Err(err) => Err(err.into()), + } } fn id(&self) -> &Transaction { @@ -95,7 +101,7 @@ impl Operation for SubscribeOp { self, conn_manager: &'a mut CB, op_storage: &'a OpManager, - input: Self::Message, + input: &'a Self::Message, client_id: Option, ) -> Pin> + Send + 'a>> { Box::pin(async move { @@ -112,9 +118,9 @@ impl Operation for SubscribeOp { let sender = op_storage.ring.own_location(); new_state = self.state; return_msg = Some(SubscribeMsg::SeekNode { - id, - key, - target, + id: *id, + key: key.clone(), + target: *target, subscriber: sender, skip_list: vec![sender.peer], htl: 0, @@ -133,24 +139,22 @@ impl Operation for SubscribeOp { OperationResult { return_msg: Some(Message::from(SubscribeMsg::ReturnSub { key: key.clone(), - id, + id: *id, subscribed: false, sender, - target: subscriber, + target: *subscriber, })), state: None, } }; - if !op_storage.ring.is_contract_cached(&key) { - tracing::info!("Contract {} not found while processing info", key); - tracing::info!("Trying to found the contract from another node"); + if !op_storage.ring.is_contract_cached(key) { + tracing::debug!(tx = %id, "Contract {} not found at {}, trying other peer", key, target.peer); - let Some(new_target) = - op_storage.ring.closest_caching(&key, &[sender.peer]) + let Some(new_target) = op_storage.ring.closest_caching(key, &[sender.peer]) else { - tracing::warn!("no peer found while trying getting contract {key}"); - return Err(OpError::RingError(RingError::NoCachingPeers(key))); + tracing::warn!(tx = %id, "No peer found while trying getting contract {key}"); + return Err(OpError::RingError(RingError::NoCachingPeers(key.clone()))); }; let new_htl = htl + 1; @@ -161,14 +165,15 @@ impl Operation for SubscribeOp { let mut new_skip_list = skip_list.clone(); new_skip_list.push(target.peer); + tracing::debug!(tx = %id, "Forward request to peer: {}", new_target.peer); // Retry seek node when the contract to subscribe has not been found in this node conn_manager .send( &new_target.peer, (SubscribeMsg::SeekNode { - id, + id: *id, key: key.clone(), - subscriber, + subscriber: *subscriber, target: new_target, skip_list: new_skip_list.clone(), htl: new_htl, @@ -176,27 +181,28 @@ impl Operation for SubscribeOp { .into(), ) .await?; - } else if op_storage.ring.add_subscriber(&key, subscriber).is_err() { + } else if op_storage.ring.add_subscriber(key, *subscriber).is_err() { // max number of subscribers for this contract reached return Ok(return_err()); } match self.state { Some(SubscribeState::ReceivedRequest) => { - tracing::info!( - "Peer {} successfully subscribed to contract {}", + tracing::debug!( + tx = %id, + "Peer {} successfully subscribed to contract {key}", subscriber.peer, - key ); - new_state = Some(SubscribeState::Completed); + new_state = None; // TODO review behaviour, if the contract is not cached should return subscribed false? return_msg = Some(SubscribeMsg::ReturnSub { - sender: target, - target: subscriber, - id, - key, + sender: *target, + target: *subscriber, + id: *id, + key: key.clone(), subscribed: true, }); + op_storage.completed(*id); } _ => return Err(OpError::InvalidStateTransition(self.id)), } @@ -209,8 +215,8 @@ impl Operation for SubscribeOp { id, } => { tracing::warn!( - "Contract `{}` not found at potential subscription provider {}", - key, + tx = %id, + "Contract `{key}` not found at potential subscription provider {}", sender.peer ); // will error out in case it has reached max number of retries @@ -224,28 +230,28 @@ impl Operation for SubscribeOp { skip_list.push(sender.peer); if let Some(target) = op_storage .ring - .closest_caching(&key, skip_list.as_slice()) + .closest_caching(key, skip_list.as_slice()) .into_iter() .next() { let subscriber = op_storage.ring.own_location(); return_msg = Some(SubscribeMsg::SeekNode { - id, - key, + id: *id, + key: key.clone(), subscriber, target, skip_list: vec![target.peer], htl: 0, }); } else { - return Err(RingError::NoCachingPeers(key).into()); + return Err(RingError::NoCachingPeers(key.clone()).into()); } new_state = Some(SubscribeState::AwaitingResponse { skip_list, retries: retries + 1, }); } else { - return Err(OpError::MaxRetriesExceeded(id, "sub".to_owned())); + return Err(OpError::MaxRetriesExceeded(*id, id.tx_type())); } } _ => return Err(OpError::InvalidStateTransition(self.id)), @@ -255,24 +261,29 @@ impl Operation for SubscribeOp { subscribed: true, key, sender, - target: _, - id: _, + id, + target, + .. } => { - tracing::warn!( - "Subscribed to `{}` not found at potential subscription provider {}", - key, - sender.peer - ); - op_storage.ring.add_subscription(key); - let _ = client_id; - // todo: should inform back to the network event loop? - match self.state { Some(SubscribeState::AwaitingResponse { .. }) => { - new_state = None; + tracing::debug!( + tx = %id, + target = ?target.peer, + this = ?op_storage.ring.own_location().peer, + "Subscribed to `{key}` at provider {}", sender.peer + ); + op_storage.ring.add_subscription(key.clone()); + // todo: should inform back to the network event loop in case a client + // is waiting for response + let _ = client_id; + new_state = Some(SubscribeState::Completed); return_msg = None; + op_storage.completed(*id); + } + _other => { + return Err(OpError::InvalidStateTransition(self.id)); } - _ => return Err(OpError::InvalidStateTransition(self.id)), } } _ => return Err(OpError::UnexpectedOpState), @@ -289,9 +300,9 @@ fn build_op_result( msg: Option, ttl: Duration, ) -> Result { - let output_op = Some(SubscribeOp { + let output_op = state.map(|state| SubscribeOp { id, - state, + state: Some(state), _ttl: ttl, }); Ok(OperationResult { @@ -300,8 +311,8 @@ fn build_op_result( }) } -pub(crate) fn start_op(key: ContractKey, peer: &PeerKey) -> SubscribeOp { - let id = Transaction::new(::tx_type_id(), peer); +pub(crate) fn start_op(key: ContractKey) -> SubscribeOp { + let id = Transaction::new::(); let state = Some(SubscribeState::PrepareRequest { id, key }); SubscribeOp { id, @@ -310,6 +321,7 @@ pub(crate) fn start_op(key: ContractKey, peer: &PeerKey) -> SubscribeOp { } } +#[derive(Debug)] enum SubscribeState { /// Prepare the request to subscribe. PrepareRequest { @@ -379,7 +391,7 @@ mod messages { use super::*; - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum SubscribeMsg { FetchRouting { id: Transaction, @@ -416,26 +428,8 @@ mod messages { Self::ReturnSub { id, .. } => id, } } - } - impl SubscribeMsg { - pub(crate) fn id(&self) -> &Transaction { - match self { - Self::SeekNode { id, .. } => id, - Self::FetchRouting { id, .. } => id, - Self::RequestSub { id, .. } => id, - Self::ReturnSub { id, .. } => id, - } - } - - pub fn sender(&self) -> Option<&PeerKeyLocation> { - match self { - Self::ReturnSub { sender, .. } => Some(sender), - _ => None, - } - } - - pub fn target(&self) -> Option<&PeerKeyLocation> { + fn target(&self) -> Option<&PeerKeyLocation> { match self { Self::SeekNode { target, .. } => Some(target), Self::ReturnSub { target, .. } => Some(target), @@ -443,12 +437,21 @@ mod messages { } } - pub fn terminal(&self) -> bool { + fn terminal(&self) -> bool { use SubscribeMsg::*; matches!(self, ReturnSub { .. } | SeekNode { .. }) } } + impl SubscribeMsg { + pub fn sender(&self) -> Option<&PeerKeyLocation> { + match self { + Self::ReturnSub { sender, .. } => Some(sender), + _ => None, + } + } + } + impl Display for SubscribeMsg { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let id = self.id(); @@ -469,9 +472,8 @@ mod test { use freenet_stdlib::client_api::ContractRequest; use super::*; - use crate::node::tests::{check_connectivity, NodeSpecification, SimNetwork}; + use crate::node::tests::{NodeSpecification, SimNetwork}; - #[ignore] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn successful_subscribe_op_between_nodes() -> Result<(), anyhow::Error> { const NUM_NODES: usize = 4usize; @@ -489,13 +491,6 @@ mod test { } .into(); let first_node = NodeSpecification { - owned_contracts: Vec::new(), - non_owned_contracts: vec![contract_key], - events_to_generate: HashMap::from_iter([(1, event)]), - contract_subscribers: HashMap::new(), - }; - - let second_node = NodeSpecification { owned_contracts: vec![( ContractContainer::Wasm(ContractWasmAPIVersion::V1(contract)), contract_val, @@ -504,15 +499,35 @@ mod test { events_to_generate: HashMap::new(), contract_subscribers: HashMap::new(), }; + let second_node = NodeSpecification { + owned_contracts: Vec::new(), + non_owned_contracts: vec![contract_key.clone()], + events_to_generate: HashMap::from_iter([(1, event)]), + contract_subscribers: HashMap::new(), + }; let subscribe_specs = HashMap::from_iter([ - ("node-0".to_string(), first_node), - ("node-1".to_string(), second_node), + ("node-0".into(), first_node), + ("node-1".into(), second_node), ]); - let mut sim_nodes = SimNetwork::new(NUM_GW, NUM_NODES, 3, 2, 4, 2).await; - sim_nodes.build_with_specs(subscribe_specs).await; - check_connectivity(&sim_nodes, NUM_NODES, Duration::from_secs(3)).await?; - + let mut sim_nw = SimNetwork::new( + "successful_subscribe_op_between_nodes", + NUM_GW, + NUM_NODES, + 4, + 3, + 5, + 2, + ) + .await; + sim_nw.start_with_spec(subscribe_specs).await; + sim_nw.check_connectivity(Duration::from_secs(3)).await?; + sim_nw + .trigger_event("node-1", 1, Some(Duration::from_secs(2))) + .await?; + assert!(sim_nw.has_got_contract("node-1", &contract_key)); + tokio::time::sleep(Duration::from_secs(3)).await; + assert!(sim_nw.is_subscribed_to_contract("node-1", &contract_key)); Ok(()) } } diff --git a/crates/core/src/operations/update.rs b/crates/core/src/operations/update.rs index 42339191f..bfdae46a6 100644 --- a/crates/core/src/operations/update.rs +++ b/crates/core/src/operations/update.rs @@ -3,10 +3,22 @@ pub(crate) use self::messages::UpdateMsg; use crate::{client_events::ClientId, node::ConnectionBridge}; -use super::{op_trait::Operation, OpError}; +use super::{op_trait::Operation, OpError, OpOutcome}; pub(crate) struct UpdateOp {} +impl UpdateOp { + pub fn outcome(&self) -> OpOutcome { + OpOutcome::Irrelevant + } + + pub fn finalized(&self) -> bool { + todo!() + } + + pub fn record_transfer(&mut self) {} +} + pub(crate) struct UpdateResult {} impl TryFrom for UpdateResult { @@ -36,7 +48,7 @@ impl Operation for UpdateOp { self, _conn_manager: &'a mut CB, _op_storage: &'a crate::node::OpManager, - _input: Self::Message, + _input: &Self::Message, _client_id: Option, ) -> std::pin::Pin< Box> + Send + 'a>, @@ -46,16 +58,35 @@ impl Operation for UpdateOp { } mod messages { + use std::fmt::Display; + use serde::{Deserialize, Serialize}; - use crate::message::{InnerMessage, Transaction}; + use crate::{ + message::{InnerMessage, Transaction}, + ring::PeerKeyLocation, + }; - #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + #[derive(Debug, Serialize, Deserialize)] pub(crate) enum UpdateMsg {} impl InnerMessage for UpdateMsg { fn id(&self) -> &Transaction { todo!() } + + fn target(&self) -> Option<&PeerKeyLocation> { + todo!() + } + + fn terminal(&self) -> bool { + todo!() + } + } + + impl Display for UpdateMsg { + fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } } } diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index e33c2551c..8b85e3811 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -232,7 +232,6 @@ impl Ring { /// Returns this node location in the ring, if any (must have join the ring already). pub fn own_location(&self) -> PeerKeyLocation { - tracing::debug!("Getting loc for peer {}", self.peer_key); let location = f64::from_le_bytes(self.own_location.load(SeqCst).to_le_bytes()); let location = if (location - -1f64).abs() < f64::EPSILON { None @@ -251,23 +250,28 @@ impl Ring { /// # Panic /// Will panic if the node checking for this condition has no location assigned. pub fn should_accept(&self, location: &Location) -> bool { + let cbl = &*self.connections_by_location.read(); let open_conn = self.open_connections.fetch_add(1, SeqCst) + 1; let my_location = &self .own_location() .location .expect("this node has no location assigned!"); - let cbl = &*self.connections_by_location.read(); let accepted = if location == my_location || cbl.contains_key(location) { false } else if open_conn < self.min_connections { true } else if open_conn >= self.max_connections { + tracing::debug!(peer = %self.peer_key, "max connections reached"); false } else { - my_location.distance(location) - < self - .median_distance_to(my_location) - .unwrap_or(Distance(0.5)) + // todo: in the future maybe use the `small worldness` metric to decide + let median_distance = self + .median_distance_to(my_location) + .unwrap_or(Distance(0.5)); + let dist_to_loc = my_location.distance(location); + let is_lower_than_median = dist_to_loc < median_distance; + tracing::debug!("dist to connection loc: {dist_to_loc}, median dist: {median_distance}, accepting: {is_lower_than_median}"); + is_lower_than_median }; if !accepted { self.open_connections.fetch_sub(1, SeqCst); @@ -343,13 +347,32 @@ impl Ring { /// Get a random peer from the known ring connections. pub fn random_peer(&self, filter_fn: F) -> Option where - F: FnMut(&&PeerKeyLocation) -> bool, + F: Fn(&PeerKey) -> bool, { - self.connections_by_location - .read() - .values() - .find(filter_fn) - .copied() + use rand::Rng; + let peers = &*self.location_for_peer.read(); + let amount = peers.len(); + if amount == 0 { + return None; + } + let mut rng = rand::thread_rng(); + let mut attempts = 0; + loop { + if attempts >= amount { + return None; + } + let selected = rng.gen_range(0..amount); + let (peer, loc) = peers.iter().nth(selected).expect("infallible"); + if !filter_fn(peer) { + attempts += 1; + continue; + } else { + return Some(PeerKeyLocation { + peer: *peer, + location: Some(*loc), + }); + } + } } /// Will return an error in case the max number of subscribers has been added. @@ -457,8 +480,7 @@ impl From<&ContractKey> for Location { impl Display for Location { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.0.to_string().as_str())?; - Ok(()) + write!(f, "{}", self.0) } } @@ -474,7 +496,8 @@ impl Eq for Location {} impl Ord for Location { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.partial_cmp(other) + self.0 + .partial_cmp(&other.0) .expect("always should return a cmp value") } } @@ -545,6 +568,12 @@ impl Ord for Distance { } } +impl Display for Distance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + #[derive(thiserror::Error, Debug)] pub(crate) enum RingError { #[error(transparent)] @@ -558,10 +587,7 @@ pub(crate) enum RingError { #[cfg(test)] mod test { use super::*; - use crate::client_events::test::MemoryEventsGen; - use tokio::sync::watch::channel; - #[ignore] #[test] fn location_dist() { let l0 = Location(0.); @@ -572,58 +598,4 @@ mod test { let l1 = Location(0.50); assert!(l0.distance(l1) == Distance(0.25)); } - - #[ignore] - #[test] - fn find_closest() { - let peer_key: PeerKey = PeerKey::random(); - - let (_, receiver) = channel((0, peer_key)); - let user_events = MemoryEventsGen::new(receiver, peer_key); - let config = NodeBuilder::new([Box::new(user_events)]); - let ring = Ring::new::<1, node::TestEventListener>(&config, &[]).unwrap(); - - fn build_pk(loc: Location) -> PeerKeyLocation { - PeerKeyLocation { - peer: PeerKey::random(), - location: Some(loc), - } - } - - { - let conns = &mut *ring.connections_by_location.write(); - conns.insert(Location(0.3), build_pk(Location(0.3))); - conns.insert(Location(0.5), build_pk(Location(0.5))); - conns.insert(Location(0.0), build_pk(Location(0.0))); - } - - assert_eq!( - Location(0.0), - ring.routing(&Location(0.9), None, &[]) - .unwrap() - .location - .unwrap() - ); - assert_eq!( - Location(0.0), - ring.routing(&Location(0.1), None, &[]) - .unwrap() - .location - .unwrap() - ); - assert_eq!( - Location(0.5), - ring.routing(&Location(0.41), None, &[]) - .unwrap() - .location - .unwrap() - ); - assert_eq!( - Location(0.3), - ring.routing(&Location(0.39), None, &[]) - .unwrap() - .location - .unwrap() - ); - } } diff --git a/crates/core/src/runtime/mod.rs b/crates/core/src/runtime.rs similarity index 91% rename from crates/core/src/runtime/mod.rs rename to crates/core/src/runtime.rs index 72a6a7a6e..590417d5d 100644 --- a/crates/core/src/runtime/mod.rs +++ b/crates/core/src/runtime.rs @@ -21,4 +21,4 @@ pub(crate) use error::RuntimeResult; pub use secrets_store::SecretsStore; pub use state_store::StateStore; pub(crate) use state_store::{StateStorage, StateStoreError}; -pub(crate) use wasm_runtime::{ContractExecError, Runtime}; +pub use wasm_runtime::{ContractExecError, Runtime}; diff --git a/crates/core/src/runtime/contract_store.rs b/crates/core/src/runtime/contract_store.rs index bc9883a35..4204ef549 100644 --- a/crates/core/src/runtime/contract_store.rs +++ b/crates/core/src/runtime/contract_store.rs @@ -51,7 +51,7 @@ pub struct ContractStore { key_to_code_part: Arc>, } // TODO: add functionality to delete old contracts which have not been used for a while -// to keep the total speed used under a configured threshold +// to keep the total space used under a configured threshold static LOCK_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); static KEY_FILE_PATH: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); diff --git a/crates/core/src/runtime/delegate.rs b/crates/core/src/runtime/delegate.rs index 368823ccd..6c22407a2 100644 --- a/crates/core/src/runtime/delegate.rs +++ b/crates/core/src/runtime/delegate.rs @@ -417,7 +417,7 @@ mod test { name: &str, ) -> Result<(DelegateContainer, Runtime), Box> { const TEST_PREFIX: &str = "delegate-api"; - let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + // let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); let contracts_dir = super::super::tests::test_dir(TEST_PREFIX); let delegates_dir = super::super::tests::test_dir(TEST_PREFIX); let secrets_dir = super::super::tests::test_dir(TEST_PREFIX); diff --git a/crates/core/src/runtime/error.rs b/crates/core/src/runtime/error.rs index d45cd86e2..7e855304b 100644 --- a/crates/core/src/runtime/error.rs +++ b/crates/core/src/runtime/error.rs @@ -2,11 +2,13 @@ use std::fmt::Display; use freenet_stdlib::prelude::{ContractKey, DelegateKey, SecretsId}; +use crate::DynError; + use super::{delegate, secrets_store, wasm_runtime, DelegateExecError}; pub type RuntimeResult = std::result::Result; -#[derive(Debug)] +#[derive(thiserror::Error, Debug)] pub struct ContractError(Box); impl ContractError { @@ -63,8 +65,6 @@ impl Display for ContractError { } } -impl std::error::Error for ContractError {} - impl From for ContractError { fn from(err: RuntimeInnerError) -> Self { Self(Box::new(err)) @@ -97,12 +97,12 @@ impl_err!(wasmer::RuntimeError); #[derive(thiserror::Error, Debug)] pub(crate) enum RuntimeInnerError { #[error(transparent)] - Any(#[from] Box), + Any(#[from] DynError), #[error(transparent)] BufferError(#[from] freenet_stdlib::memory::buf::Error), - #[error(transparent)] + #[error("{0}")] IOError(#[from] std::io::Error), #[error(transparent)] diff --git a/crates/core/src/runtime/native_api.rs b/crates/core/src/runtime/native_api.rs index 475b6bd63..c061fd78d 100644 --- a/crates/core/src/runtime/native_api.rs +++ b/crates/core/src/runtime/native_api.rs @@ -19,56 +19,81 @@ fn compute_ptr(ptr: i64, start_ptr: i64) -> *mut T { (start_ptr + ptr) as _ } -pub(crate) mod time { +pub(crate) mod log { + use wasmer::Function; + use super::*; - use chrono::{DateTime, Utc as UtcOriginal}; pub(crate) fn prepare_export(store: &mut wasmer::Store, imports: &mut Imports) { - let utc_now = Function::new_typed(store, utc_now); + let utc_now = Function::new_typed(store, info); imports.register_namespace( - "freenet_time", - [("__frnt__time__utc_now".to_owned(), utc_now.into())], + "freenet_log", + [("__frnt__logger__info".to_owned(), utc_now.into())], ); } - fn utc_now(id: i64, ptr: i64) { + // TODO: this API right now is just a patch, ideally we want to impl a tracing subscriber + // that can be used in wasm and that under the hood will just pass data to the host via + // functions like this in a structured way + fn info(id: i64, ptr: i64, len: i32) { if id == -1 { panic!("unset module id"); } let info = MEM_ADDR.get(&id).expect("instance mem space not recorded"); - let now = UtcOriginal::now(); - let ptr = compute_ptr::>(ptr, info.start_ptr); - // eprintln!("{ptr:p} ({}) outside", ptr as i64); - unsafe { - ptr.write(now); - }; + let ptr = compute_ptr::(ptr, info.start_ptr); + let msg = + unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(ptr, len as _)) }; + tracing::info!(target: "contract", contract = %info.value().key(), "{msg}"); } } -pub(crate) mod log { - use wasmer::Function; +pub(crate) mod rand { + use ::rand::{thread_rng, RngCore}; use super::*; pub(crate) fn prepare_export(store: &mut wasmer::Store, imports: &mut Imports) { - let utc_now = Function::new_typed(store, info); + let rand_bytes = Function::new_typed(store, rand_bytes); imports.register_namespace( - "freenet_logger", - [("__frnt__logger__info".to_owned(), utc_now.into())], + "freenet_rand", + [("__frnt__rand__rand_bytes".to_owned(), rand_bytes.into())], ); } - // TODO: this API right now is just a patch, ideally we want to impl a tracing subscriber - // that can be used in wasm and that under the hood will just pass data to the host via - // functions like this in a structured way - fn info(id: i64, ptr: i64, len: i32) { + fn rand_bytes(id: i64, ptr: i64, len: u32) { if id == -1 { panic!("unset module id"); } let info = MEM_ADDR.get(&id).expect("instance mem space not recorded"); let ptr = compute_ptr::(ptr, info.start_ptr); - let msg = - unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(ptr, len as _)) }; - tracing::info!(target: "contract", contract = %info.value().key(), "{msg}"); + let slice = unsafe { &mut *std::ptr::slice_from_raw_parts_mut(ptr, len as usize) }; + let mut rng = thread_rng(); + rng.fill_bytes(slice); + } +} + +pub(crate) mod time { + use super::*; + use chrono::{DateTime, Utc as UtcOriginal}; + + pub(crate) fn prepare_export(store: &mut wasmer::Store, imports: &mut Imports) { + let utc_now = Function::new_typed(store, utc_now); + imports.register_namespace( + "freenet_time", + [("__frnt__time__utc_now".to_owned(), utc_now.into())], + ); + } + + fn utc_now(id: i64, ptr: i64) { + if id == -1 { + panic!("unset module id"); + } + let info = MEM_ADDR.get(&id).expect("instance mem space not recorded"); + let now = UtcOriginal::now(); + let ptr = compute_ptr::>(ptr, info.start_ptr); + // eprintln!("{ptr:p} ({}) outside", ptr as i64); + unsafe { + ptr.write(now); + }; } } diff --git a/crates/core/src/runtime/secrets_store.rs b/crates/core/src/runtime/secrets_store.rs index 486312892..092a7057d 100644 --- a/crates/core/src/runtime/secrets_store.rs +++ b/crates/core/src/runtime/secrets_store.rs @@ -54,7 +54,7 @@ impl StoreEntriesContainer for KeyToEncryptionMap { pub enum SecretStoreError { #[error("encryption error: {0}")] Encryption(EncryptionError), - #[error(transparent)] + #[error("{0}")] IO(#[from] std::io::Error), #[error("missing cipher")] MissingCipher, diff --git a/crates/core/src/runtime/state_store.rs b/crates/core/src/runtime/state_store.rs index ea507f249..334deab2f 100644 --- a/crates/core/src/runtime/state_store.rs +++ b/crates/core/src/runtime/state_store.rs @@ -11,6 +11,17 @@ pub enum StateStoreError { MissingContract(ContractKey), } +impl From for crate::runtime::ContractError { + fn from(value: StateStoreError) -> Self { + match value { + StateStoreError::Any(err) => crate::runtime::ContractError::from(err), + err @ StateStoreError::MissingContract(_) => { + crate::runtime::ContractError::from(Into::::into(format!("{err}"))) + } + } + } +} + #[async_trait::async_trait] #[allow(clippy::type_complexity)] pub trait StateStorage { diff --git a/crates/core/src/runtime/tests/mod.rs b/crates/core/src/runtime/tests/mod.rs index 0cb8bfacf..32d0bf913 100644 --- a/crates/core/src/runtime/tests/mod.rs +++ b/crates/core/src/runtime/tests/mod.rs @@ -62,7 +62,7 @@ pub(crate) fn get_test_module(name: &str) -> Result, Box Result<(ContractStore, ContractKey), Box> { - let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + // let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); let mut store = ContractStore::new(test_dir("contract"), 10_000)?; let contract_bytes = WrappedContract::new( Arc::new(ContractCode::from(get_test_module(name)?)), diff --git a/crates/core/src/runtime/wasm_runtime.rs b/crates/core/src/runtime/wasm_runtime.rs index 7c5d32e83..4c9d2db29 100644 --- a/crates/core/src/runtime/wasm_runtime.rs +++ b/crates/core/src/runtime/wasm_runtime.rs @@ -127,8 +127,9 @@ impl Runtime { } else { (None, imports! {}) }; - native_api::time::prepare_export(&mut store, &mut top_level_imports); native_api::log::prepare_export(&mut store, &mut top_level_imports); + native_api::rand::prepare_export(&mut store, &mut top_level_imports); + native_api::time::prepare_export(&mut store, &mut top_level_imports); Ok(Self { wasm_store: store, @@ -209,7 +210,6 @@ impl Runtime { let module = if let Some(module) = self.delegate_modules.get(key) { module } else { - // FIXME let delegate = self .delegate_store .fetch_delegate(key, params) diff --git a/crates/core/src/server/mod.rs b/crates/core/src/server.rs similarity index 98% rename from crates/core/src/server/mod.rs rename to crates/core/src/server.rs index 730c2a4de..e9a56b4cf 100644 --- a/crates/core/src/server/mod.rs +++ b/crates/core/src/server.rs @@ -130,7 +130,7 @@ pub mod local_node { if let Some(cause) = cause { tracing::info!("disconnecting cause: {cause}"); } - // todo: token must live for a bit to allow reconnections + // fixme: token must live for a bit to allow reconnections if let Some(rm_token) = gw .attested_contracts .iter() diff --git a/crates/core/src/server/app_packaging.rs b/crates/core/src/server/app_packaging.rs index ce241ca4b..5fa5744a0 100644 --- a/crates/core/src/server/app_packaging.rs +++ b/crates/core/src/server/app_packaging.rs @@ -12,7 +12,7 @@ use xz2::read::{XzDecoder, XzEncoder}; pub enum WebContractError { #[error("unpacking error: {0}")] UnpackingError(Box), - #[error(transparent)] + #[error("{0}")] StoringError(std::io::Error), #[error("file not found: {0}")] FileNotFound(String), diff --git a/crates/core/src/server/errors.rs b/crates/core/src/server/errors.rs index c63cfc2f4..f62c79467 100644 --- a/crates/core/src/server/errors.rs +++ b/crates/core/src/server/errors.rs @@ -1,6 +1,7 @@ use axum::http::StatusCode; use axum::response::{Html, IntoResponse, Response}; use freenet_stdlib::client_api::ErrorKind; +use freenet_stdlib::prelude::ContractKey; use std::fmt::{Display, Formatter}; #[derive(Debug)] @@ -15,14 +16,18 @@ pub(super) enum WebSocketApiError { AxumError { error: ErrorKind, }, + MissingContract { + key: ContractKey, + }, } impl WebSocketApiError { pub fn status_code(&self) -> StatusCode { match self { WebSocketApiError::InvalidParam { .. } => StatusCode::BAD_REQUEST, - WebSocketApiError::NodeError { .. } => StatusCode::BAD_GATEWAY, + WebSocketApiError::NodeError { .. } => StatusCode::INTERNAL_SERVER_ERROR, WebSocketApiError::AxumError { .. } => StatusCode::INTERNAL_SERVER_ERROR, + WebSocketApiError::MissingContract { .. } => StatusCode::NOT_FOUND, } } @@ -32,7 +37,8 @@ impl WebSocketApiError { format!("Invalid request params: {}", error_cause) } WebSocketApiError::NodeError { error_cause } => format!("Node error: {}", error_cause), - WebSocketApiError::AxumError { error } => format!("Axum error: {}", error), + WebSocketApiError::AxumError { error } => format!("Server error: {}", error), + WebSocketApiError::MissingContract { key } => format!("Missing contract {key}"), } } } @@ -61,7 +67,12 @@ impl IntoResponse for WebSocketApiError { { (StatusCode::NOT_FOUND, error_cause) } - WebSocketApiError::NodeError { error_cause } => (StatusCode::BAD_GATEWAY, error_cause), + WebSocketApiError::NodeError { error_cause } => { + (StatusCode::INTERNAL_SERVER_ERROR, error_cause) + } + err @ WebSocketApiError::MissingContract { .. } => { + (StatusCode::NOT_FOUND, err.error_message()) + } WebSocketApiError::AxumError { error } => { (StatusCode::INTERNAL_SERVER_ERROR, format!("{error}")) } diff --git a/crates/core/src/server/path_handlers.rs b/crates/core/src/server/path_handlers.rs index 02b341d7c..a7184b392 100644 --- a/crates/core/src/server/path_handlers.rs +++ b/crates/core/src/server/path_handlers.rs @@ -45,7 +45,9 @@ pub(super) async fn contract_home( let client_id = if let Some(HostCallbackResult::NewId { id }) = response_recv.recv().await { id } else { - todo!("this is an error"); + return Err(WebSocketApiError::NodeError { + error_cause: "Couldn't register new client in the node".into(), + }); }; request_sender .send(ClientConnection::Request { @@ -120,7 +122,7 @@ pub(super) async fn contract_home( web_body } None => { - todo!("error indicating the contract is not present"); + return Err(WebSocketApiError::MissingContract { key }); } }, Some(HostCallbackResult::Result { diff --git a/crates/fdev/src/build.rs b/crates/fdev/src/build.rs index a8c491d43..58b797a07 100644 --- a/crates/fdev/src/build.rs +++ b/crates/fdev/src/build.rs @@ -51,6 +51,7 @@ fn compile_options(cli_config: &BuildToolCliConfig) -> impl Iterator, pub typescript: Option, #[serde(rename = "state-sources")] - pub state_sources: Option, + pub state_sources: Sources, pub metadata: Option, pub dependencies: Option, } @@ -227,7 +228,6 @@ mod contract { #[derive(Serialize, Deserialize, PartialEq)] #[serde(rename_all = "lowercase")] pub(crate) enum SupportedWebLangs { - Javascript, Typescript, } @@ -242,6 +242,11 @@ mod contract { embedded_deps: EmbeddedDeps, cwd: &Path, ) -> Result<(), DynError> { + let Some(web_config) = &config.webapp else { + println!("No webapp config found."); + return Ok(()); + }; + let metadata = if let Some(md) = config.webapp.as_ref().and_then(|a| a.metadata.as_ref()) { let mut buf = vec![]; File::open(md)?.read_to_end(&mut buf)?; @@ -251,73 +256,68 @@ mod contract { }; let mut archive: Builder>> = Builder::new(Cursor::new(Vec::new())); - if let Some(web_config) = &config.webapp { - println!("Bundling webapp contract state"); - match &web_config.lang { - Some(SupportedWebLangs::Typescript) => { - let child = Command::new("npm") - .args(["install"]) + + println!("Bundling webapp contract state"); + match &web_config.lang { + Some(SupportedWebLangs::Typescript) => { + let child = Command::new("npm") + .args(["install"]) + .current_dir(cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| { + eprintln!("Error while installing npm packages: {e}"); + Error::CommandFailed("npm") + })?; + pipe_std_streams(child)?; + let webpack = web_config + .typescript + .as_ref() + .map(|c| c.webpack) + .unwrap_or_default(); + use std::io::IsTerminal; + if webpack { + let cmd_args: &[&str] = + if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { + &["--color"] + } else { + &[] + }; + let child = Command::new("webpack") + .args(cmd_args) .current_dir(cwd) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .map_err(|e| { - eprintln!("Error while installing npm packages: {e}"); - Error::CommandFailed("npm") + eprintln!("Error while executing webpack command: {e}"); + Error::CommandFailed("tsc") })?; pipe_std_streams(child)?; - let webpack = web_config - .typescript - .as_ref() - .map(|c| c.webpack) - .unwrap_or_default(); - use std::io::IsTerminal; - if webpack { - let cmd_args: &[&str] = - if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { - &["--color"] - } else { - &[] - }; - let child = Command::new("webpack") - .args(cmd_args) - .current_dir(cwd) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| { - eprintln!("Error while executing webpack command: {e}"); - Error::CommandFailed("tsc") - })?; - pipe_std_streams(child)?; - println!("Compiled input using webpack"); - } else { - let cmd_args: &[&str] = - if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { - &["--pretty"] - } else { - &[] - }; - let child = Command::new("tsc") - .args(cmd_args) - .current_dir(cwd) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| { - eprintln!("Error while executing command tsc: {e}"); - Error::CommandFailed("tsc") - })?; - pipe_std_streams(child)?; - println!("Compiled input using tsc"); - } + println!("Compiled input using webpack"); + } else { + let cmd_args: &[&str] = + if std::io::stdout().is_terminal() && std::io::stderr().is_terminal() { + &["--pretty"] + } else { + &[] + }; + let child = Command::new("tsc") + .args(cmd_args) + .current_dir(cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| { + eprintln!("Error while executing command tsc: {e}"); + Error::CommandFailed("tsc") + })?; + pipe_std_streams(child)?; + println!("Compiled input using tsc"); } - Some(SupportedWebLangs::Javascript) => todo!(), - None => {} } - } else { - println!("No webapp config found."); - return Ok(()); + None => {} } let build_state = |sources: &Sources| -> Result<(), DynError> { @@ -392,15 +392,8 @@ mod contract { Ok(()) }; - if let Some(sources) = config - .webapp - .as_ref() - .and_then(|a| a.state_sources.as_ref()) - { - build_state(sources) - } else { - todo!() - } + let sources = &web_config.state_sources; + build_state(sources) } fn build_generic_state(config: &mut BuildToolConfig, cwd: &Path) -> Result<(), DynError> { @@ -602,10 +595,10 @@ mod contract { webapp: Some(WebAppContract { lang: Some(SupportedWebLangs::Typescript), typescript: Some(TypescriptConfig { webpack: true }), - state_sources: Some(Sources { + state_sources: Sources { source_dirs: Some(vec!["dist".into()]), files: None, - }), + }, metadata: None, dependencies: Some( toml::toml! { diff --git a/crates/fdev/src/commands.rs b/crates/fdev/src/commands.rs index a61a0c508..b323044b2 100644 --- a/crates/fdev/src/commands.rs +++ b/crates/fdev/src/commands.rs @@ -167,16 +167,10 @@ async fn execute_command( let contract_store = ContractStore::new(contracts_data_path, DEFAULT_MAX_CONTRACT_SIZE)?; let delegate_store = DelegateStore::new(delegates_data_path, DEFAULT_MAX_DELEGATE_SIZE)?; let secret_store = SecretsStore::new(secrets_data_path)?; - let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE).unwrap(); - let mut executor = Executor::new( - contract_store, - delegate_store, - secret_store, - state_store, - || {}, - OperationMode::Local, - ) - .await?; + let state_store = StateStore::new(Storage::new().await?, MAX_MEM_CACHE)?; + let rt = + freenet::dev_tool::Runtime::build(contract_store, delegate_store, secret_store, false)?; + let mut executor = Executor::new(state_store, || {}, OperationMode::Local, rt).await?; executor .handle_request(ClientId::FIRST, request, None) diff --git a/crates/fdev/src/inspect.rs b/crates/fdev/src/inspect.rs index 7da43cab1..623fe63b1 100644 --- a/crates/fdev/src/inspect.rs +++ b/crates/fdev/src/inspect.rs @@ -16,7 +16,6 @@ pub struct InspectCliConfig { enum FileType { Code(CodeInspection), Delegate, - Contract, } /// Inspect the packaged WASM code for Freenet. @@ -47,7 +46,6 @@ delegate API version: {version} "# ); } - FileType::Contract => todo!(), } Ok(()) diff --git a/crates/fdev/src/local_node/commands.rs b/crates/fdev/src/local_node/commands.rs index 00962506e..e479a98fe 100644 --- a/crates/fdev/src/local_node/commands.rs +++ b/crates/fdev/src/local_node/commands.rs @@ -75,7 +75,6 @@ async fn execute_command( })) => { println!("current state for {key}:"); app.printout_deser(state.as_ref())?; - todo!() } Err(err) => { println!("error: {err}"); @@ -85,15 +84,6 @@ async fn execute_command( } _ => unreachable!(), }, - ClientRequest::DelegateOp(op) => { - match node.handle_request(ClientId::FIRST, op.into(), None).await { - Ok(_res) => todo!(), - Err(err) => { - println!("error: {err}"); - } - } - } - ClientRequest::Disconnect { .. } => return Ok(true), _ => { tracing::error!("op not supported"); return Err("op not support".into()); diff --git a/crates/fdev/src/local_node/state.rs b/crates/fdev/src/local_node/state.rs index 0ded9ce22..9a25ff836 100644 --- a/crates/fdev/src/local_node/state.rs +++ b/crates/fdev/src/local_node/state.rs @@ -24,17 +24,22 @@ impl AppState { let contract_dir = Config::conf().contracts_dir(); let contract_store = ContractStore::new(contract_dir, config.max_contract_size)?; let state_store = StateStore::new(Storage::new().await?, Self::MAX_MEM_CACHE).unwrap(); + let rt = freenet::dev_tool::Runtime::build( + contract_store, + DelegateStore::default(), + SecretsStore::default(), + false, + ) + .unwrap(); Ok(AppState { local_node: Arc::new(RwLock::new( Executor::new( - contract_store, - DelegateStore::default(), - SecretsStore::default(), state_store, || { freenet::util::set_cleanup_on_exit().unwrap(); }, OperationMode::Local, + rt, ) .await?, )), diff --git a/crates/fdev/src/main.rs b/crates/fdev/src/main.rs index 9f290c0f7..7c9eeb6b6 100644 --- a/crates/fdev/src/main.rs +++ b/crates/fdev/src/main.rs @@ -2,7 +2,6 @@ use std::borrow::Cow; use clap::Parser; use freenet_stdlib::client_api::ClientRequest; -use tracing_subscriber::EnvFilter; mod build; mod commands; @@ -35,9 +34,7 @@ enum Error { #[tokio::main] async fn main() -> Result<(), anyhow::Error> { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .init(); + freenet::config::set_logger(); let cwd = std::env::current_dir()?; let config = Config::parse(); freenet::config::Config::set_op_mode(config.additional.mode); diff --git a/crates/fdev/src/new_package.rs b/crates/fdev/src/new_package.rs index e708dc7fc..44485f0d2 100644 --- a/crates/fdev/src/new_package.rs +++ b/crates/fdev/src/new_package.rs @@ -34,10 +34,10 @@ fn create_view_package(cwd: &Path) -> Result<(), DynError> { webapp: Some(WebAppContract { lang: Some(SupportedWebLangs::Typescript), typescript: Some(TypescriptConfig { webpack: true }), - state_sources: Some(Sources { + state_sources: Sources { source_dirs: Some(vec![PathBuf::from("dist")]), files: None, - }), + }, metadata: None, dependencies: None, }), @@ -152,7 +152,7 @@ fn create_web_init_files(cwd: &Path) -> Result<(), DynError> { Error::CommandFailed("npm") })?; pipe_std_streams(child)?; - // todo: change pacakge.json: + // todo: change package.json: // - include dependencies: freenet-stdlib let child = Command::new(TSC) diff --git a/stdlib b/stdlib index 0cd6ba8c9..f36475537 160000 --- a/stdlib +++ b/stdlib @@ -1 +1 @@ -Subproject commit 0cd6ba8c9a51ae03d24deef8ce35a88241de57f4 +Subproject commit f36475537a1fa2e1f19721bbfac452ce508f02b9