From 6392dafbb77aff6af9a7fc7662cf44312070a034 Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Wed, 3 Jan 2024 04:15:12 -0600 Subject: [PATCH] 184207088 integrate resource with topology (#916) Co-authored-by: Ignacio Duart --- Cargo.lock | 597 ++++++------ Cargo.toml | 1 - crates/core/Cargo.toml | 6 +- crates/core/src/contract/handler.rs | 2 +- crates/core/src/contract/storages/mod.rs | 2 +- crates/core/src/lib.rs | 5 +- crates/core/src/message.rs | 4 +- crates/core/src/node.rs | 8 +- crates/core/src/node/network_bridge.rs | 4 +- crates/core/src/node/op_state_manager.rs | 4 +- crates/core/src/resources.rs | 409 --------- crates/core/src/resources/meter.rs | 288 ------ crates/core/src/resources/rate.rs | 29 - crates/core/src/ring.rs | 247 +++-- crates/core/src/test_utils.rs | 9 + crates/core/src/topology.rs | 853 ++++++++++++++++-- crates/core/src/topology/constants.rs | 9 + crates/core/src/topology/meter.rs | 354 ++++++++ .../src/topology/outbound_request_counter.rs | 42 + crates/core/src/topology/rate.rs | 143 +++ .../src/topology/request_density_tracker.rs | 96 +- .../running_average.rs | 22 +- crates/core/src/topology/small_world_rand.rs | 26 +- crates/core/src/wasm_runtime/delegate.rs | 4 +- 24 files changed, 1848 insertions(+), 1316 deletions(-) delete mode 100644 crates/core/src/resources.rs delete mode 100644 crates/core/src/resources/meter.rs delete mode 100644 crates/core/src/resources/rate.rs create mode 100644 crates/core/src/test_utils.rs create mode 100644 crates/core/src/topology/constants.rs create mode 100644 crates/core/src/topology/meter.rs create mode 100644 crates/core/src/topology/outbound_request_counter.rs create mode 100644 crates/core/src/topology/rate.rs rename crates/core/src/{resources => topology}/running_average.rs (87%) diff --git a/Cargo.lock b/Cargo.lock index aa6df05e3..cbdc46554 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" dependencies = [ "anstyle", "anstyle-parse", @@ -128,37 +128,37 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" [[package]] name = "approx" @@ -262,18 +262,18 @@ dependencies = [ [[package]] name = "async-io" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" +checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" dependencies = [ - "async-lock 3.1.2", + "async-lock 3.2.0", "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.0.1", + "futures-lite 2.1.0", "parking", "polling 3.3.1", - "rustix 0.38.25", + "rustix 0.38.28", "slab", "tracing", "windows-sys 0.52.0", @@ -290,24 +290,24 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.1.2" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea8b3453dd7cc96711834b75400d671b73e3656975fa68d9f277163b7f7e316" +checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.1", "event-listener-strategy", "pin-project-lite", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -349,9 +349,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "atomic-write-file" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae364a6c1301604bbc6dfbf8c385c47ff82301dd01eef506195a029196d8d04" +checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" dependencies = [ "nix 0.27.1", "rand", @@ -376,9 +376,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810a80b128d70e6ed2bdf3fe8ed72c0ae56f5f5948d01c2753282dd92a84fce8" +checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d" dependencies = [ "async-trait", "axum-core", @@ -388,7 +388,7 @@ dependencies = [ "http 1.0.0", "http-body 1.0.0", "http-body-util", - "hyper 1.0.1", + "hyper 1.1.0", "hyper-util", "itoa", "matchit", @@ -402,7 +402,7 @@ dependencies = [ "sha1", "sync_wrapper", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.20.1", "tower", "tower-layer", "tower-service", @@ -410,9 +410,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0ddc355eab88f4955090a823715df47acf0b7660aab7a69ad5ce6301ee3b73" +checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa" dependencies = [ "async-trait", "bytes", @@ -494,7 +494,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -723,9 +723,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.10" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" dependencies = [ "clap_builder", "clap_derive", @@ -733,9 +733,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstream", "anstyle", @@ -752,7 +752,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -769,9 +769,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -797,9 +797,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "constant_time_eq" @@ -819,9 +819,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -829,9 +829,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "core2" @@ -964,9 +964,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +checksum = "6eb9105919ca8e40d437fc9cbb8f1975d916f1bd28afe795a48aae32a2cc8920" dependencies = [ "cfg-if", "crossbeam-channel", @@ -978,9 +978,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -988,9 +988,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -999,22 +999,20 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1022,9 +1020,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if", ] @@ -1051,12 +1049,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e95fbd621905b854affdc67943b043a0fbb6ed7385fd5a25650d19a8a6cfdf" +checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ "nix 0.27.1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1076,9 +1074,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.68+curl-8.4.0" +version = "0.4.70+curl-8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" +checksum = "3c0333d8849afe78a4c8102a429a446bfdd055832af071945520e835ae2d841e" dependencies = [ "cc", "libc", @@ -1114,7 +1112,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1138,7 +1136,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1149,7 +1147,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1229,9 +1227,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" dependencies = [ "powerfmt", "serde", @@ -1256,7 +1254,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1300,7 +1298,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1376,7 +1374,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1417,7 +1415,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1455,9 +1453,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +checksum = "84f2cdcf274580f2d63697192d744727b3198894b1bf02923643bf59e2c26712" dependencies = [ "concurrent-queue", "parking", @@ -1470,7 +1468,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.1", "pin-project-lite", ] @@ -1535,14 +1533,14 @@ checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", + "redox_syscall", + "windows-sys 0.52.0", ] [[package]] @@ -1618,15 +1616,14 @@ dependencies = [ "freenet-stdlib", "futures", "headers", - "itertools 0.11.0", - "itertools 0.12.0", + "itertools", "libp2p", "libp2p-identity", "notify", "once_cell", "opentelemetry", "opentelemetry-jaeger", - "ordered-float 4.1.1", + "ordered-float 4.2.0", "parking_lot", "pav_regression", "pico-args", @@ -1641,8 +1638,9 @@ dependencies = [ "tar", "tempfile", "thiserror", + "time", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.21.0", "tower-http", "tracing", "tracing-opentelemetry", @@ -1659,7 +1657,7 @@ version = "0.0.5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1680,13 +1678,13 @@ dependencies = [ "rand", "semver", "serde", - "serde-wasm-bindgen 0.6.1", + "serde-wasm-bindgen 0.6.3", "serde_bytes", "serde_json", "serde_with", "thiserror", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.20.1", "tracing", "tracing-subscriber", "wasm-bindgen", @@ -1711,9 +1709,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1736,9 +1734,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1746,15 +1744,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1775,9 +1773,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -1796,9 +1794,9 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" dependencies = [ "futures-core", "pin-project-lite", @@ -1806,13 +1804,13 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -1827,15 +1825,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -1845,9 +1843,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2039,9 +2037,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac", ] @@ -2057,11 +2055,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2099,9 +2097,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http 0.2.11", @@ -2151,9 +2149,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2161,12 +2159,12 @@ dependencies = [ "futures-util", "h2 0.3.22", "http 0.2.11", - "http-body 0.4.5", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -2175,9 +2173,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f9214f3e703236b221f1a9cd88ec8b4adfa5296de01ab96216361f4692f56" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" dependencies = [ "bytes", "futures-channel", @@ -2194,21 +2192,19 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca339002caeb0d159cc6e023dff48e199f081e42fa039895c7c6f38b37f2e9d" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.0.0", "http-body 1.0.0", - "hyper 1.0.1", + "hyper 1.1.0", "pin-project-lite", "socket2 0.5.5", "tokio", - "tower", - "tower-service", "tracing", ] @@ -2288,7 +2284,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6b0422c86d7ce0e97169cc42e04ae643caf278874a7a3c87b8150a220dc7e1e" dependencies = [ - "async-io 2.2.1", + "async-io 2.2.2", "core-foundation", "fnv", "futures", @@ -2312,7 +2308,7 @@ dependencies = [ "bytes", "futures", "http 0.2.11", - "hyper 0.14.27", + "hyper 0.14.28", "log", "rand", "tokio", @@ -2440,15 +2436,6 @@ dependencies = [ "waker-fn", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.0" @@ -2460,9 +2447,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" @@ -2536,9 +2523,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libloading" @@ -2873,7 +2860,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -2949,7 +2936,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.1", "libc", - "redox_syscall 0.4.1", + "redox_syscall", ] [[package]] @@ -3003,9 +2990,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" @@ -3110,9 +3097,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -3132,15 +3119,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.9.0" @@ -3183,9 +3161,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "log", @@ -3506,9 +3484,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -3524,9 +3502,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -3542,9 +3520,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -3622,7 +3600,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.1.1", + "ordered-float 4.2.0", "percent-encoding", "rand", "thiserror", @@ -3656,9 +3634,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.1.1" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", ] @@ -3703,7 +3681,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", "windows-targets 0.48.5", ] @@ -3791,7 +3769,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -3828,7 +3806,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -3866,15 +3844,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "platforms" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" +checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" [[package]] name = "polling" @@ -3901,7 +3879,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "pin-project-lite", - "rustix 0.38.25", + "rustix 0.38.28", "tracing", "windows-sys 0.52.0", ] @@ -3948,7 +3926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -3983,14 +3961,14 @@ checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" dependencies = [ "unicode-ident", ] @@ -4015,7 +3993,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -4207,15 +4185,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -4340,9 +4309,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.5" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom", @@ -4354,12 +4323,13 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.42" +version = "0.7.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5" dependencies = [ "bitvec", "bytecheck", + "bytes", "hashbrown 0.12.3", "indexmap 1.9.3", "ptr_meta", @@ -4372,9 +4342,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.42" +version = "0.7.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033" dependencies = [ "proc-macro2", "quote", @@ -4404,9 +4374,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6c4b23d99685a1408194da11270ef8e9809aff951cc70ec9b17350b087e474" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" dependencies = [ "const-oid", "digest", @@ -4493,25 +4463,25 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys 0.4.11", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.12", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.9" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.5", + "ring 0.17.7", "rustls-webpki", "sct", ] @@ -4531,7 +4501,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.5", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -4554,9 +4524,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "safe_arch" @@ -4578,11 +4548,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4597,7 +4567,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.5", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -4609,9 +4579,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "self_cell" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e388332cd64eb80cd595a00941baf513caffae8dce9cfd0467fc9c66397dade6" +checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" [[package]] name = "semver" @@ -4644,9 +4614,9 @@ dependencies = [ [[package]] name = "serde-wasm-bindgen" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ba92964781421b6cef36bf0d7da26d201e96d84e1b10e7ae6ed416e516906d" +checksum = "b9b713f70513ae1f8d92665bbbbda5c295c2cf1da5542881ae5eefe20c9af132" dependencies = [ "js-sys", "serde", @@ -4655,9 +4625,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8bb1879ea93538b78549031e2d54da3e901fd7e75f2e4dc758d760937b123d10" dependencies = [ "serde", ] @@ -4670,7 +4640,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -4686,9 +4656,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -4731,7 +4701,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -4767,9 +4737,9 @@ dependencies = [ [[package]] name = "shared-buffer" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf61602ee61e2f83dd016b3e6387245291cf728ea071c378b35088125b4d995" +checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" dependencies = [ "bytes", "memmap2 0.6.2", @@ -4862,7 +4832,7 @@ dependencies = [ "chacha20poly1305", "curve25519-dalek", "rand_core", - "ring 0.17.5", + "ring 0.17.7", "rustc_version", "sha2", "subtle", @@ -4905,9 +4875,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", "der", @@ -4915,11 +4885,11 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.11.0", + "itertools", "nom", "unicode_categories", ] @@ -5205,9 +5175,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", @@ -5278,35 +5248,35 @@ checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall 0.4.1", - "rustix 0.38.25", - "windows-sys 0.48.0", + "redox_syscall", + "rustix 0.38.28", + "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -5343,9 +5313,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -5363,9 +5333,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -5387,9 +5357,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -5412,7 +5382,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -5435,7 +5405,19 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite", + "tungstenite 0.20.1", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.21.0", ] [[package]] @@ -5569,7 +5551,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -5713,9 +5695,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" @@ -5736,6 +5718,25 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.0.0", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" @@ -5769,9 +5770,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -5957,33 +5958,10 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-downcast" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dac026d43bcca6e7ce1c0956ba68f59edf6403e8e930a5d891be72c31a44340" -dependencies = [ - "js-sys", - "once_cell", - "wasm-bindgen", - "wasm-bindgen-downcast-macros", -] - -[[package]] -name = "wasm-bindgen-downcast-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5020cfa87c7cecefef118055d44e3c1fc122c7ec25701d528ee458a0b45f38f" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.89" @@ -6002,7 +5980,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6024,9 +6002,9 @@ dependencies = [ [[package]] name = "wasmer" -version = "4.2.3" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cb1ae2956aac1fbbcf334c543c1143cdf7d5b0a5fb6c3d23a17bf37dd1f47b" +checksum = "5467c7a23f9be04d5691590bea509dbea27e5ba5810d0020bef908456a495f33" dependencies = [ "bytes", "cfg-if", @@ -6041,7 +6019,6 @@ dependencies = [ "target-lexicon", "thiserror", "wasm-bindgen", - "wasm-bindgen-downcast", "wasmer-compiler", "wasmer-compiler-cranelift", "wasmer-derive", @@ -6053,9 +6030,9 @@ dependencies = [ [[package]] name = "wasmer-compiler" -version = "4.2.3" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fd9aeef339095798d1e04957d5657d97490b1112f145cbf08b98f6393b4a0a" +checksum = "510ad01a668d774f3a103a7c219bbc0970be93e8f1b27e2fdb48d1f4ccd1deff" dependencies = [ "backtrace", "bytes", @@ -6080,9 +6057,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-cranelift" -version = "4.2.3" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "344f5f1186c122756232fe7f156cc8d2e7bf333d5a658e81e25efa3415c26d07" +checksum = "54bf93078990d83960d798de3c5935bddaba771fc2fefb9ed6bab9c0bbdea5c1" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -6099,9 +6076,9 @@ dependencies = [ [[package]] name = "wasmer-derive" -version = "4.2.3" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac8c1f2dc0ed3c7412a5546e468365184a461f8ce7dfe2a707b621724339f91" +checksum = "1b374fd34d97b1c091d8675f9bc472df52dc6787d139d3762d42c7dc84813a9b" dependencies = [ "proc-macro-error", "proc-macro2", @@ -6111,9 +6088,9 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "4.2.3" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a57ecbf218c0a9348d4dfbdac0f9d42d9201ae276dffb13e61ea4ff939ecce7" +checksum = "0caf1c87937b52aba8e9f920a278e1beda282f7439612c0b48f51a58e7a87bab" dependencies = [ "bytecheck", "enum-iterator", @@ -6127,9 +6104,9 @@ dependencies = [ [[package]] name = "wasmer-vm" -version = "4.2.3" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60c3513477bc0097250f6e34a640e2a903bb0ee57e6bb518c427f72c06ac7728" +checksum = "58315c25492bc72a33f47a7d7fb0869a0106fc0164ec051e349a9e1eddba9a01" dependencies = [ "backtrace", "cc", @@ -6144,7 +6121,7 @@ dependencies = [ "lazy_static", "libc", "mach", - "memoffset 0.8.0", + "memoffset", "more-asserts", "region", "scopeguard", @@ -6186,9 +6163,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -6196,9 +6173,9 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" dependencies = [ "js-sys", "wasm-bindgen", @@ -6468,9 +6445,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.19" +version = "0.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" dependencies = [ "memchr", ] @@ -6525,11 +6502,13 @@ dependencies = [ [[package]] name = "xattr" -version = "1.0.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" dependencies = [ "libc", + "linux-raw-sys 0.4.12", + "rustix 0.38.28", ] [[package]] @@ -6549,9 +6528,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" +checksum = "53be06678ed9e83edb1745eb72efc0bbcd7b5c3c35711a860906aed827a13d61" [[package]] name = "xz2" @@ -6597,22 +6576,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.26" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.26" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] [[package]] @@ -6632,5 +6611,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.43", ] diff --git a/Cargo.toml b/Cargo.toml index 3b5340623..00631ff12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ futures = "0.3" rand = { version = "0.8" } semver = { version = "1.0.14", features = ["serde"] } serde = { version = "1", features = ["derive"] } -serde_bytes = "0.11" serde_json = "1" serde_with = "3" tracing = "0.1" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index db15a3a04..8ee60d290 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -38,7 +38,7 @@ fastrand = { workspace = true } flatbuffers = "23.5.26" futures = "0.3.21" headers = "0.4" -itertools = "0.11" +itertools = "0.12.0" libp2p = { default-features = false, features = ["autonat", "dns", "ed25519", "identify", "macros", "noise", "ping", "tcp", "tokio", "yamux"], version = "0.52.3" } libp2p-identity = { features = ["ed25519", "rand"], version = "0.2.7" } notify = "6" @@ -56,12 +56,13 @@ stretto = { features = ["async", "sync"], version = "0.8" } tar = { version = "0.4.38" } thiserror = "1" tokio = { features = ["fs", "macros", "rt-multi-thread", "sync", "process"], version = "1" } -tokio-tungstenite = "0.20" +tokio-tungstenite = "0.21.0" tower-http = { features = ["fs", "trace"], version = "0.5" } ulid = { features = ["serde"], version = "1.1" } unsigned-varint = "0.7" wasmer = { features = ["sys"], workspace = true } xz2 = { version = "0.1" } +# enum-iterator = "1.4.1" # Tracing deps opentelemetry = "0.21.0" @@ -72,6 +73,7 @@ tracing-subscriber = { optional = true, version = "0.3.16" } # internal deps freenet-stdlib = { features = ["net"], workspace = true } +time = "0.3.30" [dev-dependencies] arbitrary = { features = ["derive"], version = "1" } diff --git a/crates/core/src/contract/handler.rs b/crates/core/src/contract/handler.rs index 32fc7740c..1f6d91a41 100644 --- a/crates/core/src/contract/handler.rs +++ b/crates/core/src/contract/handler.rs @@ -21,7 +21,7 @@ use crate::{client_events::ClientId, node::PeerCliConfig, wasm_runtime::Runtime, pub(crate) struct ClientResponsesReceiver(UnboundedReceiver<(ClientId, HostResult)>); -pub fn client_responses_channel() -> (ClientResponsesReceiver, ClientResponsesSender) { +pub(crate) fn client_responses_channel() -> (ClientResponsesReceiver, ClientResponsesSender) { let (tx, rx) = mpsc::unbounded_channel(); (ClientResponsesReceiver(rx), ClientResponsesSender(tx)) } diff --git a/crates/core/src/contract/storages/mod.rs b/crates/core/src/contract/storages/mod.rs index e55e61c94..121531307 100644 --- a/crates/core/src/contract/storages/mod.rs +++ b/crates/core/src/contract/storages/mod.rs @@ -1,7 +1,7 @@ #[cfg(feature = "sqlite")] pub mod sqlite; #[cfg(feature = "sqlite")] -pub use sqlite::{Pool as SqlitePool, SqlDbError}; +pub use sqlite::Pool as SqlitePool; #[cfg(feature = "sqlite")] pub type Storage = SqlitePool; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 0eeb93872..14224aaf4 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -12,8 +12,6 @@ mod message; mod node; /// Network operation/transaction state machines. mod operations; -/// Resource usage tracking. -mod resources; /// Ring connections and routing. mod ring; /// Router implementation. @@ -53,3 +51,6 @@ pub mod dev_tool { pub use ring::Location; pub use wasm_runtime::{ContractStore, DelegateStore, Runtime, SecretsStore, StateStore}; } + +#[cfg(test)] +pub mod test_utils; diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 2ea13ede1..9e1389b69 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -37,14 +37,14 @@ pub(crate) struct Transaction { impl Transaction { pub const NULL: &'static Transaction = &Transaction { id: Ulid(0) }; - pub fn new() -> Self { + pub(crate) fn new() -> Self { let ty = ::tx_type_id(); let id = Ulid::new(); Self::update(ty.0, id) // Self { id } } - pub fn transaction_type(&self) -> TransactionType { + pub(crate) fn transaction_type(&self) -> TransactionType { let id_byte = (self.id.0 & 0xFFu128) as u8; match id_byte { 0 => TransactionType::Connect, diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index f715403bc..316c3fc88 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -48,6 +48,7 @@ use crate::operations::handle_op_request; pub use network_bridge::inter_process::InterProcessConnManager; pub(crate) use network_bridge::{ConnectionError, EventLoopNotificationsSender, NetworkBridge}; +use crate::topology::rate::Rate; pub(crate) use op_state_manager::{OpManager, OpNotAvailable}; mod network_bridge; @@ -113,6 +114,8 @@ pub struct NodeConfig { pub(crate) rnd_if_htl_above: Option, pub(crate) max_number_conn: Option, pub(crate) min_number_conn: Option, + pub(crate) max_upstream_bandwidth: Option, + pub(crate) max_downstream_bandwidth: Option, } impl NodeConfig { @@ -130,6 +133,8 @@ impl NodeConfig { rnd_if_htl_above: None, max_number_conn: None, min_number_conn: None, + max_upstream_bandwidth: None, + max_downstream_bandwidth: None, } } @@ -277,10 +282,9 @@ impl InitPeerNode { /// Will panic if is not a valid representation. pub fn decode_peer_id>(mut bytes: T) -> Libp2pPeerId { Libp2pPeerId::from_public_key( - &identity::Keypair::try_from( + &identity::Keypair::from( identity::ed25519::Keypair::try_from_bytes(bytes.as_mut()).unwrap(), ) - .unwrap() .public(), ) } diff --git a/crates/core/src/node/network_bridge.rs b/crates/core/src/node/network_bridge.rs index 577de16f9..df83a953d 100644 --- a/crates/core/src/node/network_bridge.rs +++ b/crates/core/src/node/network_bridge.rs @@ -83,7 +83,7 @@ impl Clone for ConnectionError { } } -pub fn event_loop_notification_channel( +pub(crate) fn event_loop_notification_channel( ) -> (EventLoopNotificationsReceiver, EventLoopNotificationsSender) { let (notification_tx, notification_rx) = mpsc::channel(100); ( @@ -91,7 +91,7 @@ pub fn event_loop_notification_channel( EventLoopNotificationsSender(notification_tx), ) } -pub(super) struct EventLoopNotificationsReceiver(Receiver>); +pub(crate) struct EventLoopNotificationsReceiver(Receiver>); impl Deref for EventLoopNotificationsReceiver { type Target = Receiver>; diff --git a/crates/core/src/node/op_state_manager.rs b/crates/core/src/node/op_state_manager.rs index 589f0e0e7..cb39a28ce 100644 --- a/crates/core/src/node/op_state_manager.rs +++ b/crates/core/src/node/op_state_manager.rs @@ -201,9 +201,9 @@ impl OpManager { /// Notify the operation manager that a transaction is being transacted over the network. pub fn sending_transaction(&self, peer: &PeerId, msg: &NetMessage) { let transaction = msg.id(); - if let Some(loc) = msg.requested_location() { + if let (Some(recipient), Some(target)) = (msg.target(), msg.requested_location()) { self.ring - .record_request(loc, transaction.transaction_type()); + .record_request(*recipient, target, transaction.transaction_type()); } self.ring .live_tx_tracker diff --git a/crates/core/src/resources.rs b/crates/core/src/resources.rs deleted file mode 100644 index f535e733b..000000000 --- a/crates/core/src/resources.rs +++ /dev/null @@ -1,409 +0,0 @@ -//! # Resource Management -//! -//! The resource management module is responsible for tracking resource usage, -//! ensuring that usage does not exceed specified limits, and ensure that those -//! resources are used to maximize the utility of the network. If limits are -//! exceeded then peers are removed until the usage is below the limit.git -//! -//! ## Resources -//! -//! The resource management module tracks usage of the following resources: -//! -//! * Upstream and downstream bandwidth -//! * CPU usage -//! -//! These resources will be tracked in the future: -//! -//! * Memory usage -//! * Storage -//! -//! ## Attribution -//! -//! When used this resource usage is attributed to either: -//! -//! * Remote -//! * A connected peer -//! * Local -//! * A local delegate -//! * The user interface -//! -//! ## Resource allocation for contract subscriptions -//! -//! When one or more peers are subscribed to a contract, the required -//! resources will be allocated as follows: -//! -//! * Upstream bandwidth is allocated to the subscribed peer to which -//! the data is sent -//! * Downstream bandwidth and CPU is split evenly between all subscribed -//! peers for that contract -//! -//! ## Resource limits -//! -//! Resource limits should be set to ensure that the peer does not disrupt the -//! user's experience of their computer. We should choose intelligent defaults -//! but the limits should be user-configurable. -//! -//! ## Code component overview -//! -//! The [ResourceManager] is responsible for tracking resource usage and identifying -//! which peers to remove if/when limits are exceeded. It does this by identifying -//! the peers with the highest usage of the limit-exceeding resource relative to -//! their usefulness. The usefulness of a peer is determined by the number of -//! requests sent to that peer over a certain period of time. -//! -//! A [Meter] is used by the ResourceManager to tracking resource usage over time. -//! Resource usage is reported to the meter, which tracks the usage over a sliding -//! window of time. The meter is responsible for calculating the rate of resource -//! usage along with which peers (or delegates, or UIs) are responsible for that -//! usage. -//! -//! ## Future Improvements -//! -//! * Track non-flow resources like memory and storage -//! * Responses to specific requests will contain information about the resources used -//! by downstream peers to fulfill the request, however how this information is used -//! will require careful consideration. -#![allow(dead_code, unused)] // FIXME: remove after integration - -mod meter; -pub mod rate; -mod running_average; - -use self::meter::{AttributionSource, Meter, ResourceType}; -use crate::ring::PeerKeyLocation; -use std::time::Instant; - -pub(crate) struct ResourceManager { - limits: Limits, - meter: Meter, -} - -impl ResourceManager { - pub fn new(limits: Limits) -> Self { - ResourceManager { - meter: Meter::new_with_window_size(100), - limits, - } - } - - pub fn update_limits(&mut self, limits: Limits) { - self.limits = limits; - } - - /// Report the use of a resource. - pub(crate) fn report( - &mut self, - _time: Instant, - attribution: &AttributionSource, - resource: ResourceType, - value: f64, - ) { - self.meter.report(attribution, resource, value); - } - - /// Determines which peers should be deleted to reduce resource usage below - /// the specified limit. - /// - /// Given a resource type and a list of candidate peers, this function - /// calculates which peers should be deleted in order to bring the total - /// resource usage below the specified limit. Each candidate peer is - /// accompanied by an `f64` value representing its usefulness, typically - /// measured as the number of requests sent to the peer over a certain - /// period of time. - /// - /// Peers are prioritized for deletion based on their usefulness relative to - /// their current resource usage. The function returns a list of - /// `PeerKeyLocation` objects representing the peers that should be deleted. - /// If the total resource usage is already below the limit, an empty list is - /// returned. - /// - /// The usefulness value for each peer must be greater than zero. To prevent - /// division by zero, any usefulness value less than or equal to 0.0001 is - /// treated as 0.0001. - /// - /// # Parameters - /// - `resource_type`: The type of resource for which usage is being - /// measured. - /// - `candidates`: An iterator over `PeerValue` objects, where each - /// `PeerValue` consists of a peer and its associated usefulness value. - /// - /// # Returns - /// A `Vec` containing the peers that should be deleted to - /// bring resource usage below the limit. If no peers need to be deleted, an - /// empty vector is returned. - pub(crate) fn should_delete_peers

( - &self, - resource_type: ResourceType, - candidates: P, - ) -> Vec - where - P: IntoIterator, - { - let total_usage = match self.meter.resource_usage_rate(resource_type) { - Some(rate) => rate.per_second(), - None => return vec![], // Or handle the error as appropriate - }; - - let total_limit: f64 = self.limits.get(resource_type); - if total_usage > total_limit { - let mut candidate_costs = vec![]; - for PeerValue { peer, value } in candidates { - if let Some(cost) = self - .meter - .attributed_usage_rate(&AttributionSource::Peer(peer), resource_type) - { - const MIN_VALUE: f64 = 0.0001; - let cost_per_second = cost.per_second(); - let cost_per_value = cost_per_second / value.max(MIN_VALUE); - candidate_costs.push(CandidateCost { - peer, - total_cost: cost_per_second, - cost_per_value, - }); - } // Else, you might want to handle the case where cost is None - } - - // Sort candidate_costs by cost_per_value descending - candidate_costs.sort_by(|a, b| { - b.cost_per_value - .partial_cmp(&a.cost_per_value) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let mut to_delete = vec![]; - let excess_usage = total_usage - total_limit; - let mut total_deleted_cost = 0.0; - for candidate in candidate_costs { - if total_deleted_cost >= excess_usage { - break; - } - total_deleted_cost += candidate.total_cost; - to_delete.push(candidate.peer); - } - to_delete - } else { - vec![] - } - } -} - -#[derive(Debug, Clone, Copy)] -pub(crate) struct PeerValue { - pub peer: PeerKeyLocation, - pub value: f64, -} - -#[derive(Debug, Clone, Copy)] -struct CandidateCost { - peer: PeerKeyLocation, - total_cost: f64, - cost_per_value: f64, -} - -pub struct Limits { - pub max_upstream_bandwidth: BytesPerSecond, - pub max_downstream_bandwidth: BytesPerSecond, - pub max_cpu_usage: InstructionsPerSecond, - pub max_memory_usage: f64, - pub max_storage_usage: f64, -} - -impl Limits { - pub fn get(&self, resource_type: ResourceType) -> f64 { - match resource_type { - ResourceType::OutboundBandwidthBytes => self.max_upstream_bandwidth.into(), - ResourceType::InboundBandwidthBytes => self.max_downstream_bandwidth.into(), - ResourceType::CpuInstructions => self.max_cpu_usage.into(), - // TODO: Support non-flow resources like memory and storage use - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct BytesPerSecond(f64); -impl BytesPerSecond { - pub fn new(bytes_per_second: f64) -> Self { - BytesPerSecond(bytes_per_second) - } -} - -impl From for f64 { - fn from(val: BytesPerSecond) -> Self { - val.0 - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct InstructionsPerSecond(f64); -impl InstructionsPerSecond { - pub fn new(cpu_usage: f64) -> Self { - InstructionsPerSecond(cpu_usage) - } -} - -impl From for f64 { - fn from(val: InstructionsPerSecond) -> Self { - val.0 - } -} - -#[derive(Debug, Clone, Copy)] -pub struct ByteCount(u64); -impl ByteCount { - pub fn new(bytes: u64) -> Self { - ByteCount(bytes) - } -} - -impl From for f64 { - fn from(val: ByteCount) -> Self { - val.0 as f64 - } -} - -#[cfg(test)] -mod tests { - use crate::resources::{BytesPerSecond, InstructionsPerSecond, Limits, ResourceManager}; - - use super::*; - use std::time::Instant; - - #[test] - fn test_resource_manager_report() { - // Create a ResourceManager with arbitrary limits - let limits = Limits { - max_upstream_bandwidth: BytesPerSecond::new(1000.0), - max_downstream_bandwidth: BytesPerSecond::new(1000.0), - max_cpu_usage: InstructionsPerSecond::new(1000.0), - max_memory_usage: 1000.0, - max_storage_usage: 1000.0, - }; - let mut resource_manager = ResourceManager::new(limits); - - // Report some usage and test that the total and attributed usage are updated - let attribution = AttributionSource::Peer(PeerKeyLocation::random()); - let time = Instant::now(); - resource_manager.report( - time, - &attribution, - ResourceType::InboundBandwidthBytes, - 100.0, - ); - assert_eq!( - resource_manager - .meter - .resource_usage_rate(ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 100.0 - ); - assert_eq!( - resource_manager - .meter - .attributed_usage_rate(&attribution, ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 100.0 - ); - } - - #[test] - fn test_resource_manager_should_delete_peers() { - // Create a ResourceManager with arbitrary limits - let limits = Limits { - max_upstream_bandwidth: BytesPerSecond::new(1000.0), - max_downstream_bandwidth: BytesPerSecond::new(1000.0), - max_cpu_usage: InstructionsPerSecond::new(1000.0), - max_memory_usage: 1000.0, - max_storage_usage: 1000.0, - }; - let mut resource_manager = ResourceManager::new(limits); - - // Report some usage - let peer1 = PeerKeyLocation::random(); - let attribution1 = AttributionSource::Peer(peer1); - let peer2 = PeerKeyLocation::random(); - let attribution2 = AttributionSource::Peer(peer2); - let time = Instant::now(); - resource_manager.report( - time, - &attribution1, - ResourceType::InboundBandwidthBytes, - 400.0, - ); - resource_manager.report( - time, - &attribution2, - ResourceType::InboundBandwidthBytes, - 500.0, - ); - - // Test that no peers should be deleted when the total usage is below the limit - let candidates = vec![ - PeerValue { - peer: peer1, - value: 1.0, - }, - PeerValue { - peer: peer2, - value: 1.0, - }, - ]; - let to_delete = resource_manager - .should_delete_peers(ResourceType::InboundBandwidthBytes, candidates.clone()); - - // Test that no peers should be deleted when the total usage is below the limit - assert_eq!(to_delete.len(), 0); - - // Report more usage to exceed the limit - resource_manager.report( - time, - &attribution1, - ResourceType::InboundBandwidthBytes, - 200.0, - ); - - let to_delete = - resource_manager.should_delete_peers(ResourceType::InboundBandwidthBytes, candidates); - assert_eq!(to_delete.len(), 1); - - // Test that the peer with the highest usage is deleted - assert_eq!(to_delete[0], peer1); - } - - #[test] - fn test_update_limits() { - let limits = Limits { - max_upstream_bandwidth: BytesPerSecond::new(1000.0), - max_downstream_bandwidth: BytesPerSecond::new(1000.0), - max_cpu_usage: InstructionsPerSecond::new(1000.0), - max_memory_usage: 1000.0, - max_storage_usage: 1000.0, - }; - let mut resource_manager = ResourceManager::new(limits); - - let new_limits = Limits { - max_upstream_bandwidth: BytesPerSecond::new(2000.0), - max_downstream_bandwidth: BytesPerSecond::new(2000.0), - max_cpu_usage: InstructionsPerSecond::new(2000.0), - max_memory_usage: 2000.0, - max_storage_usage: 2000.0, - }; - resource_manager.update_limits(new_limits); - - assert_eq!( - resource_manager.limits.max_upstream_bandwidth, - BytesPerSecond::new(2000.0) - ); - assert_eq!( - resource_manager.limits.max_downstream_bandwidth, - BytesPerSecond::new(2000.0) - ); - assert_eq!( - resource_manager.limits.max_cpu_usage, - InstructionsPerSecond::new(2000.0) - ); - assert_eq!(resource_manager.limits.max_memory_usage, 2000.0); - assert_eq!(resource_manager.limits.max_storage_usage, 2000.0); - } -} diff --git a/crates/core/src/resources/meter.rs b/crates/core/src/resources/meter.rs deleted file mode 100644 index 27307011d..000000000 --- a/crates/core/src/resources/meter.rs +++ /dev/null @@ -1,288 +0,0 @@ -use std::hash::Hash; -use std::time::Duration; - -use crate::resources::rate::Rate; -use crate::resources::{BytesPerSecond, InstructionsPerSecond}; -use dashmap::DashMap; -use freenet_stdlib::prelude::*; - -use crate::ring::PeerKeyLocation; - -use super::running_average::RunningAverage; - -/// A structure that keeps track of the usage of dynamic resources which are consumed over time. -/// It provides methods to report and query resource usage, both total and attributed to specific -/// sources. -pub(super) struct Meter { - totals_by_resource: ResourceTotals, - attribution_meters: AttributionMeters, - running_average_window_size: usize, -} - -impl Meter { - /// Creates a new `Meter`. - pub fn new_with_window_size(running_average_window_size: usize) -> Self { - Meter { - totals_by_resource: ResourceTotals::new(), - attribution_meters: AttributionMeters::new(), - running_average_window_size, - } - } - - pub fn resource_usage_rate(&self, resource: ResourceType) -> Option { - // Try to get a mutable reference to the Meter for the given resource - match self.totals_by_resource.map.get_mut(&resource) { - Some(meter) => { - // Get the current measurement value - meter.get_rate() - } - None => None, // No meter found for the given resource - } - } - - pub(crate) fn attributed_usage_rate( - &self, - attribution: &AttributionSource, - resource: ResourceType, - ) -> Option { - // Try to get a mutable reference to the AttributionMeters for the given attribution - match self.attribution_meters.get_mut(attribution) { - Some(attribution_meters) => { - // Try to get a mutable reference to the Meter for the given resource - match attribution_meters.map.get_mut(&resource) { - Some(meter) => { - // Get the current measurement value - meter.get_rate() - } - None => Some(Rate::new(0.0, Duration::from_secs(1))), // No meter found for the given resource - } - } - None => None, // No AttributionMeters found for the given attribution - } - } - - /// Report the use of a resource with multiple attribution sources, splitting the usage - /// evenly between the sources. - /// This should be done in the lowest-level functions that consume the resource, taking - /// an AttributionMeter as a parameter. This will be useful for contracts with multiple - /// subscribers - where the responsibility should be split evenly among the subscribers. - pub(crate) fn report_split( - &mut self, - attributions: &[AttributionSource], - resource: ResourceType, - value: f64, - ) { - let split_value = value / attributions.len() as f64; - for attribution in attributions { - self.report(attribution, resource, split_value); - } - } - - /// Report the use of a resource. This should be done in the lowest-level - /// functions that consume the resource, taking an AttributionMeter - /// as a parameter. - pub(crate) fn report( - &mut self, - attribution: &AttributionSource, - resource: ResourceType, - value: f64, - ) { - // Report the total usage for the resource - let mut total_value = self - .totals_by_resource - .map - .entry(resource) - .or_insert_with(|| RunningAverage::new(self.running_average_window_size)); - total_value.insert(value); - - // Report the usage for a specific attribution - let resource_map = self - .attribution_meters - .entry(attribution.clone()) - .or_insert_with(ResourceTotals::new); - let mut resource_value = resource_map - .map - .entry(resource) - .or_insert_with(|| RunningAverage::new(self.running_average_window_size)); - resource_value.insert(value); - } -} - -#[derive(Eq, Hash, PartialEq, Clone, Debug)] -pub(crate) enum AttributionSource { - Peer(PeerKeyLocation), - Delegate(DelegateKey), -} - -#[derive(Eq, Hash, PartialEq, Clone, Copy, Debug)] -pub enum ResourceType { - InboundBandwidthBytes, - OutboundBandwidthBytes, - CpuInstructions, -} - -type AttributionMeters = DashMap; - -/// A structure that holds running averages of resource usage for different resource types. -struct ResourceTotals { - pub map: DashMap, -} - -impl ResourceTotals { - fn new() -> Self { - ResourceTotals { - map: DashMap::new(), - } - } -} - -// Tests -#[cfg(test)] -mod tests { - use crate::DynError; - use std::time::Duration; - - use super::*; - - #[test] - fn test_meter() { - let meter = Meter::new_with_window_size(100); - - // Test that the new Meter has empty totals_by_resource and attribution_meters - assert!(meter.totals_by_resource.map.is_empty()); - assert!(meter.attribution_meters.is_empty()); - } - - #[test] - fn test_meter_total_usage() { - let mut meter = Meter::new_with_window_size(100); - - // Test that the total usage is 0.0 for all resources - assert!(meter - .resource_usage_rate(ResourceType::InboundBandwidthBytes) - .is_none()); - assert!(meter - .resource_usage_rate(ResourceType::OutboundBandwidthBytes) - .is_none()); - assert!(meter - .resource_usage_rate(ResourceType::CpuInstructions) - .is_none()); - - // Report some usage and test that the total usage is updated - let attribution = AttributionSource::Peer(PeerKeyLocation::random()); - meter.report(&attribution, ResourceType::InboundBandwidthBytes, 100.0); - assert_eq!( - meter - .resource_usage_rate(ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 100.0 - ); - } - - #[test] - fn test_meter_attributed_usage() { - let mut meter = Meter::new_with_window_size(100); - - // Test that the attributed usage is 0.0 for all resources - let attribution = AttributionSource::Peer(PeerKeyLocation::random()); - assert!(meter - .attributed_usage_rate(&attribution, ResourceType::InboundBandwidthBytes) - .is_none()); - assert!(meter - .attributed_usage_rate(&attribution, ResourceType::OutboundBandwidthBytes) - .is_none()); - assert!(meter - .attributed_usage_rate(&attribution, ResourceType::CpuInstructions) - .is_none()); - - // Report some usage and test that the attributed usage is updated - meter.report(&attribution, ResourceType::InboundBandwidthBytes, 100.0); - assert_eq!( - meter - .attributed_usage_rate(&attribution, ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 100.0 - ); - } - - #[test] - fn test_meter_report() -> Result<(), DynError> { - let mut meter = Meter::new_with_window_size(100); - - // Report some usage and test that the total and attributed usage are updated - let attribution = AttributionSource::Peer(PeerKeyLocation::random()); - meter.report(&attribution, ResourceType::InboundBandwidthBytes, 100.0); - assert_eq!( - meter - .resource_usage_rate(ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 100.0 - ); - assert_eq!( - meter - .attributed_usage_rate(&attribution, ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 100.0 - ); - - // Report more usage and test that the total and attributed usage are updated - meter.report(&attribution, ResourceType::InboundBandwidthBytes, 200.0); - assert_eq!( - meter - .resource_usage_rate(ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 300.0 - ); - assert_eq!( - meter - .attributed_usage_rate(&attribution, ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 300.0 - ); - // Report usage for a different resource and test that the total and attributed usage are updated - meter.report(&attribution, ResourceType::CpuInstructions, 50.0); - assert_eq!( - meter - .resource_usage_rate(ResourceType::CpuInstructions) - .unwrap() - .per_second(), - 50.0 - ); - assert_eq!( - meter - .attributed_usage_rate(&attribution, ResourceType::CpuInstructions) - .unwrap() - .per_second(), - 50.0 - ); - - // Report usage for a different attribution and test that the total and attributed usage are updated - let other_attribution = AttributionSource::Peer(PeerKeyLocation::random()); - meter.report( - &other_attribution, - ResourceType::InboundBandwidthBytes, - 150.0, - ); - assert_eq!( - meter - .resource_usage_rate(ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 450.0 - ); - assert_eq!( - meter - .attributed_usage_rate(&other_attribution, ResourceType::InboundBandwidthBytes) - .unwrap() - .per_second(), - 150.0 - ); - Ok(()) - } -} diff --git a/crates/core/src/resources/rate.rs b/crates/core/src/resources/rate.rs deleted file mode 100644 index 09cb201c8..000000000 --- a/crates/core/src/resources/rate.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::time::Duration; - -#[derive(Debug, PartialEq)] -pub struct Rate { - value: f64, -} - -impl Rate { - pub fn new(value: f64, divisor: Duration) -> Self { - Rate { - value: value / divisor.as_secs_f64(), - } - } - - pub fn per_second(&self) -> f64 { - self.value - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_rate() { - let rate = Rate::new(100.0, Duration::from_secs(2)); - assert_eq!(rate.per_second(), 50.0); - } -} diff --git a/crates/core/src/ring.rs b/crates/core/src/ring.rs index b169da080..42a4ba7b6 100644 --- a/crates/core/src/ring.rs +++ b/crates/core/src/ring.rs @@ -3,8 +3,8 @@ //! Mainly maintains a healthy and optimal pool of connections to other peers in the network //! and routes requests to the optimal peers. +use std::collections::VecDeque; use std::hash::Hash; -use std::sync::atomic::AtomicBool; use std::{ cmp::Reverse, collections::BTreeMap, @@ -31,7 +31,8 @@ use tokio::sync; use tracing::Instrument; use crate::message::TransactionType; -use crate::topology::{AcquisitionStrategy, TopologyManager}; +use crate::topology::rate::Rate; +use crate::topology::{Limits, TopologyAdjustment, TopologyManager}; use crate::tracing::{NetEventLog, NetEventRegister}; use crate::util::Contains; use crate::{ @@ -86,7 +87,8 @@ impl Display for PeerKeyLocation { } } -struct Connection { +#[derive(Clone, Copy)] +pub(crate) struct Connection { location: PeerKeyLocation, open_at: Instant, } @@ -168,9 +170,6 @@ pub(crate) struct Ring { pub min_connections: usize, router: Arc>, topology_manager: RwLock, - /// Fast is for when there are less than our target number of connections so we want to acquire new connections quickly. - /// Slow is for when there are enough connections so we need to drop a connection in order to replace it. - fast_acquisition: AtomicBool, connections_by_location: RwLock>>, location_for_peer: RwLock>, own_location: AtomicU64, @@ -192,6 +191,7 @@ pub(crate) struct Ring { event_register: Box, /// Whether this peer is a gateway or not. This will affect behavior of the node when acquiring /// and dropping connections. + #[allow(unused)] is_gateway: bool, } @@ -204,9 +204,13 @@ pub(crate) struct Ring { // } impl Ring { - const DEFAULT_MIN_CONNECTIONS: usize = 10; + const DEFAULT_MIN_CONNECTIONS: usize = 5; - const DEFAULT_MAX_CONNECTIONS: usize = 20; + const DEFAULT_MAX_CONNECTIONS: usize = 200; + + const DEFAULT_MAX_UPSTREAM_BANDWIDTH: Rate = Rate::new_per_second(1_000_000.0); + + const DEFAULT_MAX_DOWNSTREAM_BANDWIDTH: Rate = Rate::new_per_second(1_000_000.0); /// Max number of subscribers for a contract. const MAX_SUBSCRIBERS: usize = 10; @@ -254,12 +258,29 @@ impl Ring { Self::DEFAULT_MAX_CONNECTIONS }; + let max_upstream_bandwidth = if let Some(v) = config.max_upstream_bandwidth { + v + } else { + Self::DEFAULT_MAX_UPSTREAM_BANDWIDTH + }; + + let max_downstream_bandwidth = if let Some(v) = config.max_downstream_bandwidth { + v + } else { + Self::DEFAULT_MAX_DOWNSTREAM_BANDWIDTH + }; + + let topology_manager = RwLock::new(TopologyManager::new(Limits { + max_upstream_bandwidth, + max_downstream_bandwidth, + min_connections, + max_connections, + })); + let router = Arc::new(RwLock::new(Router::new(&[]))); GlobalExecutor::spawn(Self::refresh_router(router.clone(), event_register.clone())); // Just initialize with a fake location, this will be later updated when the peer has an actual location assigned. - let topology_manager = RwLock::new(TopologyManager::new(Location::new(0.0))); - let ring = Ring { rnd_if_htl_above, max_hops_to_live, @@ -267,7 +288,6 @@ impl Ring { min_connections, router, topology_manager, - fast_acquisition: AtomicBool::new(true), connections_by_location: RwLock::new(BTreeMap::new()), location_for_peer: RwLock::new(BTreeMap::new()), own_location, @@ -294,11 +314,13 @@ impl Ring { } else { tracing::info_span!(parent: current_span, "connection_maintenance") }; + GlobalExecutor::spawn( ring.clone() .connection_maintenance(event_loop_notifier, live_tx_tracker, missing_candidate_rx) .instrument(span), ); + Ok(ring) } @@ -362,7 +384,6 @@ impl Ring { u64::from_le_bytes(loc.0.to_le_bytes()), std::sync::atomic::Ordering::Release, ); - self.topology_manager.write().this_peer_location = loc; } else { self.own_location.store( u64::from_le_bytes((-1f64).to_le_bytes()), @@ -417,17 +438,9 @@ impl Ring { } else if open_conn >= self.max_connections { false } else { - let strategy = if self - .fast_acquisition - .load(std::sync::atomic::Ordering::Acquire) - { - AcquisitionStrategy::Fast - } else { - AcquisitionStrategy::Slow - }; self.topology_manager .write() - .evaluate_new_connection(location, strategy) + .evaluate_new_connection(location, Instant::now()) .unwrap_or(false) }; if !accepted { @@ -437,10 +450,15 @@ impl Ring { accepted } - pub fn record_request(&self, requested_location: Location, request_type: TransactionType) { + pub fn record_request( + &self, + recipient: PeerKeyLocation, + target: Location, + request_type: TransactionType, + ) { self.topology_manager .write() - .record_request(requested_location, request_type); + .record_request(recipient, target, request_type); } pub fn add_connection(&self, loc: Location, peer: PeerId) { @@ -461,9 +479,8 @@ impl Ring { fn refresh_density_request_cache(&self) { let cbl = self.connections_by_location.read(); - let current_neighbors = &Self::current_neighbors(&cbl); - let topology_manager = &mut *self.topology_manager.write(); - let _ = topology_manager.refresh_cache(current_neighbors); + let topology_manager = &mut self.topology_manager.write(); + let _ = topology_manager.refresh_cache(&cbl); } /// Return the most optimal peer caching a given contract. @@ -498,6 +515,9 @@ impl Ring { } pub fn routing_finished(&self, event: crate::router::RouteEvent) { + self.topology_manager + .write() + .report_outbound_request(event.peer, event.contract_location); self.router.write().add_event(event); } @@ -611,65 +631,39 @@ impl Ring { .next_back() } - fn current_neighbors( - connections_by_location: &BTreeMap>, - ) -> BTreeMap { - connections_by_location - .iter() - .fold(BTreeMap::new(), |mut map, (loc, conns)| { - map.insert(*loc, conns.len()); - map - }) - } - async fn connection_maintenance( self: Arc, notifier: EventLoopNotificationsSender, live_tx_tracker: LiveTransactionTracker, mut missing_candidates: sync::mpsc::Receiver, ) -> Result<(), DynError> { - /// Peers whose connection should be dropped. - fn should_swap<'a>( - _connections: impl Iterator, - is_gateway: bool, - ) -> Vec { - // todo: instead we should be using ConnectionEvaluator here - // todo: if the peer is a gateway behaviour on how quickly we drop connections may be different - let _ = is_gateway; - vec![] - } - #[cfg(not(test))] const CONNECTION_AGE_THRESOLD: Duration = Duration::from_secs(60 * 5); #[cfg(test)] const CONNECTION_AGE_THRESOLD: Duration = Duration::from_secs(5); - #[cfg(not(test))] - const REMOVAL_TICK_DURATION: Duration = Duration::from_secs(60 * 5); - #[cfg(test)] - const REMOVAL_TICK_DURATION: Duration = Duration::from_secs(1); - const ACQUIRE_CONNS_TICK_DURATION: Duration = Duration::from_secs(2); + const CHECK_TICK_DURATION: Duration = Duration::from_secs(10); const REGENERATE_DENSITY_MAP_INTERVAL: Duration = Duration::from_secs(60); - let mut check_interval = tokio::time::interval(REMOVAL_TICK_DURATION); + let mut check_interval = tokio::time::interval(CHECK_TICK_DURATION); check_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let mut acquire_max_connections = tokio::time::interval(ACQUIRE_CONNS_TICK_DURATION); - acquire_max_connections.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut refresh_density_map = tokio::time::interval(REGENERATE_DENSITY_MAP_INTERVAL); refresh_density_map.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut missing = BTreeMap::new(); #[cfg(not(test))] - let retry_interval = REMOVAL_TICK_DURATION * 2; + let retry_peers_missing_candidates_interval = Duration::from_secs(60 * 5) * 2; #[cfg(test)] - let retry_interval = Duration::from_secs(5); + let retry_peers_missing_candidates_interval = Duration::from_secs(5); // if the peer is just starting wait a bit before // we even attempt acquiring more connections - tokio::time::sleep(ACQUIRE_CONNS_TICK_DURATION).await; + tokio::time::sleep(Duration::from_secs(2)).await; let mut live_tx = None; + let mut pending_conn_adds = VecDeque::new(); 'outer: loop { + // loop { match missing_candidates.try_recv() { Ok(missing_candidate) => { @@ -684,106 +678,79 @@ impl Ring { } // eventually peers which failed to return candidates should be retried when enough time has passed - let retry_missing_candidates_until = Instant::now() - retry_interval; - missing.split_off(&Reverse(retry_missing_candidates_until)); + let retry_missing_candidates_until = + Instant::now() - retry_peers_missing_candidates_interval; - let open_connections = self - .open_connections - .load(std::sync::atomic::Ordering::SeqCst); - - if let Some(tx) = &live_tx { - if !live_tx_tracker.still_alive(tx) { - let _ = live_tx.take(); - } else if open_connections < self.max_connections { - acquire_max_connections.tick().await; - } else { - check_interval.tick().await; - } - } + // remove all missing candidates which have been retried + missing.split_off(&Reverse(retry_missing_candidates_until)); - if open_connections < self.max_connections { - self.fast_acquisition - .store(true, std::sync::atomic::Ordering::Release); - // requires more connections - let ideal_location = { - let loc = { self.topology_manager.read().get_best_candidate_location() }; - match loc { - Ok(loc) => loc, - Err(_) => { - tracing::trace!(peer = %self.own_location(), "Insufficient data gathered by the topology manager"); - acquire_max_connections.tick().await; - continue; - } - } - }; + if let Some(ideal_location) = pending_conn_adds.pop_front() { live_tx = self .acquire_new( ideal_location, &missing.values().collect::>(), ¬ifier, - self.max_connections - open_connections, ) .await .map_err(|error| { tracing::debug!(?error, "Shutting down connection maintenance task"); error })?; + } - acquire_max_connections.tick().await; - continue; + // if there are no open connections, we need to acquire more + if let Some(tx) = &live_tx { + if !live_tx_tracker.still_alive(tx) { + let _ = live_tx.take(); + } } - let mut should_swap = { + let neighbor_locations = { let peers = self.connections_by_location.read(); - should_swap( - peers - .values() - .flatten() - .filter(|conn| { - conn.open_at.elapsed() > CONNECTION_AGE_THRESOLD - && !live_tx_tracker.has_live_connection(&conn.location.peer) - }) - .map(|conn| &conn.location), - self.is_gateway, - ) + peers + .iter() + .map(|(loc, conns)| { + let conns: Vec<_> = conns + .iter() + .filter(|conn| { + conn.open_at.elapsed() > CONNECTION_AGE_THRESOLD + && !live_tx_tracker.has_live_connection(&conn.location.peer) + }) + .cloned() + .collect(); + (*loc, conns) + }) + .filter(|(_, conns)| !conns.is_empty()) + .collect() }; - if !should_swap.is_empty() { - self.fast_acquisition - .store(false, std::sync::atomic::Ordering::Release); - let ideal_location = { - let loc = { self.topology_manager.read().get_best_candidate_location() }; - match loc { - Ok(loc) => loc, - Err(_) => { - tracing::debug!(peer = %self.own_location(), "Insufficient data gathered by the topology manager"); - check_interval.tick().await; - continue; - } + + let adjustment = self.topology_manager.write().adjust_topology( + &neighbor_locations, + &self.own_location().location, + Instant::now(), + ); + match adjustment { + TopologyAdjustment::AddConnections(target_locs) => { + pending_conn_adds.extend(target_locs); + continue; + } + TopologyAdjustment::RemoveConnections(mut should_disconnect_peers) => { + for peer in should_disconnect_peers.drain(..) { + notifier + .send(Either::Right(crate::message::NodeEvent::DropConnection( + peer.peer, + ))) + .await + .map_err(|error| { + tracing::debug!( + ?error, + "Shutting down connection maintenance task" + ); + error + })?; } - }; - live_tx = self - .acquire_new( - ideal_location, - &missing.values().collect::>(), - ¬ifier, - should_swap.len(), - ) - .await - .map_err(|error| { - tracing::warn!(?error, "Shutting down connection maintenance task"); - error - })?; - for peer in should_swap.drain(..) { - notifier - .send(Either::Right(crate::message::NodeEvent::DropConnection( - peer, - ))) - .await - .map_err(|error| { - tracing::debug!(?error, "Shutting down connection maintenance task"); - error - })?; } + TopologyAdjustment::NoChange => {} } tokio::select! { @@ -801,7 +768,6 @@ impl Ring { ideal_location: Location, skip_list: &[&PeerId], notifier: &EventLoopNotificationsSender, - missing_connections: usize, ) -> Result, DynError> { use crate::message::InnerMessage; let Some(query_target) = self.routing(ideal_location, None, skip_list) else { @@ -814,6 +780,7 @@ impl Ring { %ideal_location, "Adding new connections" ); + let missing_connections = self.max_connections - self.open_connections(); let msg = connect::ConnectMsg::Request { id: Transaction::new::(), msg: connect::ConnectRequest::FindOptimalPeer { diff --git a/crates/core/src/test_utils.rs b/crates/core/src/test_utils.rs new file mode 100644 index 000000000..91fa15cd1 --- /dev/null +++ b/crates/core/src/test_utils.rs @@ -0,0 +1,9 @@ +pub fn with_tracing(f: impl FnOnce() -> T) -> T { + let subscriber = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_line_number(true) + .with_file(true) + .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE) + .finish(); + tracing::subscriber::with_default(subscriber, f) +} diff --git a/crates/core/src/topology.rs b/crates/core/src/topology.rs index fd673109f..20f6f0025 100644 --- a/crates/core/src/topology.rs +++ b/crates/core/src/topology.rs @@ -1,24 +1,30 @@ -use crate::{ - message::TransactionType, - ring::{Distance, Location}, -}; -use request_density_tracker::CachedDensityMap; +use crate::{message::TransactionType, ring::Location}; +use anyhow::anyhow; +use connection_evaluator::ConnectionEvaluator; +use meter::Meter; +use outbound_request_counter::OutboundRequestCounter; +use request_density_tracker::{CachedDensityMap, RequestDensityTracker}; +use std::cmp::Ordering; use std::{ - collections::BTreeMap, - time::{Duration, Instant}, + collections::{BTreeMap, HashMap}, + time::Instant, }; +use tracing::{debug, error, event, info, span, Level}; -mod connection_evaluator; -mod request_density_tracker; +pub mod connection_evaluator; +mod constants; +pub(crate) mod meter; +pub(crate) mod outbound_request_counter; +pub(crate) mod rate; +pub mod request_density_tracker; +pub(crate) mod running_average; mod small_world_rand; +use crate::ring::{Connection, PeerKeyLocation}; +use crate::topology::meter::{AttributionSource, ResourceType}; +use crate::topology::rate::{Rate, RateProportion}; +use constants::*; use request_density_tracker::DensityMapError; -use small_world_rand::random_link_distance; - -const SLOW_CONNECTION_EVALUATOR_WINDOW_DURATION: Duration = Duration::from_secs(5 * 60); -const FAST_CONNECTION_EVALUATOR_WINDOW_DURATION: Duration = Duration::from_secs(60); -const REQUEST_DENSITY_TRACKER_WINDOW_SIZE: usize = 10_000; -const RANDOM_CLOSEST_DISTANCE: f64 = 1.0 / 1000.0; /// The goal of `TopologyManager` is to select new connections such that the /// distribution of connections is as close as possible to the @@ -39,49 +45,62 @@ const RANDOM_CLOSEST_DISTANCE: f64 = 1.0 / 1000.0; /// time window. The goal of this is to select the best connections over time /// from incoming join requests. pub(crate) struct TopologyManager { - slow_connection_evaluator: connection_evaluator::ConnectionEvaluator, - fast_connection_evaluator: connection_evaluator::ConnectionEvaluator, - request_density_tracker: request_density_tracker::RequestDensityTracker, + limits: Limits, + meter: Meter, + source_creation_times: HashMap, + slow_connection_evaluator: ConnectionEvaluator, + fast_connection_evaluator: ConnectionEvaluator, + request_density_tracker: RequestDensityTracker, + pub(crate) outbound_request_counter: OutboundRequestCounter, /// Must be updated when new neightbors are discovered. cached_density_map: CachedDensityMap, - pub this_peer_location: Location, + connection_acquisition_strategy: ConnectionAcquisitionStrategy, } impl TopologyManager { /// Create a new TopologyManager specifying the peer's own Location - pub(crate) fn new(this_peer_location: Location) -> Self { + pub(crate) fn new(limits: Limits) -> Self { TopologyManager { - slow_connection_evaluator: connection_evaluator::ConnectionEvaluator::new( + meter: Meter::new_with_window_size(100), + limits, + source_creation_times: HashMap::new(), + slow_connection_evaluator: ConnectionEvaluator::new( SLOW_CONNECTION_EVALUATOR_WINDOW_DURATION, ), - fast_connection_evaluator: connection_evaluator::ConnectionEvaluator::new( + fast_connection_evaluator: ConnectionEvaluator::new( FAST_CONNECTION_EVALUATOR_WINDOW_DURATION, ), - request_density_tracker: request_density_tracker::RequestDensityTracker::new( + request_density_tracker: RequestDensityTracker::new( REQUEST_DENSITY_TRACKER_WINDOW_SIZE, ), cached_density_map: CachedDensityMap::new(), - this_peer_location, + outbound_request_counter: OutboundRequestCounter::new( + OUTBOUND_REQUEST_COUNTER_WINDOW_SIZE, + ), + connection_acquisition_strategy: ConnectionAcquisitionStrategy::Fast, } } pub(crate) fn refresh_cache( &mut self, - current_neighbors: &BTreeMap, + neighbor_locations: &BTreeMap>, ) -> Result<(), DensityMapError> { self.cached_density_map - .set(&self.request_density_tracker, current_neighbors)?; + .set(&self.request_density_tracker, neighbor_locations)?; Ok(()) } /// Record a request and the location it's targeting pub(crate) fn record_request( &mut self, - requested_location: Location, + recipient: PeerKeyLocation, + target: Location, request_type: TransactionType, ) { - tracing::debug!(%request_type, %requested_location, "Recording request for location"); - self.request_density_tracker.sample(requested_location); + debug!(%request_type, %recipient, "Recording request sent to peer"); + + self.request_density_tracker.sample(target); + self.outbound_request_counter.record_request(recipient); } /// Decide whether to accept a connection from a new candidate peer based on its location @@ -90,19 +109,6 @@ impl TopologyManager { pub(crate) fn evaluate_new_connection( &mut self, candidate_location: Location, - acquisition_strategy: AcquisitionStrategy, - ) -> Result { - self.evaluate_new_connection_with_current_time( - candidate_location, - acquisition_strategy, - Instant::now(), - ) - } - - fn evaluate_new_connection_with_current_time( - &mut self, - candidate_location: Location, - acquisition_strategy: AcquisitionStrategy, current_time: Instant, ) -> Result { tracing::debug!( @@ -115,14 +121,14 @@ impl TopologyManager { .ok_or(DensityMapError::EmptyNeighbors)?; let score = density_map.get_density_at(candidate_location)?; - let accept = match acquisition_strategy { - AcquisitionStrategy::Slow => { + let accept = match self.connection_acquisition_strategy { + ConnectionAcquisitionStrategy::Slow => { self.fast_connection_evaluator .record_only_with_current_time(score, current_time); self.slow_connection_evaluator .record_and_eval_with_current_time(score, current_time) } - AcquisitionStrategy::Fast => { + ConnectionAcquisitionStrategy::Fast => { self.slow_connection_evaluator .record_only_with_current_time(score, current_time); self.fast_connection_evaluator @@ -133,8 +139,12 @@ impl TopologyManager { Ok(accept) } + #[cfg(test)] /// Get the ideal location for a new connection based on current neighbors and request density - pub(crate) fn get_best_candidate_location(&self) -> Result { + fn get_best_candidate_location( + &self, + this_peer_location: &Location, + ) -> Result { let density_map = self .cached_density_map .get() @@ -142,35 +152,366 @@ impl TopologyManager { let best_location = match density_map.get_max_density() { Ok(location) => { - tracing::debug!("Max density found at location: {:?}", location); + debug!("Max density found at location: {:?}", location); location } Err(_) => { - tracing::debug!( + debug!( "An error occurred while getting max density, falling back to random location" ); - self.random_location() + *this_peer_location } }; Ok(best_location) } - /// Generates a random location that is close to the current peer location with a small - /// world distribution. - fn random_location(&self) -> Location { - tracing::debug!("Generating random location"); - let distance = random_link_distance(Distance::new(RANDOM_CLOSEST_DISTANCE)); - let location_f64 = if rand::random() { - self.this_peer_location.as_f64() - distance.as_f64() + #[cfg(test)] + pub(self) fn update_limits(&mut self, limits: Limits) { + self.limits = limits; + } + + /// Report the use of a resource with multiple attribution sources, splitting the usage + /// evenly between the sources. + /// This should be done in the lowest-level functions that consume the resource, taking + /// an AttributionMeter as a parameter. This will be useful for contracts with multiple + /// subscribers - where the responsibility should be split evenly among the subscribers. + #[allow(dead_code)] // todo: maybe use this + pub(crate) fn report_split_resource_usage( + &mut self, + attributions: &[AttributionSource], + resource: ResourceType, + value: f64, + at_time: Instant, + ) { + let split_value = value / attributions.len() as f64; + for attribution in attributions { + self.report_resource_usage(attribution, resource, split_value, at_time); + } + } + + #[allow(dead_code)] // fixme: use this + pub(crate) fn report_resource_usage( + &mut self, + attribution: &AttributionSource, + resource: ResourceType, + amount: f64, + at_time: Instant, + ) { + if let Some(creation_time) = self.source_creation_times.get(attribution) { + if at_time < *creation_time { + self.source_creation_times + .insert(attribution.clone(), at_time); + } } else { - self.this_peer_location.as_f64() + distance.as_f64() + self.source_creation_times + .insert(attribution.clone(), at_time); + } + + self.meter.report(attribution, resource, amount, at_time); + } + + /// Record an outbound request to a peer, along with the target Location of that request + pub(crate) fn report_outbound_request(&mut self, peer: PeerKeyLocation, target: Location) { + self.request_density_tracker.sample(target); + self.outbound_request_counter.record_request(peer); + } + + /// Calculate total usage for a resource type extrapolating usage for resources that + /// are younger than [SOURCE_RAMP_UP_DURATION] + fn extrapolated_usage(&mut self, resource_type: &ResourceType, now: Instant) -> Usage { + let function_span = span!(Level::DEBUG, "extrapolated_usage_function"); + let _enter = function_span.enter(); + + let mut total_usage: Rate = Rate::new_per_second(0.0); + let mut usage_per_source: HashMap = HashMap::new(); + + // Step 1: Collect data + let collect_data_span = span!(Level::DEBUG, "collect_data"); + let _collect_data_guard = collect_data_span.enter(); + debug!("Collecting data from source_creation_times"); + let mut usage_data = Vec::new(); + for (source, creation_time) in self.source_creation_times.iter() { + let ramping_up = now.duration_since(*creation_time) <= SOURCE_RAMP_UP_DURATION; + debug!( + "Source: {:?}, Creation time: {:?}, Ramping up: {}", + source, creation_time, ramping_up + ); + usage_data.push((source.clone(), ramping_up)); + } + drop(_collect_data_guard); + + // Step 2: Process data + let process_data_span = span!(Level::DEBUG, "process_data"); + let _process_data_guard = process_data_span.enter(); + debug!("Processing data for usage calculation"); + for (source, ramping_up) in usage_data { + let usage_rate: Option = if ramping_up { + debug!("Source {:?} is ramping up", source); + self.meter.get_adjusted_usage_rate(resource_type, now) + } else { + debug!("Source {:?} is not ramping up", source); + self.attributed_usage_rate(&source, resource_type, now) + }; + debug!("Usage rate for source {:?}: {:?}", source, usage_rate); + total_usage += usage_rate.unwrap_or(Rate::new_per_second(0.0)); + usage_per_source.insert(source, usage_rate.unwrap_or(Rate::new_per_second(0.0))); + } + drop(_process_data_guard); + + debug!("Total usage: {:?}", total_usage); + debug!("Usage per source: {:?}", usage_per_source); + + Usage { + total: total_usage, + per_source: usage_per_source, + } + } + + pub(crate) fn attributed_usage_rate( + &mut self, + source: &AttributionSource, + resource_type: &ResourceType, + now: Instant, + ) -> Option { + self.meter.attributed_usage_rate(source, resource_type, now) + } + + // A function that will determine if any peers should be added or removed based on + // the current resource usage, and either add or remove them + pub(crate) fn adjust_topology( + &mut self, + neighbor_locations: &BTreeMap>, + my_location: &Option, + at_time: Instant, + ) -> TopologyAdjustment { + debug!( + "Adjusting topology at {:?}. Current neighbors: {:?}", + at_time, + neighbor_locations.len() + ); + + if neighbor_locations.len() < self.limits.min_connections { + let mut locations = Vec::new(); + let below_threshold = self.limits.min_connections - neighbor_locations.len(); + if below_threshold > 0 { + for _i in 0..below_threshold { + match my_location { + Some(location) => { + // The first few connect messages should target the peer's own + // location (if known), to reduce the danger of a peer failing to + // cluster + locations.push(*location); + } + None => { + locations.push(Location::random()); + } + } + } + info!( + minimum_num_peers_hard_limit = self.limits.min_connections, + num_peers = neighbor_locations.len(), + to_add = below_threshold, + "Adding peers at random locations to reach minimum number of peers" + ); + } + return TopologyAdjustment::AddConnections(locations); + } + + let increase_usage_if_below: RateProportion = + RateProportion::new(MINIMUM_DESIRED_RESOURCE_USAGE_PROPORTION); + let decrease_usage_if_above: RateProportion = + RateProportion::new(MAXIMUM_DESIRED_RESOURCE_USAGE_PROPORTION); + + let usage_rates = self.calculate_usage_proportion(at_time); + + let (resource_type, usage_proportion) = usage_rates.max_usage_rate; + + // Detailed resource usage information + debug!("Usage proportions: {:?}", usage_rates); + + let adjustment: anyhow::Result = + if neighbor_locations.len() > self.limits.max_connections { + debug!( + "Number of neighbors ({:?}) is above maximum ({:?}), removing connections", + neighbor_locations.len(), + self.limits.max_connections + ); + + self.update_connection_acquisition_strategy(ConnectionAcquisitionStrategy::Slow); + + Ok(self.select_connections_to_remove(&resource_type, at_time)) + } else if usage_proportion < increase_usage_if_below { + debug!( + "{:?} resource usage ({:?}) is below threshold ({:?}), adding connections", + resource_type, usage_proportion, increase_usage_if_below + ); + self.update_connection_acquisition_strategy(ConnectionAcquisitionStrategy::Fast); + self.select_connections_to_add(neighbor_locations) + } else if usage_proportion > decrease_usage_if_above { + debug!( + "{:?} resource usage ({:?}) is above threshold ({:?}), removing connections", + resource_type, usage_proportion, decrease_usage_if_above + ); + Ok(self.select_connections_to_remove(&resource_type, at_time)) + } else { + debug!( + "{:?} resource usage is within acceptable bounds: {:?}", + resource_type, usage_proportion + ); + Ok(TopologyAdjustment::NoChange) + }; + + match &adjustment { + Ok(TopologyAdjustment::AddConnections(connections)) => { + debug!("Added connections: {:?}", connections); + } + Ok(TopologyAdjustment::RemoveConnections(connections)) => { + debug!("Removed connections: {:?}", connections); + } + Ok(TopologyAdjustment::NoChange) => { + debug!("No topology change required."); + } + Err(e) => { + error!("Couldn't adjust topology due to error: {:?}", e); + } + } + + adjustment.unwrap_or(TopologyAdjustment::NoChange) + } + + fn calculate_usage_proportion(&mut self, at_time: Instant) -> UsageRates { + let mut usage_rate_per_type = HashMap::new(); + for resource_type in ResourceType::all() { + let usage = self.extrapolated_usage(&resource_type, at_time); + let proportion = usage.total.proportion_of(&self.limits.get(&resource_type)); + usage_rate_per_type.insert(resource_type, proportion); + } + + let max_usage_rate = usage_rate_per_type + .iter() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(Ordering::Equal)) + .unwrap(); + + UsageRates { + // TODO: Is there a way to avoid this clone()? + usage_rate_per_type: usage_rate_per_type.clone(), + max_usage_rate: (*max_usage_rate.0, *max_usage_rate.1), + } + } + + /// modify the current connection acquisition strategy + fn update_connection_acquisition_strategy( + &mut self, + new_strategy: ConnectionAcquisitionStrategy, + ) { + self.connection_acquisition_strategy = new_strategy; + } + + fn select_connections_to_add( + &mut self, + neighbor_locations: &BTreeMap>, + ) -> anyhow::Result { + let function_span = span!(Level::INFO, "add_connections"); + let _enter = function_span.enter(); + + debug!("Starting to compute density map"); + let density_map = self + .request_density_tracker + .create_density_map(neighbor_locations)?; + debug!("Density map computed successfully"); + + debug!("Attempting to get max density location"); + let max_density_location = match density_map.get_max_density() { + Ok(location) => { + debug!(location = ?location, "Max density location found"); + location + } + Err(e) => { + error!("Failed to get max density location: {:?}", e); + return Err(anyhow!(e)); + } }; - let location_f64 = location_f64.rem_euclid(1.0); // Ensure result is in [0.0, 1.0) - Location::new(location_f64) + + info!("Adding new connection to {:?}", max_density_location); + Ok(TopologyAdjustment::AddConnections(vec![ + max_density_location, + ])) + } + + fn select_connections_to_remove( + &mut self, + exceeded_usage_for_resource_type: &ResourceType, + at_time: Instant, + ) -> TopologyAdjustment { + let function_span = span!(Level::INFO, "remove_connections"); + let _enter = function_span.enter(); + + let mut worst: Option<(PeerKeyLocation, f64)> = None; + + for (source, source_usage) in self + .meter + .get_usage_rates(exceeded_usage_for_resource_type, at_time) + { + let loop_span = span!(Level::DEBUG, "source_loop", ?source); + let _loop_enter = loop_span.enter(); + + event!(Level::DEBUG, "Checking source"); + + if let Some(creation_time) = self.source_creation_times.get(&source) { + if Instant::now().duration_since(*creation_time) <= SOURCE_RAMP_UP_DURATION { + event!(Level::DEBUG, "Source is in ramp-up time, skipping"); + continue; + } + } else { + event!(Level::DEBUG, "No creation time for source, skipping"); + continue; + } + + match source { + AttributionSource::Peer(peer) => { + let peer_span = span!(Level::DEBUG, "peer_processing", ?peer); + let _peer_enter = peer_span.enter(); + + let request_count = + self.outbound_request_counter.get_request_count(&peer) as f64; + + let value_per_usage = request_count / source_usage.per_second(); + + event!( + Level::DEBUG, + request_count = request_count, + usage = source_usage.per_second(), + value_per_usage = value_per_usage + ); + + if let Some((_, worst_value_per_usage)) = worst { + if value_per_usage < worst_value_per_usage { + worst = Some((peer, value_per_usage)); + event!(Level::DEBUG, "Found a worse peer"); + } + } else { + worst = Some((peer, value_per_usage)); + event!(Level::DEBUG, "Setting initial worst peer"); + } + } + _ => { + event!(Level::DEBUG, "Non-peer source, skipping"); + } + } + } + + if let Some((peer, _)) = worst { + event!(Level::INFO, action = "Recommend peer for removal", peer = ?peer); + TopologyAdjustment::RemoveConnections(vec![peer]) + } else { + event!(Level::WARN, "Couldn't find a suitable peer to remove"); + TopologyAdjustment::NoChange + } } } -pub(crate) enum AcquisitionStrategy { +#[derive(PartialEq, Debug, Clone, Copy)] +pub(crate) enum ConnectionAcquisitionStrategy { /// Acquire new connections slowly, be picky Slow, @@ -180,24 +521,37 @@ pub(crate) enum AcquisitionStrategy { #[cfg(test)] mod tests { - use super::TopologyManager; + use super::{Limits, TopologyManager}; + use crate::topology::rate::Rate; use crate::{message::TransactionType, ring::Location}; #[test] - fn test_topology_manager() { + fn test_topology() { const NUM_REQUESTS: usize = 1_000; - let mut topology_manager = TopologyManager::new(Location::new(0.39)); + let mut topology_manager = TopologyManager::new(Limits { + max_upstream_bandwidth: Rate::new_per_second(1000.0), + max_downstream_bandwidth: Rate::new_per_second(1000.0), + min_connections: 5, + max_connections: 200, + }); let mut current_neighbors = std::collections::BTreeMap::new(); // Insert neighbors from 0.0 to 0.9 for i in 0..10 { - current_neighbors.insert(Location::new(i as f64 / 10.0), 0); + current_neighbors.insert(Location::new(i as f64 / 10.0), vec![]); } + let this_peer_location = Location::new(0.39); + // Simulate a bunch of random requests clustered around 0.35 for _ in 0..NUM_REQUESTS { - let requested_location = topology_manager.random_location(); - topology_manager.record_request(requested_location, TransactionType::Get); + let requested_location = random_location(&random_location(&this_peer_location)); + // FIXME: Is PeerKeyLocation unimportant for this test? + topology_manager.record_request( + PeerKeyLocation::random(), + requested_location, + TransactionType::Get, + ); } topology_manager @@ -208,7 +562,9 @@ mod tests { ) .unwrap(); - let best_candidate_location = topology_manager.get_best_candidate_location().unwrap(); + let best_candidate_location = topology_manager + .get_best_candidate_location(&this_peer_location) + .unwrap(); // Should be half way between 0.3 and 0.4 as that is where the most requests were. assert_eq!(best_candidate_location, Location::new(0.35)); @@ -234,4 +590,367 @@ mod tests { // the request epicenter assert_eq!(best_location, Location::new(0.4)); } + + /// Generates a random location that is close to the current peer location with a small + /// world distribution. + fn random_location(this_peer_location: &Location) -> Location { + tracing::debug!("Generating random location"); + let distance = small_world_rand::test_utils::random_link_distance(Distance::new(0.001)); + let location_f64 = if rand::random() { + this_peer_location.as_f64() - distance.as_f64() + } else { + this_peer_location.as_f64() + distance.as_f64() + }; + let location_f64 = location_f64.rem_euclid(1.0); + Location::new(location_f64) + } + + use super::*; + use crate::ring::Distance; + use crate::test_utils::with_tracing; + use std::time::{Duration, Instant}; + + #[test] + fn test_resource_manager_report() { + with_tracing(|| { + // Create a TopologyManager with arbitrary limits + let limits = Limits { + max_upstream_bandwidth: Rate::new_per_second(1000.0), + max_downstream_bandwidth: Rate::new_per_second(1000.0), + max_connections: 200, + min_connections: 5, + }; + let mut resource_manager = TopologyManager::new(limits); + + // Report some usage and test that the total and attributed usage are updated + let attribution = AttributionSource::Peer(PeerKeyLocation::random()); + let now = Instant::now(); + resource_manager.report_resource_usage( + &attribution, + ResourceType::InboundBandwidthBytes, + 100.0, + now, + ); + assert_eq!( + resource_manager + .meter + .attributed_usage_rate(&attribution, &ResourceType::InboundBandwidthBytes, now) + .unwrap() + .per_second(), + 100.0 + ); + }); + } + + #[test] + fn test_remove_connections() { + with_tracing(|| { + let mut resource_manager = setup_topology_manager(1000.0); + let peers = generate_random_peers(5); + // Total bw usage will be way higher than the limit of 1000 + let bw_usage_by_peer = vec![1000, 1100, 1200, 2000, 1600]; + // Report usage from outside the ramp-up time window so it isn't ignored + let report_time = Instant::now() - SOURCE_RAMP_UP_DURATION - Duration::from_secs(30); + report_resource_usage( + &mut resource_manager, + &peers, + &bw_usage_by_peer, + report_time, + ); + let requests_per_peer = vec![20, 19, 18, 9, 9]; + report_outbound_requests(&mut resource_manager, &peers, &requests_per_peer); + let worst_ix = find_worst_peer(&peers, &bw_usage_by_peer, &requests_per_peer); + assert_eq!(worst_ix, 3); + let worst_peer = &peers[worst_ix]; + let mut neighbor_locations = BTreeMap::new(); + for peer in &peers { + neighbor_locations.insert(peer.location.unwrap(), vec![]); + } + + let adjustment = + resource_manager.adjust_topology(&neighbor_locations, &None, Instant::now()); + match adjustment { + TopologyAdjustment::RemoveConnections(peers) => { + assert_eq!(peers.len(), 1); + assert_eq!(peers[0], *worst_peer); + } + _ => panic!("Expected to remove a peer, adjustment was {:?}", adjustment), + } + }); + } + + #[test] + fn test_add_connections() { + with_tracing(|| { + let mut resource_manager = setup_topology_manager(1000.0); + // Generate 5 peers with locations specified in a vec! + let mut peers: Vec = generate_random_peers(5); + let peer_locations: Vec = [0.1, 0.3, 0.5, 0.7, 0.9] + .into_iter() + .map(Location::new) + .collect(); + for (ix, peer) in peers.iter_mut().enumerate() { + peer.location = Some(peer_locations[ix]); + } + + // Total bw usage will be way lower than MINIMUM_DESIRED_RESOURCE_USAGE_PROPORTION, triggering + // the TopologyManager to add a connection + let bw_usage_by_peer = vec![10, 20, 30, 25, 30]; + // Report usage from outside the ramp-up time window so it isn't ignored + let report_time = Instant::now() - SOURCE_RAMP_UP_DURATION - Duration::from_secs(30); + report_resource_usage( + &mut resource_manager, + &peers, + &bw_usage_by_peer, + report_time, + ); + let requests_per_peer = vec![20, 19, 18, 9, 9]; + report_outbound_requests(&mut resource_manager, &peers, &requests_per_peer); + + let mut neighbor_locations = BTreeMap::new(); + for peer in &peers { + neighbor_locations.insert(peer.location.unwrap(), vec![]); + } + + let adjustment = + resource_manager.adjust_topology(&neighbor_locations, &None, Instant::now()); + + match adjustment { + TopologyAdjustment::AddConnections(locations) => { + assert_eq!(locations.len(), 1); + // Location should be between peers[0] and peers[1] because they have the highest + // number of requests per hour for any adjacent peers + assert!(locations[0] >= peers[0].location.unwrap()); + assert!(locations[0] <= peers[1].location.unwrap()); + } + _ => panic!( + "Expected to add a connection, adjustment was {:?}", + adjustment + ), + } + }); + } + + // Test with no adjustment because the usage is within acceptable bounds + #[test] + fn test_no_adjustment() { + with_tracing(|| { + let mut resource_manager = setup_topology_manager(1000.0); + let peers = generate_random_peers(5); + // Total bw usage will be way lower than MINIMUM_DESIRED_RESOURCE_USAGE_PROPORTION, triggering + // the TopologyManager to add a connection + let bw_usage_by_peer = vec![150, 200, 100, 100, 200]; + // Report usage from outside the ramp-up time window so it isn't ignored + let report_time = Instant::now() - SOURCE_RAMP_UP_DURATION - Duration::from_secs(30); + report_resource_usage( + &mut resource_manager, + &peers, + &bw_usage_by_peer, + report_time, + ); + let requests_per_peer = vec![20, 19, 18, 9, 9]; + report_outbound_requests(&mut resource_manager, &peers, &requests_per_peer); + + let mut neighbor_locations = BTreeMap::new(); + for peer in &peers { + neighbor_locations.insert(peer.location.unwrap(), vec![]); + } + + let adjustment = + resource_manager.adjust_topology(&neighbor_locations, &None, report_time); + + match adjustment { + TopologyAdjustment::NoChange => {} + _ => panic!("Expected no adjustment, adjustment was {:?}", adjustment), + } + }); + } + + // Test with no peers + #[test] + fn test_no_peers() { + with_tracing(|| { + let mut resource_manager = setup_topology_manager(1000.0); + let peers = generate_random_peers(0); + // Total bw usage will be way lower than MINIMUM_DESIRED_RESOURCE_USAGE_PROPORTION, triggering + // the TopologyManager to add a connection + let bw_usage_by_peer = vec![]; + // Report usage from outside the ramp-up time window so it isn't ignored + let report_time = Instant::now() - SOURCE_RAMP_UP_DURATION - Duration::from_secs(30); + report_resource_usage( + &mut resource_manager, + &peers, + &bw_usage_by_peer, + report_time, + ); + let requests_per_peer = vec![]; + report_outbound_requests(&mut resource_manager, &peers, &requests_per_peer); + + let mut neighbor_locations = BTreeMap::new(); + for peer in &peers { + neighbor_locations.insert(peer.location.unwrap(), vec![]); + } + + let my_location = Location::random(); + + let adjustment = resource_manager.adjust_topology( + &neighbor_locations, + &Some(my_location), + report_time, + ); + + match adjustment { + TopologyAdjustment::AddConnections(v) => { + assert!(!v.is_empty()); + for location in v { + assert_eq!(location, my_location); + } + } + _ => panic!("Expected AddConnections, but was: {:?}", adjustment), + } + }); + } + + fn setup_topology_manager(max_downstream_rate: f64) -> TopologyManager { + let limits = Limits { + // This won't be used + max_upstream_bandwidth: Rate::new_per_second(100000.0), + max_downstream_bandwidth: Rate::new_per_second(max_downstream_rate), + max_connections: 200, + min_connections: 5, + }; + TopologyManager::new(limits) + } + + fn generate_random_peers(num_peers: usize) -> Vec { + (0..num_peers).map(|_| PeerKeyLocation::random()).collect() + } + + fn report_resource_usage( + resource_manager: &mut TopologyManager, + peers: &[PeerKeyLocation], + bw_usage_by_peer: &[usize], + up_to_time: Instant, + ) { + for (i, peer) in peers.iter().enumerate() { + // Report usage for the last 10 minutes, which is 2X longer than the ramp-up time + for seconds in 1..600 { + let report_time = up_to_time - Duration::from_secs(600 - seconds); + tracing::trace!( + "Reporting {} bytes of inbound bandwidth for peer {:?} at {:?}", + bw_usage_by_peer[i], + peer, + report_time + ); + resource_manager.report_resource_usage( + &AttributionSource::Peer(*peer), + ResourceType::InboundBandwidthBytes, + bw_usage_by_peer[i] as f64, + report_time, + ); + } + } + } + + fn report_outbound_requests( + resource_manager: &mut TopologyManager, + peers: &[PeerKeyLocation], + requests_per_peer: &[usize], + ) { + for (i, requests) in requests_per_peer.iter().enumerate() { + for _ in 0..*requests { + // For simplicity we'll just assume that the target location of the request is the + // neighboring peer's own location + resource_manager.report_outbound_request(peers[i], peers[i].location.unwrap()); + } + } + } + + fn find_worst_peer( + peers: &Vec, + bw_usage_by_peer: &[usize], + requests_per_peer: &[usize], + ) -> usize { + let mut values = vec![]; + for ix in 0..peers.len() { + let peer = peers[ix]; + let value = requests_per_peer[ix] as f64 / bw_usage_by_peer[ix] as f64; + values.push(value); + debug!( + "Peer {:?} has value {}/{} = {}", + peer, requests_per_peer[ix], bw_usage_by_peer[ix], value + ); + } + let mut worst_ix = 0; + for (ix, value) in values.iter().enumerate() { + if *value < values[worst_ix] { + worst_ix = ix; + } + } + worst_ix + } + + #[test] + fn test_update_limits() { + let limits = Limits { + max_upstream_bandwidth: Rate::new_per_second(1000.0), + max_downstream_bandwidth: Rate::new_per_second(1000.0), + max_connections: 200, + min_connections: 5, + }; + let mut topology_manager = TopologyManager::new(limits); + + let new_limits = Limits { + max_upstream_bandwidth: Rate::new_per_second(2000.0), + max_downstream_bandwidth: Rate::new_per_second(2000.0), + max_connections: 200, + min_connections: 5, + }; + topology_manager.update_limits(new_limits); + + assert_eq!( + topology_manager.limits.max_upstream_bandwidth, + Rate::new_per_second(2000.0) + ); + assert_eq!( + topology_manager.limits.max_downstream_bandwidth, + Rate::new_per_second(2000.0) + ); + } +} + +#[derive(Debug, Clone)] +pub(crate) enum TopologyAdjustment { + AddConnections(Vec), + RemoveConnections(Vec), + NoChange, +} + +struct Usage { + total: Rate, + #[allow(unused)] + per_source: HashMap, +} + +pub(crate) struct Limits { + pub max_upstream_bandwidth: Rate, + pub max_downstream_bandwidth: Rate, + pub min_connections: usize, + pub max_connections: usize, +} + +impl Limits { + pub fn get(&self, resource_type: &ResourceType) -> Rate { + match resource_type { + ResourceType::OutboundBandwidthBytes => self.max_upstream_bandwidth, + ResourceType::InboundBandwidthBytes => self.max_downstream_bandwidth, + } + } +} + +#[derive(Debug, Clone)] +struct UsageRates { + #[allow(unused)] + usage_rate_per_type: HashMap, + max_usage_rate: (ResourceType, RateProportion), } diff --git a/crates/core/src/topology/constants.rs b/crates/core/src/topology/constants.rs new file mode 100644 index 000000000..041af30aa --- /dev/null +++ b/crates/core/src/topology/constants.rs @@ -0,0 +1,9 @@ +use std::time::Duration; + +pub(super) const SLOW_CONNECTION_EVALUATOR_WINDOW_DURATION: Duration = Duration::from_secs(5 * 60); +pub(super) const FAST_CONNECTION_EVALUATOR_WINDOW_DURATION: Duration = Duration::from_secs(60); +pub(super) const REQUEST_DENSITY_TRACKER_WINDOW_SIZE: usize = 10_000; +pub(super) const SOURCE_RAMP_UP_DURATION: Duration = Duration::from_secs(5 * 60); +pub(super) const OUTBOUND_REQUEST_COUNTER_WINDOW_SIZE: usize = 10000; +pub(super) const MINIMUM_DESIRED_RESOURCE_USAGE_PROPORTION: f64 = 0.5; +pub(super) const MAXIMUM_DESIRED_RESOURCE_USAGE_PROPORTION: f64 = 0.9; diff --git a/crates/core/src/topology/meter.rs b/crates/core/src/topology/meter.rs new file mode 100644 index 000000000..8cfa60729 --- /dev/null +++ b/crates/core/src/topology/meter.rs @@ -0,0 +1,354 @@ +use std::collections::HashMap; +use std::hash::Hash; +use std::time::{Duration, Instant}; + +use dashmap::DashMap; +use freenet_stdlib::prelude::*; + +use crate::ring::PeerKeyLocation; +use crate::topology::rate::Rate; + +use super::running_average::RunningAverage; + +// Default usage is assumed to be the 50th percentile of usage for the resource. +const DEFAULT_USAGE_PERCENTILE: f64 = 0.5; + +// Recache the estimated usage rate this often +const ESTIMATED_USAGE_RATE_CACHE_TIME: Duration = Duration::from_secs(60); + +/// A structure that keeps track of the usage of dynamic resources which are consumed over time. +/// It provides methods to report and query resource usage, both total and attributed to specific +/// sources. +pub(crate) struct Meter { + attribution_meters: AttributionMeters, + running_average_window_size: usize, + cached_estimated_usage_rate: DashMap, +} + +impl Meter { + /// Creates a new `Meter`. + pub fn new_with_window_size(running_average_window_size: usize) -> Self { + Meter { + attribution_meters: AttributionMeters::new(), + running_average_window_size, + cached_estimated_usage_rate: DashMap::new(), + } + } + + /// The measured usage rate for a resource attributed to a specific source. + pub(crate) fn attributed_usage_rate( + &self, + attribution: &AttributionSource, + resource: &ResourceType, + at_time: Instant, + ) -> Option { + match self.attribution_meters.get(attribution) { + Some(attribution_meters) => { + match attribution_meters.map.get(resource) { + Some(meter) => { + // Get the current measurement value + meter.get_rate_at_time(at_time) + } + None => Some(Rate::new(0.0, Duration::from_secs(1))), // No meter found for the given resource + } + } + None => None, // No AttributionMeters found for the given attribution + } + } + + /// Returns the estimated usage rate for a resource of a given type. + /// + /// This function uses a percentile defined by `DEFAULT_USAGE_PERCENTILE` to estimate the usage rate + /// for resources with unknown usage. It caches the estimated rates and refreshes them every + /// `ESTIMATED_USAGE_RATE_CACHE_TIME` duration to avoid frequent recalculations. + /// + /// # Arguments + /// + /// * `resource` - A reference to the type of resource for which the usage rate is estimated. + /// * `now` - The current `Instant` used to determine if the cached value is still valid. + /// + /// # Returns + /// + /// An `Option` which is `Some(rate)` if an estimated rate is available, or `None` if it can't be determined. + pub(crate) fn get_adjusted_usage_rate( + &mut self, + resource: &ResourceType, + at_time: Instant, + ) -> Option { + if let Some(cached) = self.cached_estimated_usage_rate.get(resource) { + let (cached_rate, cached_time) = cached.value(); + if at_time - *cached_time <= ESTIMATED_USAGE_RATE_CACHE_TIME { + return Some(*cached_rate); + } + } + + match self.calculate_estimated_usage_rate(resource, at_time) { + Some(estimated_usage_rate) => { + self.cached_estimated_usage_rate + .insert(*resource, (estimated_usage_rate, at_time)); + Some(estimated_usage_rate) + } + None => None, + } + } + + /// Returns a HashMap of AttributionSource to Rate of the usage rate for + /// each attribution source. This does not adjust the usage rate for sources + /// that are ramping up. + pub(crate) fn get_usage_rates( + &self, + resource: &ResourceType, + at_time: Instant, + ) -> HashMap { + let mut rates = HashMap::new(); + + for attribution_meter in self.attribution_meters.iter() { + let attribution = attribution_meter.key(); + if let Some(rate) = self.attributed_usage_rate(attribution, resource, at_time) { + rates.insert(attribution.clone(), rate); + } + } + + rates + } + + /// Estimates the usage rate for a given resource type based on existing data. + /// + /// This function calculates the estimated usage rate by taking the 50th percentile value (or another + /// specified percentile defined by [DEFAULT_USAGE_PERCENTILE]) from the set of known rates for the + /// specified resource type. It disregards resources with no known rate (which may leader to + /// higher estimates). + /// + /// # Arguments + /// + /// * `resource` - A reference to the resource type for which the usage rate is to be estimated. + /// + /// # Returns + /// + /// An `Option` which is `Some(rate)` if an estimated rate can be determined from available data, + /// or `None` if no data is available for the given resource type. + /// + /// # Panics + /// + /// This function may panic if `DEFAULT_USAGE_PERCENTILE` is set to an invalid value that is not within + /// the range [0.0, 1.0]. + fn calculate_estimated_usage_rate( + &mut self, + resource: &ResourceType, + at_time: Instant, + ) -> Option { + let rates: Vec = self + .attribution_meters + .iter() + // Filter out resources with no Rate and collect their rates + .filter_map(|t| { + t.map + .get(resource) + .and_then(|m| m.get_rate_at_time(at_time)) + }) + .collect(); + + if rates.is_empty() { + return None; + } + + // Sort the collected rates + let mut sorted_rates = rates; + sorted_rates.sort_unstable(); // Using sort_unstable for potentially better performance + + // Calculate the index for the estimated usage rate + let percentile_index = + (DEFAULT_USAGE_PERCENTILE * sorted_rates.len() as f64).round() as usize; + let estimated_index = percentile_index.min(sorted_rates.len().saturating_sub(1)); + + sorted_rates.get(estimated_index).cloned() + } + + /// Report the use of a resource. This should be done in the lowest-level + /// functions that consume the resource, taking an AttributionMeter + /// as a parameter. + #[allow(dead_code)] // fixme: use this + pub(crate) fn report( + &mut self, + attribution: &AttributionSource, + resource: ResourceType, + value: f64, + at_time: Instant, + ) { + // Report the usage for a specific attribution + let resource_map = self + .attribution_meters + .entry(attribution.clone()) + .or_insert_with(ResourceTotals::new); + let mut resource_value = resource_map + .map + .entry(resource) + .or_insert_with(|| RunningAverage::new(self.running_average_window_size)); + resource_value.insert_with_time(at_time, value); + } +} + +#[allow(dead_code)] // todo use this +#[derive(Eq, Hash, PartialEq, Clone, Debug)] +pub(crate) enum AttributionSource { + Peer(PeerKeyLocation), + Delegate(DelegateKey), +} + +#[derive(Eq, Hash, PartialEq, Clone, Copy, Debug)] +pub(crate) enum ResourceType { + InboundBandwidthBytes, + OutboundBandwidthBytes, +} + +impl ResourceType { + pub(crate) fn all() -> [ResourceType; 2] { + [ + ResourceType::InboundBandwidthBytes, + ResourceType::OutboundBandwidthBytes, + ] + } +} + +type AttributionMeters = DashMap; + +/// A structure that holds running averages of resource usage for different resource types. +struct ResourceTotals { + pub map: DashMap, +} + +impl ResourceTotals { + fn new() -> Self { + ResourceTotals { + map: DashMap::new(), + } + } +} + +// Tests +#[cfg(test)] +mod tests { + use crate::DynError; + + use super::*; + + #[test] + fn test_empty_meter() { + let meter = Meter::new_with_window_size(100); + + assert!(meter + .attributed_usage_rate( + &AttributionSource::Peer(PeerKeyLocation::random()), + &ResourceType::InboundBandwidthBytes, + Instant::now(), + ) + .is_none()); + assert!(meter.attribution_meters.is_empty()); + } + + #[test] + fn test_meter_attributed_usage() { + let mut meter = Meter::new_with_window_size(100); + + // Test that the attributed usage is 0.0 for all resources + let attribution = AttributionSource::Peer(PeerKeyLocation::random()); + assert!(meter + .attributed_usage_rate( + &attribution, + &ResourceType::InboundBandwidthBytes, + Instant::now() + ) + .is_none()); + assert!(meter + .attributed_usage_rate( + &attribution, + &ResourceType::OutboundBandwidthBytes, + Instant::now() + ) + .is_none()); + + // Report some usage and test that the attributed usage is updated + meter.report( + &attribution, + ResourceType::InboundBandwidthBytes, + 100.0, + Instant::now(), + ); + assert_eq!( + meter + .attributed_usage_rate( + &attribution, + &ResourceType::InboundBandwidthBytes, + Instant::now() + ) + .unwrap() + .per_second(), + 100.0 + ); + } + + #[test] + fn test_meter_report() -> Result<(), DynError> { + let mut meter = Meter::new_with_window_size(100); + + // Report some usage and test that the total and attributed usage are updated + let attribution = AttributionSource::Peer(PeerKeyLocation::random()); + meter.report( + &attribution, + ResourceType::InboundBandwidthBytes, + 100.0, + Instant::now(), + ); + assert_eq!( + meter + .attributed_usage_rate( + &attribution, + &ResourceType::InboundBandwidthBytes, + Instant::now() + ) + .unwrap() + .per_second(), + 100.0 + ); + + // Report more usage and test that the total and attributed usage are updated + meter.report( + &attribution, + ResourceType::InboundBandwidthBytes, + 200.0, + Instant::now(), + ); + assert_eq!( + meter + .attributed_usage_rate( + &attribution, + &ResourceType::InboundBandwidthBytes, + Instant::now() + ) + .unwrap() + .per_second(), + 300.0 + ); + + // Report usage for a different attribution and test that the total and attributed usage are updated + let other_attribution = AttributionSource::Peer(PeerKeyLocation::random()); + meter.report( + &other_attribution, + ResourceType::InboundBandwidthBytes, + 150.0, + Instant::now(), + ); + assert_eq!( + meter + .attributed_usage_rate( + &other_attribution, + &ResourceType::InboundBandwidthBytes, + Instant::now() + ) + .unwrap() + .per_second(), + 150.0 + ); + Ok(()) + } +} diff --git a/crates/core/src/topology/outbound_request_counter.rs b/crates/core/src/topology/outbound_request_counter.rs new file mode 100644 index 000000000..adc73fef1 --- /dev/null +++ b/crates/core/src/topology/outbound_request_counter.rs @@ -0,0 +1,42 @@ +use crate::ring::PeerKeyLocation; +use std::collections::{HashMap, VecDeque}; + +pub(crate) struct OutboundRequestCounter { + window_size: usize, + request_window: VecDeque, + counts_by_peer: HashMap, + total_count: usize, +} + +impl OutboundRequestCounter { + pub(crate) fn new(window_size: usize) -> Self { + OutboundRequestCounter { + window_size, + request_window: VecDeque::new(), + counts_by_peer: HashMap::new(), + total_count: 0, + } + } + + pub(crate) fn record_request(&mut self, peer: PeerKeyLocation) { + self.request_window.push_back(peer); + self.counts_by_peer + .entry(peer) + .and_modify(|count| *count += 1) + .or_insert(1); + self.total_count += 1; + while self.request_window.len() > self.window_size { + let removed_request = self.request_window.pop_front().unwrap(); + let count = self.counts_by_peer.get_mut(&removed_request).unwrap(); + *count -= 1; + self.total_count -= 1; + if *count == 0 { + self.counts_by_peer.remove(&removed_request); + } + } + } + + pub(crate) fn get_request_count(&self, peer: &PeerKeyLocation) -> usize { + self.counts_by_peer.get(peer).copied().unwrap_or(0) + } +} diff --git a/crates/core/src/topology/rate.rs b/crates/core/src/topology/rate.rs new file mode 100644 index 000000000..091f26bf6 --- /dev/null +++ b/crates/core/src/topology/rate.rs @@ -0,0 +1,143 @@ +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; +use std::time::Duration; + +#[derive(Debug, PartialEq, Clone, Copy, Serialize, Deserialize)] +pub struct Rate { + value: f64, +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub struct RateProportion { + value: f64, +} + +impl Rate { + pub fn new(value: f64, divisor: Duration) -> Self { + debug_assert!(value >= 0.0, "Value must be non-negative"); + let divisor_as_secs = divisor.as_secs_f64(); + debug_assert!(divisor_as_secs > 0.0, "Divisor must be greater than 0"); + Rate { + value: value / divisor_as_secs, + } + } + + pub const fn new_per_second(value: f64) -> Self { + Rate { value } + } + + pub const fn per_second(&self) -> f64 { + self.value + } + + pub fn proportion_of(&self, other: &Rate) -> RateProportion { + debug_assert!( + other.value > 0.0, + "Cannot calculate proportion of zero rate" + ); + RateProportion { + value: self.value / other.value, + } + } +} + +impl Eq for Rate {} + +impl Ord for Rate { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap_or(Ordering::Equal) + } +} + +impl PartialOrd for Rate { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialOrd for RateProportion { + fn partial_cmp(&self, other: &Self) -> Option { + self.value.partial_cmp(&other.value) + } +} + +impl std::ops::Add for Rate { + type Output = Self; + + fn add(self, other: Self) -> Self { + Rate { + value: self.value + other.value, + } + } +} + +impl std::ops::Sub for Rate { + type Output = Self; + + fn sub(self, other: Self) -> Self { + Rate { + value: self.value - other.value, + } + } +} + +impl std::ops::AddAssign for Rate { + fn add_assign(&mut self, other: Self) { + self.value += other.value; + } +} + +impl RateProportion { + pub(crate) fn new(value: f64) -> Self { + debug_assert!( + (0.0..=1.0).contains(&value), + "Proportion must be between 0 and 1" + ); + RateProportion { value } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rate() { + let rate = Rate::new(100.0, Duration::from_secs(2)); + assert_eq!(rate.per_second(), 50.0); + } + + // Test that the rate is calculated correctly when the divisor is 1 second + #[test] + fn test_rate_per_second() { + let rate = Rate::new_per_second(100.0); + assert_eq!(rate.per_second(), 100.0); + } + + // Verify that adding two rates works + #[test] + fn test_add() { + let rate1 = Rate::new(100.0, Duration::from_secs(2)); + let rate2 = Rate::new(200.0, Duration::from_secs(2)); + let rate3 = rate1 + rate2; + assert_eq!(rate3.per_second(), 150.0); + } + + // Test that += works + #[test] + fn test_add_assign() { + let mut rate1 = Rate::new(100.0, Duration::from_secs(2)); + let rate2 = Rate::new(200.0, Duration::from_secs(2)); + rate1 += rate2; + assert_eq!(rate1.per_second(), 150.0); + } + + // Test that subtraction works + #[test] + fn test_sub() { + let rate1 = Rate::new(100.0, Duration::from_secs(2)); + let rate2 = Rate::new(200.0, Duration::from_secs(2)); + let rate3 = rate2 - rate1; + assert_eq!(rate3.per_second(), 50.0); + } +} diff --git a/crates/core/src/topology/request_density_tracker.rs b/crates/core/src/topology/request_density_tracker.rs index a74abf3ec..42223c21c 100644 --- a/crates/core/src/topology/request_density_tracker.rs +++ b/crates/core/src/topology/request_density_tracker.rs @@ -1,11 +1,11 @@ -use crate::ring::Location; +use crate::ring::{Connection, Location}; use std::collections::{BTreeMap, VecDeque}; use thiserror::Error; /// Tracks requests sent by a node to its neighbors and creates a density map, which /// is useful for determining which new neighbors to connect to based on their /// location. -pub(super) struct RequestDensityTracker { +pub(crate) struct RequestDensityTracker { /// Amount of requests done to an specific location. request_locations: BTreeMap, /// Request locations sorted by order of execution. @@ -15,7 +15,7 @@ pub(super) struct RequestDensityTracker { } impl RequestDensityTracker { - pub fn new(window_size: usize) -> Self { + pub(crate) fn new(window_size: usize) -> Self { Self { request_locations: BTreeMap::new(), request_list: VecDeque::with_capacity(window_size), @@ -24,7 +24,7 @@ impl RequestDensityTracker { } } - pub fn sample(&mut self, value: Location) { + pub(crate) fn sample(&mut self, value: Location) { self.samples += 1; self.request_list.push_back(value); @@ -42,31 +42,30 @@ impl RequestDensityTracker { } } - pub fn create_density_map( + pub(crate) fn create_density_map( &self, - neighbors: &BTreeMap, + neighbor_locations: &BTreeMap>, ) -> Result { - debug_assert!(!neighbors.is_empty()); - if neighbors.is_empty() { + if neighbor_locations.is_empty() { return Err(DensityMapError::EmptyNeighbors); } let mut neighbor_request_counts = BTreeMap::new(); for (sample_location, sample_count) in self.request_locations.iter() { - let previous_neighbor = neighbors + let previous_neighbor = neighbor_locations .range(..*sample_location) .next_back() - .or_else(|| neighbors.iter().next_back()); - let next_neighbor = neighbors + .or_else(|| neighbor_locations.iter().next_back()); + let next_neighbor = neighbor_locations .range(*sample_location..) .next() - .or_else(|| neighbors.iter().next()); + .or_else(|| neighbor_locations.iter().next()); match (previous_neighbor, next_neighbor) { (Some((previous_neighbor_location, _)), Some((next_neighbor_location, _))) => { - if sample_location.distance(*previous_neighbor_location) - < sample_location.distance(*next_neighbor_location) + if sample_location.distance(previous_neighbor_location) + < sample_location.distance(next_neighbor_location) { *neighbor_request_counts .entry(*previous_neighbor_location) @@ -77,7 +76,6 @@ impl RequestDensityTracker { .or_insert(0) += sample_count; } } - // The None cases have been removed as they should not occur given the new logic _ => unreachable!( "This shouldn't be possible given that we verify neighbors is not empty" ), @@ -90,7 +88,7 @@ impl RequestDensityTracker { } } -pub(super) struct DensityMap { +pub(crate) struct DensityMap { neighbor_request_counts: BTreeMap, } @@ -138,7 +136,10 @@ impl DensityMap { } pub fn get_max_density(&self) -> Result { + tracing::debug!("get_max_density called"); + if self.neighbor_request_counts.is_empty() { + tracing::warn!("No neighbors to get max density from"); return Err(DensityMapError::EmptyNeighbors); } @@ -147,6 +148,8 @@ impl DensityMap { let mut max_density_location = Location::new(0.0); let mut max_density = 0; + tracing::debug!("Starting to iterate over neighbor pairs"); + for ( (previous_neighbor_location, previous_neighbor_count), (next_neighbor_location, next_neighbor_count), @@ -155,8 +158,22 @@ impl DensityMap { .iter() .zip(self.neighbor_request_counts.iter().skip(1)) { + // tracing span with location of first and last neighbor locations + let span = tracing::debug_span!( + "neighbor_pair", + previous_neighbor_location = previous_neighbor_location.as_f64(), + next_neighbor_location = next_neighbor_location.as_f64() + ); + let _enter = span.enter(); let combined_count = previous_neighbor_count + next_neighbor_count; + tracing::debug!("Combined count for neighbor pair: {}", combined_count); + if combined_count > max_density { + tracing::debug!( + "New max density found: {} at location {:?}", + combined_count, + max_density_location + ); max_density = combined_count; max_density_location = Location::new( (previous_neighbor_location.as_f64() + next_neighbor_location.as_f64()) / 2.0, @@ -164,6 +181,8 @@ impl DensityMap { } } + tracing::debug!("Checking first and last neighbors"); + // We need to also check the first and last neighbors as locations are circular let first_neighbor = self.neighbor_request_counts.iter().next(); let last_neighbor = self.neighbor_request_counts.iter().next_back(); @@ -173,6 +192,11 @@ impl DensityMap { ) = (first_neighbor, last_neighbor) { let combined_count = first_neighbor_count + last_neighbor_count; + tracing::debug!( + "Combined count for first and last neighbor: {}", + combined_count + ); + if combined_count > max_density { // max_density = combined_count; Not needed as this is the last check let distance = first_neighbor_location.distance(*last_neighbor_location); @@ -181,9 +205,14 @@ impl DensityMap { mp += 1.0; } max_density_location = Location::new(mp); + tracing::debug!( + "New max density found at the edge: location {:?}", + max_density_location + ); } } + tracing::debug!("Returning max density location: {:?}", max_density_location); Ok(max_density_location) } } @@ -201,9 +230,9 @@ impl CachedDensityMap { pub fn set( &mut self, tracker: &RequestDensityTracker, - current_neighbors: &BTreeMap, + neighbor_locations: &BTreeMap>, ) -> Result<(), DensityMapError> { - let density_map = tracker.create_density_map(current_neighbors)?; + let density_map = tracker.create_density_map(neighbor_locations)?; self.density_map = Some(density_map); Ok(()) } @@ -222,9 +251,22 @@ pub(crate) enum DensityMapError { #[cfg(test)] mod tests { use super::*; + use std::sync::RwLock; #[test] fn test_create_density_map() { + let neighbors = RwLock::new(BTreeMap::new()); + neighbors + .write() + .unwrap() + .insert(Location::new(0.2), vec![]); + neighbors + .write() + .unwrap() + .insert(Location::new(0.6), vec![]); + + let neighbors = neighbors.read(); + let mut sw = RequestDensityTracker::new(5); sw.sample(Location::new(0.21)); sw.sample(Location::new(0.22)); @@ -232,11 +274,7 @@ mod tests { sw.sample(Location::new(0.61)); sw.sample(Location::new(0.62)); - let mut neighbors = BTreeMap::new(); - neighbors.insert(Location::new(0.2), 1); - neighbors.insert(Location::new(0.6), 1); - - let result = sw.create_density_map(&neighbors); + let result = sw.create_density_map(&neighbors.unwrap()); assert!(result.is_ok()); let result = result.unwrap(); assert_eq!( @@ -259,8 +297,8 @@ mod tests { sw.sample(Location::new(0.62)); let mut neighbors = BTreeMap::new(); - neighbors.insert(Location::new(0.6), 1); - neighbors.insert(Location::new(0.9), 1); + neighbors.insert(Location::new(0.6), vec![]); + neighbors.insert(Location::new(0.9), vec![]); let result = sw.create_density_map(&neighbors); assert!(result.is_ok()); @@ -285,8 +323,8 @@ mod tests { sw.sample(Location::new(0.60)); let mut neighbors = BTreeMap::new(); - neighbors.insert(Location::new(0.2), 1); - neighbors.insert(Location::new(0.6), 1); + neighbors.insert(Location::new(0.2), vec![]); + neighbors.insert(Location::new(0.6), vec![]); let result = sw.create_density_map(&neighbors); assert!(result.is_ok()); @@ -321,8 +359,8 @@ mod tests { sw.sample(Location::new(0.62)); let mut neighbors = BTreeMap::new(); - neighbors.insert(Location::new(0.2), 1); - neighbors.insert(Location::new(0.6), 1); + neighbors.insert(Location::new(0.2), vec![]); + neighbors.insert(Location::new(0.6), vec![]); let result = sw.create_density_map(&neighbors); assert!(result.is_ok()); diff --git a/crates/core/src/resources/running_average.rs b/crates/core/src/topology/running_average.rs similarity index 87% rename from crates/core/src/resources/running_average.rs rename to crates/core/src/topology/running_average.rs index 8bc247186..5e1e6b79f 100644 --- a/crates/core/src/resources/running_average.rs +++ b/crates/core/src/topology/running_average.rs @@ -1,9 +1,9 @@ -use crate::resources::rate::Rate; +use crate::topology::rate::Rate; use std::time::Duration; use std::{collections::VecDeque, time::Instant}; #[derive(Clone, Debug)] -pub(super) struct RunningAverage { +pub(crate) struct RunningAverage { max_samples: usize, samples: VecDeque<(Instant, f64)>, sum_samples: f64, @@ -20,14 +20,10 @@ impl RunningAverage { } } - pub fn insert(&mut self, value: f64) { - self.insert_with_time(Instant::now(), value); - } - - fn insert_with_time(&mut self, now: Instant, value: f64) { + pub(crate) fn insert_with_time(&mut self, now: Instant, value: f64) { // Require that now is after the last sample time if let Some((last_sample_time, _)) = self.samples.back() { - assert!(now >= *last_sample_time); + debug_assert!(now >= *last_sample_time); } self.total_sample_count += 1; if self.samples.len() < self.max_samples { @@ -39,11 +35,7 @@ impl RunningAverage { } } - pub fn get_rate(&self) -> Option { - self.get_rate_at_time(Instant::now()) - } - - fn get_rate_at_time(&self, now: Instant) -> Option { + pub(crate) fn get_rate_at_time(&self, now: Instant) -> Option { if self.samples.is_empty() { return None; } @@ -53,10 +45,6 @@ impl RunningAverage { let divisor = sample_duration.max(MINIMUM_TIME_WINDOW); Some(Rate::new(self.sum_samples, divisor)) } - - pub fn total_sample_count(&self) -> usize { - self.total_sample_count - } } #[cfg(test)] diff --git a/crates/core/src/topology/small_world_rand.rs b/crates/core/src/topology/small_world_rand.rs index 5f5e1c005..787f98374 100644 --- a/crates/core/src/topology/small_world_rand.rs +++ b/crates/core/src/topology/small_world_rand.rs @@ -1,24 +1,28 @@ -use rand::Rng; +#[cfg(test)] +pub(super) mod test_utils { + use rand::Rng; -use crate::ring::Distance; + use crate::ring::Distance; -// Function to generate a random link distance based on Kleinberg's d^{-1} distribution -pub(super) fn random_link_distance(d_min: Distance) -> Distance { - let d_max = 0.5; + // Function to generate a random link distance based on Kleinberg's d^{-1} distribution + pub(in crate::topology) fn random_link_distance(d_min: Distance) -> Distance { + let d_max = 0.5; - // Generate a uniform random number between 0 and 1 - let u: f64 = rand::thread_rng().gen_range(0.0..1.0); + // Generate a uniform random number between 0 and 1 + let u: f64 = rand::thread_rng().gen_range(0.0..1.0); - // Correct Inverse CDF: F^{-1}(u) = d_min * (d_max / d_min).powf(u) - let d = d_min.as_f64() * (d_max / d_min.as_f64()).powf(u); + // Correct Inverse CDF: F^{-1}(u) = d_min * (d_max / d_min).powf(u) + let d = d_min.as_f64() * (d_max / d_min.as_f64()).powf(u); - Distance::new(d) + Distance::new(d) + } } #[cfg(test)] mod tests { + use crate::ring::Distance; - use super::*; + use super::test_utils::*; use statrs::distribution::*; #[test] diff --git a/crates/core/src/wasm_runtime/delegate.rs b/crates/core/src/wasm_runtime/delegate.rs index 2eb853391..bdf0041ac 100644 --- a/crates/core/src/wasm_runtime/delegate.rs +++ b/crates/core/src/wasm_runtime/delegate.rs @@ -471,7 +471,7 @@ mod test { assert_eq!(outbound.len(), 1); assert!(matches!( - outbound.get(0), + outbound.first(), Some(OutboundDelegateMsg::ApplicationMessage(msg)) if *msg.payload == expected_payload )); @@ -487,7 +487,7 @@ mod test { bincode::serialize(&OutboundAppMessage::MessageSigned(vec![4, 5, 2])).unwrap(); assert_eq!(outbound.len(), 1); assert!(matches!( - outbound.get(0), + outbound.first(), Some(OutboundDelegateMsg::ApplicationMessage(msg)) if *msg.payload == expected_payload )); std::mem::drop(temp_dir);