From a77eda6028facbf718e73b23dbf062eadd887f38 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Fri, 26 Apr 2024 16:51:42 +0400 Subject: [PATCH 1/6] feat(webui): display connections user agent (#1019) Description --- Display user agent in connections pages for VN and indexer Update bindings Fix displayDuration function Motivation and Context --- ![image](https://github.com/tari-project/tari-dan/assets/1057902/5976ee94-fcee-4b80-b3c7-743615c8b9e9) How Has This Been Tested? --- Manually What process can a PR reviewer use to test or verify this change? --- Connections in web uis Breaking Changes --- - [x] None - [ ] Requires data directory to be deleted - [ ] Other - Please specify --- .../tari_indexer/src/json_rpc/handlers.rs | 1 + .../src/routes/VN/Components/Connections.tsx | 30 ++++--------------- .../tari_indexer_web_ui/src/utils/helpers.tsx | 27 +++++++++++++++++ .../src/json_rpc/handlers.rs | 1 + .../src/routes/VN/Components/Connections.tsx | 28 ++++------------- .../src/utils/helpers.tsx | 8 +++-- .../types/tari-indexer-client/Connection.ts | 1 + .../types/validator-node-client/Connection.ts | 1 + clients/tari_indexer_client/src/types.rs | 3 +- clients/validator_node_client/src/types.rs | 3 +- networking/core/src/connection.rs | 6 +++- networking/core/src/worker.rs | 15 ++++++++++ 12 files changed, 71 insertions(+), 53 deletions(-) diff --git a/applications/tari_indexer/src/json_rpc/handlers.rs b/applications/tari_indexer/src/json_rpc/handlers.rs index 9bba2e049..2190368c6 100644 --- a/applications/tari_indexer/src/json_rpc/handlers.rs +++ b/applications/tari_indexer/src/json_rpc/handlers.rs @@ -256,6 +256,7 @@ impl JsonRpcHandlers { }, age: conn.age(), ping_latency: conn.ping_latency, + user_agent: conn.user_agent, }) .collect(); diff --git a/applications/tari_indexer_web_ui/src/routes/VN/Components/Connections.tsx b/applications/tari_indexer_web_ui/src/routes/VN/Components/Connections.tsx index 8d9accc40..0a9afde94 100644 --- a/applications/tari_indexer_web_ui/src/routes/VN/Components/Connections.tsx +++ b/applications/tari_indexer_web_ui/src/routes/VN/Components/Connections.tsx @@ -37,11 +37,8 @@ import { Form } from "react-router-dom"; import Fade from "@mui/material/Fade"; import CopyToClipboard from "../../../Components/CopyToClipboard"; import type { Connection } from "@tariproject/typescript-bindings/tari-indexer-client"; +import { displayDuration } from "../../../utils/helpers"; -interface Duration { - secs: number; - nanos: number; -} const useInterval = (fn: () => Promise, ms: number) => { const timeout = useRef(); @@ -132,16 +129,17 @@ function Connections() { - Peer id + Peer ID Address Age Direction Latency + User Agent {connections && - connections.map(({ connection_id, address, age, direction, peer_id, ping_latency }) => ( + connections.map(({ connection_id, address, age, direction, peer_id, ping_latency, user_agent }) => ( {peer_id ? shortenString(peer_id) : "--"} @@ -151,6 +149,7 @@ function Connections() { {displayDuration(age)} {direction} {ping_latency ? displayDuration(ping_latency) : "--"} + {user_agent ? user_agent.replace(/^\/tari\//, "") : "--"} ))} @@ -159,23 +158,4 @@ function Connections() { ); } -function displayDuration(duration: Duration) { - if (duration.secs === 0) { - if (duration.nanos > 1000000) { - return `${(duration.nanos / 1000000).toFixed(2)}ms`; - } - if (duration.nanos > 1000) { - return `${(duration.nanos / 1000).toFixed(2)}µs`; - } - return `${duration.nanos / 1000}ns`; - } - if (duration.secs > 60 * 60) { - return `${(duration.secs / 60 / 60).toFixed(0)}h${(duration.secs / 60).toFixed(0)}m`; - } - if (duration.secs > 60) { - return `${(duration.secs / 60).toFixed(0)}m${(duration.secs % 60).toFixed(0)}s`; - } - return `${duration.secs}.${(duration.nanos / 1000000).toFixed(0)}s`; -} - export default Connections; diff --git a/applications/tari_indexer_web_ui/src/utils/helpers.tsx b/applications/tari_indexer_web_ui/src/utils/helpers.tsx index 96a0a9ecb..483cbbed4 100644 --- a/applications/tari_indexer_web_ui/src/utils/helpers.tsx +++ b/applications/tari_indexer_web_ui/src/utils/helpers.tsx @@ -58,4 +58,31 @@ const renderJson = (json: any) => { } }; +export interface Duration { + secs: number; + nanos: number; +} + +export function displayDuration(duration: Duration) { + if (duration.secs === 0) { + if (duration.nanos > 1000000) { + return `${(duration.nanos / 1000000).toFixed(2)}ms`; + } + if (duration.nanos > 1000) { + return `${(duration.nanos / 1000).toFixed(2)}µs`; + } + return `${duration.nanos}ns`; + } + if (duration.secs >= 60 * 60) { + const minutes_secs = duration.secs - Math.floor(duration.secs / 60 / 60) * 60 * 60; + return `${(duration.secs / 60 / 60).toFixed(0)}h${Math.floor(minutes_secs / 60)}m`; + } + if (duration.secs >= 60) { + const secs = duration.secs - Math.floor(duration.secs / 60) * 60; + return `${(duration.secs / 60).toFixed(0)}m${secs.toFixed(0)}s`; + } + return `${duration.secs}s`; +} + + export { renderJson }; diff --git a/applications/tari_validator_node/src/json_rpc/handlers.rs b/applications/tari_validator_node/src/json_rpc/handlers.rs index 1bf6230e3..1ce31c3e8 100644 --- a/applications/tari_validator_node/src/json_rpc/handlers.rs +++ b/applications/tari_validator_node/src/json_rpc/handlers.rs @@ -528,6 +528,7 @@ impl JsonRpcHandlers { }, age: conn.age(), ping_latency: conn.ping_latency, + user_agent: conn.user_agent, }) .collect(); diff --git a/applications/tari_validator_node_web_ui/src/routes/VN/Components/Connections.tsx b/applications/tari_validator_node_web_ui/src/routes/VN/Components/Connections.tsx index 4feeddc95..df7ad668b 100644 --- a/applications/tari_validator_node_web_ui/src/routes/VN/Components/Connections.tsx +++ b/applications/tari_validator_node_web_ui/src/routes/VN/Components/Connections.tsx @@ -37,6 +37,7 @@ import { Form } from "react-router-dom"; import Fade from "@mui/material/Fade"; import CopyToClipboard from "../../../Components/CopyToClipboard"; import type { Connection } from "@tariproject/typescript-bindings/validator-node-client"; +import { displayDuration } from "../../../utils/helpers"; const useInterval = (fn: () => Promise, ms: number) => { const timeout = useRef(); @@ -128,18 +129,19 @@ function Connections() {
- Peer id + Peer ID Address Age Direction Latency + User Agent {connections && - connections.map(({ connection_id, address, age, direction, peer_id, ping_latency }) => ( + connections.map(({ connection_id, address, age, direction, peer_id, ping_latency, user_agent }) => ( - + {peer_id ? shortenString(peer_id) : "--"} @@ -147,6 +149,7 @@ function Connections() { {displayDuration(age)} {direction} {ping_latency ? displayDuration(ping_latency) : "--"} + {user_agent ? user_agent.replace(/^\/tari\//, "") : "--"} ))} @@ -156,23 +159,4 @@ function Connections() { ); } -function displayDuration(duration: { secs: number; nanos: number }) { - if (duration.secs === 0) { - if (duration.nanos > 1000000) { - return `${(duration.nanos / 1000000).toFixed(2)}ms`; - } - if (duration.nanos > 1000) { - return `${(duration.nanos / 1000).toFixed(2)}µs`; - } - return `${duration.nanos / 1000}ns`; - } - if (duration.secs > 60 * 60) { - return `${(duration.secs / 60 / 60).toFixed(0)}h${(duration.secs / 60).toFixed(0)}m`; - } - if (duration.secs > 60) { - return `${(duration.secs / 60).toFixed(0)}m${(duration.secs % 60).toFixed(0)}s`; - } - return `${duration.secs}.${(duration.nanos / 1000000).toFixed(0)}s`; -} - export default Connections; diff --git a/applications/tari_validator_node_web_ui/src/utils/helpers.tsx b/applications/tari_validator_node_web_ui/src/utils/helpers.tsx index 03ca2cdf4..da81c4d0b 100644 --- a/applications/tari_validator_node_web_ui/src/utils/helpers.tsx +++ b/applications/tari_validator_node_web_ui/src/utils/helpers.tsx @@ -121,10 +121,12 @@ export function displayDuration(duration: Duration) { return `${duration.nanos}ns`; } if (duration.secs >= 60 * 60) { - return `${(duration.secs / 60 / 60).toFixed(0)}h${(duration.secs / 60).toFixed(0)}m`; + const minutes_secs = duration.secs - Math.floor(duration.secs / 60 / 60) * 60 * 60; + return `${(duration.secs / 60 / 60).toFixed(0)}h${Math.floor(minutes_secs / 60)}m`; } if (duration.secs >= 60) { - return `${(duration.secs / 60).toFixed(0)}m${(duration.secs % 60).toFixed(0)}s`; + const secs = duration.secs - Math.floor(duration.secs / 60) * 60; + return `${(duration.secs / 60).toFixed(0)}m${secs.toFixed(0)}s`; } - return `${duration.secs}.${(duration.nanos / 1000000).toFixed(0).padStart(3, "0").replace(/0+$/, "")}s`; + return `${duration.secs}s`; } diff --git a/bindings/src/types/tari-indexer-client/Connection.ts b/bindings/src/types/tari-indexer-client/Connection.ts index e45fd8801..a51952229 100644 --- a/bindings/src/types/tari-indexer-client/Connection.ts +++ b/bindings/src/types/tari-indexer-client/Connection.ts @@ -8,4 +8,5 @@ export interface Connection { direction: ConnectionDirection; age: { secs: number; nanos: number }; ping_latency: { secs: number; nanos: number } | null; + user_agent: string | null; } diff --git a/bindings/src/types/validator-node-client/Connection.ts b/bindings/src/types/validator-node-client/Connection.ts index e45fd8801..a51952229 100644 --- a/bindings/src/types/validator-node-client/Connection.ts +++ b/bindings/src/types/validator-node-client/Connection.ts @@ -8,4 +8,5 @@ export interface Connection { direction: ConnectionDirection; age: { secs: number; nanos: number }; ping_latency: { secs: number; nanos: number } | null; + user_agent: string | null; } diff --git a/clients/tari_indexer_client/src/types.rs b/clients/tari_indexer_client/src/types.rs index 60f79080d..5e89520cc 100644 --- a/clients/tari_indexer_client/src/types.rs +++ b/clients/tari_indexer_client/src/types.rs @@ -1,7 +1,7 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use multiaddr::Multiaddr; use serde::{Deserialize, Serialize}; @@ -438,6 +438,7 @@ pub struct Connection { pub age: Duration, #[cfg_attr(feature = "ts", ts(type = "{secs: number, nanos: number} | null"))] pub ping_latency: Option, + pub user_agent: Option>, } #[derive(Serialize, Debug)] diff --git a/clients/validator_node_client/src/types.rs b/clients/validator_node_client/src/types.rs index ff2a404c3..4ea069b80 100644 --- a/clients/validator_node_client/src/types.rs +++ b/clients/validator_node_client/src/types.rs @@ -20,7 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -use std::{ops::RangeInclusive, time::Duration}; +use std::{ops::RangeInclusive, sync::Arc, time::Duration}; use multiaddr::Multiaddr; use serde::{Deserialize, Serialize}; @@ -731,6 +731,7 @@ pub struct Connection { pub age: Duration, #[cfg_attr(feature = "ts", ts(type = "{secs: number, nanos: number} | null"))] pub ping_latency: Option, + pub user_agent: Option>, } #[derive(Serialize, Debug)] diff --git a/networking/core/src/connection.rs b/networking/core/src/connection.rs index 6c038ed07..611f671a6 100644 --- a/networking/core/src/connection.rs +++ b/networking/core/src/connection.rs @@ -1,7 +1,10 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use libp2p::{core::ConnectedPoint, swarm::ConnectionId, PeerId}; @@ -15,6 +18,7 @@ pub struct Connection { pub num_concurrent_dial_errors: usize, pub established_in: Duration, pub ping_latency: Option, + pub user_agent: Option>, } impl Connection { diff --git a/networking/core/src/worker.rs b/networking/core/src/worker.rs index 5b8ec6b0e..b73226230 100644 --- a/networking/core/src/worker.rs +++ b/networking/core/src/worker.rs @@ -4,6 +4,7 @@ use std::{ collections::{hash_map::Entry, HashMap}, hash::Hash, + sync::Arc, time::{Duration, Instant}, }; @@ -764,6 +765,7 @@ where num_concurrent_dial_errors, established_in, ping_latency: None, + user_agent: None, }); let Some(waiters) = self.pending_dial_requests.remove(&peer_id) else { @@ -793,6 +795,8 @@ where return Ok(()); } + self.update_connected_peers(&peer_id, &info); + let is_relay = info.protocols.iter().any(|p| *p == relay::HOP_PROTOCOL_NAME); let is_connected_through_relay = self @@ -840,6 +844,17 @@ where Ok(()) } + fn update_connected_peers(&mut self, peer_id: &PeerId, info: &identify::Info) { + let Some(conns_mut) = self.active_connections.get_mut(peer_id) else { + return; + }; + + let user_agent = Arc::new(info.agent_version.clone()); + for conn_mut in conns_mut { + conn_mut.user_agent = Some(user_agent.clone()); + } + } + /// Establishes a relay circuit for the given peer if the it is the selected relay peer. Returns true if the circuit /// was established from this call. fn establish_relay_circuit_on_connect(&mut self, peer_id: &PeerId) -> bool { From 81d884a6ec16a53eab7dcc33d83031a83a39d46f Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Tue, 30 Apr 2024 17:51:52 +0400 Subject: [PATCH 2/6] fix(consensus)!: deferred execution of txs with unversioned inputs (#1023) Description --- - Mark transactions as deferred and execute in consensus - Set resolved inputs for each transaction that includes which lock is required - Remove input refs from transactions - Determine read locks from transaction execution result Motivation and Context --- Deferred execution previously had a default COMMIT decision which led to bugs when running rejected transactions (they would still get a COMMIT decision, but the validator does not have an accepted result to commit). This PR partially implements late execution for local-only transactions. Additional work is required to support this and it has been disabled i.e. will ABORT the transaction for transactions without versions. There was a bug where inputs without versions were provided but filled inputs for those substates were provided, the transaction would still detect that one or more inputs do not have a version. Input refs are unnecessary because the validator will always know which substates were written at lock time. TODOs: - missing transactions needs to differentiate between transactions pending execution and transactions that are deferred. - correct pledging procedure for unversioned inputs - blocks should generate a diff which is committed on a 3-chain - current committed state + diffs from uncommitted blocks can be used to determine inputs for subsequent transactions How Has This Been Tested? --- Existing tests pass, a new consensus unit test was written which is ignored because it fails, requires additional work to succeed, manually (previously working transactions still work). What process can a PR reviewer use to test or verify this change? --- Submit a transaction with a missing version Breaking Changes --- - [ ] None - [x] Requires data directory to be deleted - [ ] Other - Please specify --- Cargo.lock | 3 +- Cargo.toml | 2 +- .../tari_dan_app_utilities/Cargo.toml | 14 +- .../src/transaction_executor.rs | 2 +- .../src/handlers/accounts.rs | 12 +- .../AssetVault/Components/SendMoney.tsx | 21 +- .../tari_indexer/src/dry_run/error.rs | 3 + .../tari_indexer/src/dry_run/processor.rs | 16 +- .../tari_indexer/src/json_rpc/handlers.rs | 24 +- .../src/transaction_manager/mod.rs | 33 +- .../tari_validator_node/src/bootstrap.rs | 5 +- .../consensus/block_transaction_executor.rs | 131 +++++--- .../src/consensus/metrics.rs | 1 + .../src/dry_run_transaction_processor.rs | 3 +- .../src/json_rpc/handlers.rs | 13 +- .../src/p2p/services/mempool/error.rs | 2 - .../src/p2p/services/mempool/executor.rs | 67 +++- .../src/p2p/services/mempool/service.rs | 216 +++++++----- .../src/p2p/services/mempool/traits.rs | 9 +- .../validators/after/has_involved_shards.rs | 2 +- .../mempool/validators/after/input_refs.rs | 39 --- .../services/mempool/validators/after/mod.rs | 2 - .../src/substate_resolver.rs | 29 +- .../src/command/account.rs | 1 - .../src/command/transaction.rs | 5 - .../src/Components/StatusChip.tsx | 3 +- bindings/index.ts | 2 + bindings/src/types/Decision.ts | 2 +- bindings/src/types/ExecutedTransaction.ts | 2 + bindings/src/types/SubstateLockFlag.ts | 3 + bindings/src/types/Transaction.ts | 4 +- bindings/src/types/UnsignedTransaction.ts | 4 +- .../types/VersionedSubstateIdLockIntent.ts | 8 + clients/wallet_daemon_client/src/types.rs | 1 + .../common_types/src/validator_metadata.rs | 8 +- dan_layer/consensus/src/hotstuff/error.rs | 4 +- .../src/hotstuff/on_inbound_message.rs | 18 +- .../consensus/src/hotstuff/on_propose.rs | 17 +- .../on_ready_to_vote_on_local_block.rs | 314 +++++++++--------- .../hotstuff/on_receive_foreign_proposal.rs | 2 +- .../on_receive_requested_transactions.rs | 2 +- dan_layer/consensus/src/hotstuff/worker.rs | 51 ++- dan_layer/consensus/src/messages/message.rs | 16 +- .../src/traits/transaction_executor.rs | 11 +- dan_layer/consensus_tests/src/consensus.rs | 62 +++- .../consensus_tests/src/support/harness.rs | 67 +++- .../consensus_tests/src/support/network.rs | 69 ++-- .../src/support/transaction.rs | 97 +++--- .../src/support/transaction_executor.rs | 6 +- dan_layer/engine/src/runtime/impl.rs | 8 +- dan_layer/engine/src/state_store/memory.rs | 28 +- dan_layer/engine_types/Cargo.toml | 1 - .../engine_types/src/base_layer_hashing.rs | 135 +------- dan_layer/engine_types/src/commit_result.rs | 2 + dan_layer/epoch_manager/src/validator_node.rs | 52 --- dan_layer/indexer_lib/Cargo.toml | 9 +- .../indexer_lib/src/transaction_autofiller.rs | 54 ++- dan_layer/p2p/proto/consensus.proto | 1 + dan_layer/p2p/proto/transaction.proto | 12 +- dan_layer/p2p/src/conversions/consensus.rs | 2 + dan_layer/p2p/src/conversions/transaction.rs | 40 ++- .../up.sql | 84 ++--- dan_layer/state_store_sqlite/src/reader.rs | 44 ++- dan_layer/state_store_sqlite/src/schema.rs | 6 +- .../src/sql_models/block.rs | 4 +- .../src/sql_models/transaction.rs | 7 +- dan_layer/state_store_sqlite/src/writer.rs | 179 +++++----- .../storage/src/consensus_models/command.rs | 10 +- .../consensus_models/executed_transaction.rs | 95 ++++-- .../storage/src/consensus_models/substate.rs | 72 +++- .../src/consensus_models/transaction.rs | 125 +++++-- .../consensus_models/transaction_decision.rs | 9 + .../src/consensus_models/transaction_pool.rs | 29 +- dan_layer/storage/src/state_store/mod.rs | 11 +- .../src/template_test.rs | 6 +- dan_layer/transaction/Cargo.toml | 1 + dan_layer/transaction/src/builder.rs | 54 +-- dan_layer/transaction/src/signature.rs | 5 +- dan_layer/transaction/src/transaction.rs | 152 ++++++--- .../transaction/src/unsigned_transaction.rs | 14 +- dan_layer/wallet/storage_sqlite/Cargo.toml | 8 +- .../2023-02-08-122514_initial/up.sql | 2 +- .../wallet/storage_sqlite/src/models/mod.rs | 2 +- .../storage_sqlite/src/models/transaction.rs | 18 +- dan_layer/wallet/storage_sqlite/src/schema.rs | 2 +- dan_layer/wallet/storage_sqlite/src/writer.rs | 7 +- integration_tests/build.rs | 7 + integration_tests/src/validator_node_cli.rs | 22 -- utilities/tariswap_test_bench/src/tariswap.rs | 7 +- 89 files changed, 1546 insertions(+), 1208 deletions(-) delete mode 100644 applications/tari_validator_node/src/p2p/services/mempool/validators/after/input_refs.rs create mode 100644 bindings/src/types/SubstateLockFlag.ts create mode 100644 bindings/src/types/VersionedSubstateIdLockIntent.ts delete mode 100644 dan_layer/epoch_manager/src/validator_node.rs create mode 100644 integration_tests/build.rs diff --git a/Cargo.lock b/Cargo.lock index 287b161d6..c17707e6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9395,7 +9395,6 @@ version = "0.5.1" dependencies = [ "base64 0.21.7", "blake2", - "borsh", "digest 0.10.7", "hex", "lazy_static", @@ -9542,6 +9541,7 @@ name = "tari_indexer_lib" version = "0.5.1" dependencies = [ "async-trait", + "futures 0.3.30", "log", "rand", "serde", @@ -9995,6 +9995,7 @@ dependencies = [ name = "tari_transaction" version = "0.5.1" dependencies = [ + "indexmap 2.2.6", "rand", "serde", "tari_common_types", diff --git a/Cargo.toml b/Cargo.toml index aa927af14..b7f2b773a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,7 +179,7 @@ httpmock = "0.6.8" humantime = "2.1.0" humantime-serde = "1.1.1" include_dir = "0.7.2" -indexmap = "2.1.0" +indexmap = "2.2.6" indoc = "1.0.6" itertools = "0.11.0" lazy_static = "1.4.0" diff --git a/applications/tari_dan_app_utilities/Cargo.toml b/applications/tari_dan_app_utilities/Cargo.toml index 46955e54f..ffbc8b267 100644 --- a/applications/tari_dan_app_utilities/Cargo.toml +++ b/applications/tari_dan_app_utilities/Cargo.toml @@ -26,7 +26,7 @@ tari_transaction = { workspace = true } tari_validator_node_client = { workspace = true } tari_bor = { workspace = true, default-features = true } tari_indexer_lib = { workspace = true } -tari_networking = { workspace = true} +tari_networking = { workspace = true } tari_validator_node_rpc = { workspace = true } anyhow = { workspace = true } @@ -49,11 +49,11 @@ serde = { workspace = true, features = ["default", "derive"] } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ - "default", - "macros", - "time", - "sync", - "rt-multi-thread", -]} + "default", + "macros", + "time", + "sync", + "rt-multi-thread", +] } tokio-stream = { workspace = true, features = ["sync"] } config = { workspace = true } diff --git a/applications/tari_dan_app_utilities/src/transaction_executor.rs b/applications/tari_dan_app_utilities/src/transaction_executor.rs index 24d5f5d79..e1904011f 100644 --- a/applications/tari_dan_app_utilities/src/transaction_executor.rs +++ b/applications/tari_dan_app_utilities/src/transaction_executor.rs @@ -76,7 +76,7 @@ where TTemplateProvider: TemplateProvider