-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(watcher): add monitored events to be used by notifier (#1122)
Description --- Creates the following events, to be used by the notifier subsequently: - `Exited`: manually cancelling of the child process, returning its status code - `Crashed`: the child process finished for some reason but was not cancelled manually - `InternalError`: the watcher received an error when attempting to check the child process - `Submitted`: the watcher has submitted a registration transaction for the node (tracks `tx_id` and `block`) - `WarnExpiration`: the watcher emits a warning if the current block is near the expiration block of a registration - `Running`: the child process is running as expected, do nothing Picks up the events from the backend and logs them in `monitoring.rs`. In addition, whenever it spawns a new child process it also saved its process id to a file `validator.pid`. This is updated automatically and can be read from when we want to check whether a node process already exists. Currently, the lifetime of a child is dependent on the parent process. How Has This Been Tested? --- Running the `tari_watcher` together with `tari_swarm_daemon`. It then displayed (through logs) the node being registered and then warned when it was near expiration (less than 100 blocks from the expiration block, defined as `registered_at_block + 1000`).
- Loading branch information
1 parent
6dc89c8
commit f5f3860
Showing
13 changed files
with
581 additions
and
234 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Copyright 2024 The Tari Project | ||
| // SPDX-License-Identifier: BSD-3-Clause | ||
|
|
||
| use tokio::time::Duration; | ||
|
|
||
| pub const CONSENSUS_CONSTANT_REGISTRATION_DURATION: u64 = 1000; // in blocks: 100 epochs * 10 blocks/epoch | ||
|
|
||
| pub const DEFAULT_PROJECT_ROOT: &str = env!("CARGO_MANIFEST_DIR"); | ||
| pub const DEFAULT_WATCHER_CONFIG_PATH: &str = "data/watcher/config.toml"; | ||
| pub const DEFAULT_VALIDATOR_PID_PATH: &str = "data/watcher/validator.pid"; | ||
| pub const DEFAULT_VALIDATOR_NODE_BINARY_PATH: &str = "target/release/tari_validator_node"; | ||
| pub const DEFAULT_MINOTARI_MINER_BINARY_PATH: &str = "target/release/minotari_miner"; | ||
| pub const DEFAULT_BASE_NODE_GRPC_ADDRESS: &str = "http://127.0.0.1:12001"; // note: protocol | ||
| pub const DEFAULT_BASE_WALLET_GRPC_ADDRESS: &str = "http://127.0.0.1:12003"; // note: protocol | ||
|
|
||
| pub const DEFAULT_PROCESS_MONITORING_INTERVAL: Duration = Duration::from_secs(20); // time to sleep before checking VN process status | ||
| pub const DEFAULT_THRESHOLD_WARN_EXPIRATION: u64 = 100; // warn at this many blocks before the registration expires |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| // Copyright 2024 The Tari Project | ||
| // SPDX-License-Identifier: BSD-3-Clause | ||
|
|
||
| use fern::FormatCallback; | ||
|
|
||
| pub fn init_logger() -> Result<(), log::SetLoggerError> { | ||
| fn should_skip(target: &str) -> bool { | ||
| const SKIP: [&str; 3] = ["hyper::", "h2::", "tower::"]; | ||
| if SKIP.iter().any(|s| target.starts_with(s)) { | ||
| return true; | ||
| } | ||
|
|
||
| false | ||
| } | ||
|
|
||
| let colors = fern::colors::ColoredLevelConfig::new() | ||
| .info(fern::colors::Color::Green) | ||
| .debug(fern::colors::Color::Yellow) | ||
| .error(fern::colors::Color::Red); | ||
| fern::Dispatch::new() | ||
| .format(move |out, message, record| { | ||
| if should_skip(record.target()) { | ||
| return; | ||
| } | ||
|
|
||
| let fallback = |out: FormatCallback<'_>| out.finish(format_args!( | ||
| "[{} {} {}] {}", | ||
| humantime::format_rfc3339(std::time::SystemTime::now()), | ||
| record.metadata().target(), | ||
| colors.color(record.level()), | ||
| message | ||
| )); | ||
|
|
||
| // Example: [Validator node-#1] 12:55 INFO Received vote for block #NodeHeight(88) d9abc7b1bb66fd912848f5bc4e5a69376571237e3243dc7f6a91db02bb5cf37c from a08cf5038e8e3cda8e3716c79f769cd42fad05f7110628efb5be6a40e28bc94c (4 of 3) | ||
| // Implement a naive parsing of the log message to extract the target, level and the log message from each running process | ||
| let message_str = message.to_string(); | ||
| let Some((target, rest)) = message_str.split_once( ']') else { | ||
| fallback(out); | ||
| return; | ||
| }; | ||
|
|
||
| let mut parts = rest.trim().splitn(3, ' '); | ||
|
|
||
| // Skip the time | ||
| if parts.next().is_none() { | ||
| fallback(out); | ||
| return; | ||
| } | ||
|
|
||
| let Some(level) = parts.next() | ||
| .and_then(|s| s.parse().ok()) | ||
| .map(|l| colors.color(l)) else { | ||
| fallback(out); | ||
| return; | ||
| }; | ||
|
|
||
| let Some(log) = parts.next() else { | ||
| fallback(out); | ||
| return; | ||
| }; | ||
|
|
||
| out.finish(format_args!( | ||
| "{} {} {}] {} {}", | ||
| humantime::format_rfc3339(std::time::SystemTime::now()), | ||
| record.metadata().target(), | ||
| target, | ||
| level, | ||
| log | ||
| )) | ||
| }) | ||
| .filter(|record_metadata| record_metadata.target().starts_with("tari_watcher")) // skip tokio frame prints | ||
| .level(log::LevelFilter::Debug) | ||
| .chain(std::io::stdout()) | ||
| // .chain(fern::log_file("output.log").unwrap()) | ||
| .apply() | ||
| } |
Oops, something went wrong.