feat(control): merge config file with CLI args

Cargo.toml

@@ -57,6 +57,7 @@ clap = { version = "4.5", features = ["derive"] }
 clap_complete = { version = "4.5" }
 clap_mangen = { version = "0.2" }
 clap-markdown = "0.1"
+clap-serde-derive = "0.2"
 clap-stdin = "0.4"
 colored = "2"
 crossterm = { version = "0.27", default-features = false }

crates/snops-common/Cargo.toml

@@ -28,6 +28,7 @@ regex.workspace = true
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+serde_yaml.workspace = true
 sha2.workspace = true
 sled.workspace = true
 strum_macros.workspace = true

crates/snops-common/src/util.rs

@@ -1,5 +1,12 @@
-use std::{fmt::Debug, io::Read, path::PathBuf};
+use std::{
+    ffi::OsStr,
+    fmt::Debug,
+    fs::File,
+    io::{BufReader, Read},
+    path::{Path, PathBuf},
+};
 
+use serde::de::DeserializeOwned;
 use sha2::{Digest, Sha256};
 
 /// A wrapper struct that has an "opaque" `Debug` implementation for types
@@ -41,3 +48,21 @@ pub fn sha256_file(path: &PathBuf) -> Result<String, std::io::Error> {
 
     Ok(format!("{:x}", digest.finalize()))
 }
+
+pub fn parse_file_from_extension<T: DeserializeOwned>(
+    path: &Path,
+    file: File,
+) -> Result<T, Box<dyn core::error::Error>> {
+    // TODO: toml
+    let reader = BufReader::new(file);
+    let ext = path.extension().and_then(OsStr::to_str).unwrap_or_else(|| {
+        tracing::warn!("invalid parse extension; falling back to yaml");
+        "yaml"
+    });
+
+    Ok(match ext {
+        "yaml" | "yml" => serde_yaml::from_reader(reader)?,
+        "json" => serde_yaml::from_reader(reader)?,
+        _ => unimplemented!("unknown parse extension {ext}"),
+    })
+}

crates/snops/Cargo.toml

@@ -24,6 +24,7 @@ bytes.workspace = true
 checkpoint.workspace = true
 chrono = { workspace = true, features = ["serde"] }
 clap = { workspace = true, features = ["env"] }
+clap-serde-derive.workspace = true
 dashmap = { workspace = true, features = ["serde"] }
 duration-str.workspace = true
 fixedbitset.workspace = true

crates/snops/src/cannon/mod.rs

@@ -199,7 +199,7 @@ impl CannonInstance {
     pub fn get_local_query(&self) -> String {
         format!(
             "http://{}/api/v1/env/{}/cannons/{}",
-            self.global_state.cli.get_local_addr(),
+            self.global_state.config.get_local_addr(),
             self.env_id,
             self.id
         )
@@ -309,11 +309,11 @@ impl ExecutionContext {
             // demox needs to locate it
             ComputeTarget::Demox { .. } => {
                 let host = state
-                    .cli
+                    .config
                     .hostname
                     .as_ref()
                     .ok_or(ExecutionContextError::NoHostnameConfigured)?;
-                format!("{host}:{}{suffix}", state.cli.port)
+                format!("{host}:{}{suffix}", state.config.port)
             }
         };
         trace!("cannon {env_id}.{cannon_id} using realtime query {query_path}");

crates/snops/src/cli.rs

@@ -8,15 +8,34 @@ use std::{
 #[cfg(any(feature = "clipages", feature = "mangen"))]
 use clap::CommandFactory;
 use clap::Parser;
+use clap_serde_derive::ClapSerde;
+use serde::{de::Error, Deserialize};
 use url::Url;
 
-#[derive(Debug, Parser)]
+#[derive(Parser)]
 pub struct Cli {
-    #[clap(long = "bind", default_value_t = IpAddr::V4(Ipv4Addr::UNSPECIFIED))]
+    /// A path to a config file. A config file is a YAML file; all config
+    /// arguments are valid YAML fields.
+    #[arg(short, long = "config")]
+    pub config_path: Option<PathBuf>,
+
+    #[command(flatten)]
+    pub config: <Config as ClapSerde>::Opt,
+
+    #[cfg(any(feature = "clipages", feature = "mangen"))]
+    #[clap(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(ClapSerde, Debug)]
+pub struct Config {
+    #[default(IpAddr::V4(Ipv4Addr::UNSPECIFIED))]
+    #[clap(long = "bind")]
     pub bind_addr: IpAddr,
 
     /// Control plane server port
-    #[arg(long, default_value_t = 1234)]
+    #[default(1234)]
+    #[arg(long)]
     pub port: u16,
 
     // TODO: store services in a file config or something?
@@ -29,24 +48,22 @@ pub struct Cli {
     #[arg(long)]
     pub loki: Option<Url>,
 
-    #[arg(long, default_value_t = PrometheusLocation::Docker)]
+    #[default(PrometheusLocation::Docker)]
+    #[arg(long)]
     pub prometheus_location: PrometheusLocation,
 
     /// Path to the directory containing the stored data
-    #[arg(long, default_value = "snops-control-data")]
+    #[default(PathBuf::from("snops-control-data"))]
+    #[arg(long)]
     pub path: PathBuf,
 
-    #[arg(long)]
     /// Hostname to advertise to the control plane, used when resolving the
     /// control plane's address for external cannons can be an external IP
     /// or FQDN, will have the port appended
     ///
     /// must contain http:// or https://
+    #[arg(long)]
     pub hostname: Option<String>,
-
-    #[cfg(any(feature = "clipages", feature = "mangen"))]
-    #[clap(subcommand)]
-    pub command: Commands,
 }
 
 #[cfg(any(feature = "clipages", feature = "mangen"))]
@@ -80,7 +97,9 @@ impl Cli {
 
         std::process::exit(0);
     }
+}
 
+impl Config {
     pub fn get_local_addr(&self) -> SocketAddr {
         let ip = if self.bind_addr.is_unspecified() {
             IpAddr::V4(Ipv4Addr::LOCALHOST)
@@ -125,3 +144,12 @@ impl FromStr for PrometheusLocation {
         })
     }
 }
+
+impl<'de> Deserialize<'de> for PrometheusLocation {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        PrometheusLocation::from_str(<&str>::deserialize(deserializer)?).map_err(D::Error::custom)
+    }
+}

crates/snops/src/main.rs

@@ -1,8 +1,10 @@
-use std::io;
+use std::{fs::File, io};
 
 use clap::Parser;
-use cli::Cli;
+use clap_serde_derive::ClapSerde;
+use cli::{Cli, Config};
 use schema::storage::{DEFAULT_AGENT_BINARY, DEFAULT_AOT_BINARY};
+use snops_common::util::parse_file_from_extension;
 use tracing::{info, level_filters::LevelFilter};
 use tracing_subscriber::{prelude::*, reload, EnvFilter};
 
@@ -64,12 +66,41 @@ async fn main() {
     // For documentation purposes will exit after running the command.
     #[cfg(any(feature = "clipages", feature = "mangen"))]
     Cli::parse().run();
-    let cli = Cli::parse();
+    let mut cli = Cli::parse();
+
+    // build a full config object from CLI and from optional config file
+    let config = 'config: {
+        match cli.config_path {
+            // file passed: try to open it
+            Some(path) => match File::open(&path) {
+                // file opened: try to parse it
+                Ok(file) => {
+                    match parse_file_from_extension::<<Config as ClapSerde>::Opt>(&path, file) {
+                        // merge file with CLI args
+                        Ok(config) => break 'config Config::from(config).merge(&mut cli.config),
+
+                        // file parse fail
+                        Err(e) => {
+                            tracing::warn!("failed to parse config file, ignoring. error: {e}")
+                        }
+                    }
+                }
+
+                // file open fail
+                Err(e) => tracing::warn!("failed to open config file, ignoring. error: {e}"),
+            },
+
+            // no file
+            None => (),
+        };
+
+        Config::from(&mut cli.config)
+    };
 
     info!("Using AOT binary:\n{}", DEFAULT_AOT_BINARY.to_string());
     info!("Using Agent binary:\n{}", DEFAULT_AGENT_BINARY.to_string());
 
-    server::start(cli, reload_handler)
+    server::start(config, reload_handler)
         .await
         .expect("start server");
 }

crates/snops/src/persist/storage.rs

@@ -10,7 +10,7 @@ use tracing::{info, warn};
 
 use super::prelude::*;
 use crate::{
-    cli::Cli,
+    cli::Config,
     schema::{
         error::StorageError,
         storage::{
@@ -93,9 +93,9 @@ impl From<&LoadedStorage> for PersistStorage {
 }
 
 impl PersistStorage {
-    pub async fn load(self, cli: &Cli) -> Result<LoadedStorage, StorageError> {
+    pub async fn load(self, config: &Config) -> Result<LoadedStorage, StorageError> {
         let id = self.id;
-        let mut storage_path = cli.path.join(STORAGE_DIR);
+        let mut storage_path = config.path.join(STORAGE_DIR);
         storage_path.push(self.network.to_string());
         storage_path.push(id.to_string());
         let committee_file = storage_path.join("committee.json");

crates/snops/src/schema/storage/loaded.rs

@@ -14,7 +14,7 @@ use snops_common::{
 use tracing::{info, trace};
 
 use super::{DEFAULT_AOT_BINARY, STORAGE_DIR};
-use crate::{cli::Cli, schema::error::StorageError, state::GlobalState};
+use crate::{cli::Config, schema::error::StorageError, state::GlobalState};
 
 // IndexMap<addr, private_key>
 pub type AleoAddrMap = IndexMap<String, String>;
@@ -191,11 +191,11 @@ impl LoadedStorage {
     }
 
     pub fn path(&self, state: &GlobalState) -> PathBuf {
-        self.path_cli(&state.cli)
+        self.path_config(&state.config)
     }
 
-    pub fn path_cli(&self, cli: &Cli) -> PathBuf {
-        let mut path = cli.path.join(STORAGE_DIR);
+    pub fn path_config(&self, config: &Config) -> PathBuf {
+        let mut path = config.path.join(STORAGE_DIR);
         path.push(self.network.to_string());
         path.push(self.id.to_string());
         path
@@ -280,7 +280,7 @@ impl LoadedStorage {
         };
 
         // derive the path to the binary
-        let mut download_path = state.cli.path.join(STORAGE_DIR);
+        let mut download_path = state.config.path.join(STORAGE_DIR);
         download_path.push(network.to_string());
         download_path.push(storage_id.to_string());
         download_path.push("binaries");

crates/snops/src/server/content.rs

@@ -39,7 +39,7 @@ async fn not_found(uri: Uri, res: Response) -> Response {
 
 pub(super) async fn init_routes(state: &GlobalState) -> Router<AppState> {
     // create storage path
-    let storage_path = state.cli.path.join("storage");
+    let storage_path = state.config.path.join("storage");
     tracing::debug!("storage path: {:?}", storage_path);
     tokio::fs::create_dir_all(&storage_path)
         .await

crates/snops/src/server/mod.rs

@@ -35,7 +35,7 @@ use self::{
     rpc::ControlRpcServer,
 };
 use crate::{
-    cli::Cli,
+    cli::Config,
     db,
     logging::{log_request, req_stamp},
     server::rpc::{MuxedMessageIncoming, MuxedMessageOutgoing},
@@ -52,16 +52,16 @@ pub mod models;
 pub mod prometheus;
 mod rpc;
 
-pub async fn start(cli: Cli, log_level_handler: ReloadHandler) -> Result<(), StartError> {
-    let db = db::Database::open(&cli.path.join("store"))?;
-    let socket_addr = SocketAddr::new(cli.bind_addr, cli.port);
+pub async fn start(config: Config, log_level_handler: ReloadHandler) -> Result<(), StartError> {
+    let db = db::Database::open(&config.path.join("store"))?;
+    let socket_addr = SocketAddr::new(config.bind_addr, config.port);
 
-    let prometheus = cli
+    let prometheus = config
         .prometheus
         .as_ref()
         .and_then(|p| PrometheusClient::try_from(p.as_str()).ok());
 
-    let state = GlobalState::load(cli, db, prometheus, log_level_handler).await?;
+    let state = GlobalState::load(config, db, prometheus, log_level_handler).await?;
 
     let app = Router::new()
         .route("/agent", get(agent_ws_handler))
@@ -160,7 +160,7 @@ async fn handle_socket(
     let id: AgentId = 'insertion: {
         let client = client.clone();
         let mut handshake = Handshake {
-            loki: state.cli.loki.as_ref().map(|u| u.to_string()),
+            loki: state.config.loki.as_ref().map(|u| u.to_string()),
             ..Default::default()
         };