From ee33fd4bc3c742e76b316d9ebed28ceb9a88017c Mon Sep 17 00:00:00 2001 From: Priyansh Rathi Date: Sat, 19 Aug 2023 01:08:40 -0700 Subject: [PATCH] vsock: Add new VMs at runtime Implement ability to add new VMs at runtime. Removing or modifying the configuration of an existing VM at runtime is not supported at the moment. To use this feature, you must use the `--config` CLI argument to specify the initial configuration with the `--watch` flag set. The path to the config file provided must be a symlink to the actual YAML config. To update the VM configuration at runtime, simply change the symlink to point to the YAML config file with the new configuration. Signed-off-by: Priyansh Rathi --- Cargo.lock | 118 +++++++++++++++++++ crates/vsock/Cargo.toml | 1 + crates/vsock/src/main.rs | 245 ++++++++++++++++++++++++++++----------- 3 files changed, 299 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9006c53..7db3893b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -100,6 +115,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -475,6 +505,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "glob" version = "0.3.1" @@ -533,6 +569,28 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "futures-core", + "inotify-sys", + "libc", + "tokio", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "intmap" version = "2.0.0" @@ -648,6 +706,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + [[package]] name = "nom" version = "7.1.3" @@ -689,6 +767,15 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -902,6 +989,12 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -997,6 +1090,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1100,6 +1203,20 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + [[package]] name = "toml" version = "0.5.11" @@ -1274,6 +1391,7 @@ dependencies = [ "env_logger", "epoll", "futures", + "inotify", "log", "serde", "serde_yaml", diff --git a/crates/vsock/Cargo.toml b/crates/vsock/Cargo.toml index 2b37c561..bcacc517 100644 --- a/crates/vsock/Cargo.toml +++ b/crates/vsock/Cargo.toml @@ -30,6 +30,7 @@ vmm-sys-util = "0.11" config = "0.13" serde = "1" serde_yaml = "0.9" +inotify = "0.10.2" [dev-dependencies] virtio-queue = { version = "0.9", features = ["test-utils"] } diff --git a/crates/vsock/src/main.rs b/crates/vsock/src/main.rs index fa38bfb6..224ded4e 100644 --- a/crates/vsock/src/main.rs +++ b/crates/vsock/src/main.rs @@ -9,15 +9,17 @@ mod vhu_vsock_thread; mod vsock_conn; use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, convert::TryFrom, + path::Path, process::exit, - sync::{Arc, RwLock}, + sync::{Arc, Mutex, RwLock}, thread, }; use crate::vhu_vsock::{CidMap, VhostUserVsockBackend, VsockConfig}; use clap::{Args, Parser}; +use inotify::{EventMask, Inotify, WatchMask}; use log::{error, info, warn}; use serde::Deserialize; use thiserror::Error as ThisError; @@ -25,6 +27,8 @@ use vhost::{vhost_user, vhost_user::Listener}; use vhost_user_backend::VhostUserDaemon; use vm_memory::{GuestMemoryAtomic, GuestMemoryMmap}; +type JoinHandle = thread::JoinHandle>; + const DEFAULT_GUEST_CID: u64 = 3; const DEFAULT_TX_BUFFER_SIZE: u32 = 64 * 1024; const DEFAULT_GROUP_NAME: &str = "default"; @@ -55,6 +59,14 @@ enum BackendError { CouldNotCreateBackend(vhu_vsock::Error), #[error("Could not create daemon: {0}")] CouldNotCreateDaemon(vhost_user_backend::Error), + #[error("Failed to start config watcher")] + FailedToStartConfigWatcher, +} + +#[derive(Debug, ThisError)] +enum UpdateError { + #[error("Could not parse updated config file")] + ConfigParse, } #[derive(Args, Clone, Debug)] @@ -101,6 +113,12 @@ struct ConfigFileVsockParam { groups: Option, } +#[derive(Debug, Clone)] +struct CliVsockConfig { + vsock_configs: Vec, + watched_config: Option, +} + #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct VsockArgs { @@ -118,6 +136,11 @@ struct VsockArgs { /// Load from a given configuration file #[arg(long)] config: Option, + + /// Watch the configuration file for changes and update the configuration accordingly at runtime. + /// This option is only valid when used with the `config` option. + #[arg(long, requires = "config")] + watch: bool, } fn parse_vm_params(s: &str) -> Result { @@ -155,58 +178,74 @@ fn parse_vm_params(s: &str) -> Result { )) } -impl VsockArgs { - pub fn parse_config(&self) -> Option, CliError>> { - if let Some(c) = &self.config { - let b = config::Config::builder() - .add_source(config::File::new(c.as_str(), config::FileFormat::Yaml)) - .build(); - if let Ok(s) = b { - let mut v = s.get::>("vms").unwrap(); - if !v.is_empty() { - let parsed: Vec = v - .drain(..) - .map(|p| { - VsockConfig::new( - p.guest_cid.unwrap_or(DEFAULT_GUEST_CID), - p.socket.trim().to_string(), - p.uds_path.trim().to_string(), - p.tx_buffer_size.unwrap_or(DEFAULT_TX_BUFFER_SIZE), - p.groups.map_or(vec![DEFAULT_GROUP_NAME.to_string()], |g| { - g.trim().split('+').map(String::from).collect() - }), - ) - }) - .collect(); - return Some(Ok(parsed)); - } else { - return Some(Err(CliError::ConfigParse)); - } - } else { - return Some(Err(CliError::ConfigParse)); - } +pub(crate) fn parse_config(config: &str) -> Result, CliError> { + let b = config::Config::builder() + .add_source(config::File::new(config, config::FileFormat::Yaml)) + .build(); + if let Ok(s) = b { + let mut v = s.get::>("vms").unwrap(); + if !v.is_empty() { + let parsed: Vec = v + .drain(..) + .map(|p| { + VsockConfig::new( + p.guest_cid.unwrap_or(DEFAULT_GUEST_CID), + p.socket.trim().to_string(), + p.uds_path.trim().to_string(), + p.tx_buffer_size.unwrap_or(DEFAULT_TX_BUFFER_SIZE), + p.groups.map_or(vec![DEFAULT_GROUP_NAME.to_string()], |g| { + g.trim().split('+').map(String::from).collect() + }), + ) + }) + .collect(); + Ok(parsed) + } else { + Err(CliError::ConfigParse) } - None + } else { + Err(CliError::ConfigParse) } } -impl TryFrom for Vec { +impl CliVsockConfig { + pub fn new(vsock_configs: Vec, watched_config: Option) -> Self { + Self { + vsock_configs, + watched_config, + } + } +} + +impl TryFrom for CliVsockConfig { type Error = CliError; fn try_from(cmd_args: VsockArgs) -> Result { // we try to use the configuration first, if failed, then fall back to the manual settings. - match cmd_args.parse_config() { - Some(c) => c, + match &cmd_args.config { + Some(c) => parse_config(c).map(|v| { + Self::new( + v, + if cmd_args.watch { + cmd_args.config + } else { + None + }, + ) + }), _ => match cmd_args.vm { - Some(v) => Ok(v), + Some(v) => Ok(Self::new(v, None)), _ => cmd_args.param.map_or(Err(CliError::NoArgsProvided), |p| { - Ok(vec![VsockConfig::new( - p.guest_cid, - p.socket.trim().to_string(), - p.uds_path.trim().to_string(), - p.tx_buffer_size, - p.groups.trim().split('+').map(String::from).collect(), - )]) + Ok(CliVsockConfig::new( + vec![VsockConfig::new( + p.guest_cid, + p.socket.trim().to_string(), + p.uds_path.trim().to_string(), + p.tx_buffer_size, + p.groups.trim().split('+').map(String::from).collect(), + )], + None, + )) }), }, } @@ -215,11 +254,13 @@ impl TryFrom for Vec { /// This is the public API through which an external program starts the /// vhost-device-vsock backend server. -pub(crate) fn start_backend_server( - config: VsockConfig, +pub(crate) fn start_backend_server_thread( + config: Arc>, cid_map: Arc>, ) -> Result<(), BackendError> { loop { + let config = config.read().unwrap().clone(); + let backend = Arc::new( VhostUserVsockBackend::new(config.clone(), cid_map.clone()) .map_err(BackendError::CouldNotCreateBackend)?, @@ -264,21 +305,93 @@ pub(crate) fn start_backend_server( } } -pub(crate) fn start_backend_servers(configs: &[VsockConfig]) -> Result<(), BackendError> { +pub(crate) fn start_backend_server( + config: &VsockConfig, + cid_map: Arc>, + handles: Arc>>, +) { + let c = Arc::new(RwLock::new(config.clone())); + let guest_cid = config.get_guest_cid(); + + let handle = thread::Builder::new() + .name(format!("vhu-vsock-cid-{}", guest_cid)) + .spawn(move || start_backend_server_thread(c, cid_map)) + .unwrap(); + handles.lock().unwrap().push_back(handle); +} + +pub(crate) fn update_config( + config: &str, + cid_map: &Arc>, + handles: &Arc>>, +) -> Result<(), UpdateError> { + let vsock_configs = parse_config(config).map_err(|_| UpdateError::ConfigParse)?; + for c in vsock_configs.iter() { + let guest_cid = c.get_guest_cid(); + if !cid_map.read().unwrap().contains_key(&guest_cid) { + start_backend_server(c, cid_map.clone(), handles.clone()); + } + } + + Ok(()) +} + +pub(crate) fn start_config_watcher( + cid_map: Arc>, + handles: Arc>>, + watched_config: String, +) -> Result<(), BackendError> { + let mut inotify = Inotify::init().map_err(|_| BackendError::FailedToStartConfigWatcher)?; + + let watched_config_path = Path::new(&watched_config); + let watched_config_file_name = watched_config_path.file_name().unwrap(); + let watched_config_dir = watched_config_path.parent().unwrap(); + + inotify + .watches() + .add(watched_config_dir, WatchMask::MOVED_TO) + .map_err(|_| BackendError::FailedToStartConfigWatcher)?; + + let mut buffer = [0u8; 4096]; + loop { + let events = inotify.read_events_blocking(&mut buffer).unwrap(); + + for event in events { + if event.mask.contains(EventMask::MOVED_TO) { + let dest_file_name = event.name.unwrap(); + if dest_file_name == watched_config_file_name { + if let Err(e) = update_config(&watched_config, &cid_map, &handles) { + error!("Error updating config: {:?}", e); + } + } + } + } + } +} + +pub(crate) fn start_backend_servers(cli_vsock_config: &CliVsockConfig) -> Result<(), BackendError> { let cid_map: Arc> = Arc::new(RwLock::new(HashMap::new())); - let mut handles = Vec::new(); + let handles = Arc::new(Mutex::new(VecDeque::new())); + + for c in cli_vsock_config.vsock_configs.iter() { + start_backend_server(c, cid_map.clone(), handles.clone()); + } - for c in configs.iter() { - let config = c.clone(); - let cid_map = cid_map.clone(); + if let Some(watched_config) = cli_vsock_config.watched_config.clone() { + let handles2 = handles.clone(); let handle = thread::Builder::new() - .name(format!("vhu-vsock-cid-{}", c.get_guest_cid())) - .spawn(move || start_backend_server(config, cid_map)) + .name("vhu-vsock-config-watcher".to_string()) + .spawn(move || start_config_watcher(cid_map, handles2, watched_config)) .unwrap(); - handles.push(handle); + handles.lock().unwrap().push_back(handle); } - for handle in handles { + loop { + if handles.lock().unwrap().is_empty() { + break; + } + + let handle = handles.lock().unwrap().pop_front().unwrap(); handle.join().unwrap()?; } @@ -288,7 +401,7 @@ pub(crate) fn start_backend_servers(configs: &[VsockConfig]) -> Result<(), Backe fn main() { env_logger::init(); - let configs = match Vec::::try_from(VsockArgs::parse()) { + let cli_vsock_config = match CliVsockConfig::try_from(VsockArgs::parse()) { Ok(c) => c, Err(e) => { println!("Error parsing arguments: {}", e); @@ -296,7 +409,7 @@ fn main() { } }; - if let Err(e) = start_backend_servers(&configs) { + if let Err(e) = start_backend_servers(&cli_vsock_config) { error!("{e}"); exit(1); } @@ -327,6 +440,7 @@ mod tests { }), vm: None, config: None, + watch: false, } } fn from_file(config: &str) -> Self { @@ -334,6 +448,7 @@ mod tests { param: None, vm: None, config: Some(config.to_string()), + watch: false, } } } @@ -346,10 +461,10 @@ mod tests { let uds_path = test_dir.path().join("vm4.vsock").display().to_string(); let args = VsockArgs::from_args(3, &socket_path, &uds_path, 64 * 1024, "group1"); - let configs = Vec::::try_from(args); - assert!(configs.is_ok()); + let cli_vsock_config = CliVsockConfig::try_from(args); + assert!(cli_vsock_config.is_ok()); - let configs = configs.unwrap(); + let configs = cli_vsock_config.unwrap().vsock_configs; assert_eq!(configs.len(), 1); let config = &configs[0]; @@ -393,10 +508,10 @@ mod tests { let args = VsockArgs::parse_from(params); - let configs = Vec::::try_from(args); - assert!(configs.is_ok()); + let cli_vsock_config = CliVsockConfig::try_from(args); + assert!(cli_vsock_config.is_ok()); - let configs = configs.unwrap(); + let configs = cli_vsock_config.unwrap().vsock_configs; assert_eq!(configs.len(), 3); let config = configs.get(0).unwrap(); @@ -460,7 +575,7 @@ mod tests { .unwrap(); let args = VsockArgs::from_file(&config_path.display().to_string()); - let configs = Vec::::try_from(args).unwrap(); + let configs = CliVsockConfig::try_from(args).unwrap().vsock_configs; assert_eq!(configs.len(), 1); let config = &configs[0]; @@ -488,7 +603,7 @@ mod tests { .unwrap(); let args = VsockArgs::from_file(&config_path.display().to_string()); - let configs = Vec::::try_from(args).unwrap(); + let configs = CliVsockConfig::try_from(args).unwrap().vsock_configs; assert_eq!(configs.len(), 1); let config = &configs[0];