From d20d7f7ec9588298eec6531dea9af39baec5e5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sat, 7 Dec 2024 14:20:46 +0100 Subject: [PATCH 1/6] mkdirs + task result markers + allow legacy wit + schema fixes (cherry picked from commit 1ead76b852c36a7574c507d764e7169137acbd35) --- Cargo.lock | 35 ++- wasm-rpc-stubgen/Cargo.toml | 2 + .../schema/golem-wasm-rpc.schema.json | 18 +- wasm-rpc-stubgen/src/commands/app.rs | 279 ++++++++++++++---- wasm-rpc-stubgen/src/model/app.rs | 13 +- wasm-rpc-stubgen/src/model/app_raw.rs | 2 + wasm-rpc-stubgen/src/model/template.rs | 1 + wasm-rpc-stubgen/src/wit_resolve.rs | 6 + 8 files changed, 297 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cb79f66..7ded79c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,12 +176,24 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "assert2" version = "0.3.15" @@ -511,6 +523,19 @@ dependencies = [ "typenum", ] +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec 0.7.6", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -922,6 +947,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "cookie" version = "0.18.1" @@ -1919,6 +1950,7 @@ version = "0.0.0" dependencies = [ "anyhow", "assert2", + "blake3", "cargo-component", "cargo-component-core", "cargo_toml", @@ -1944,6 +1976,7 @@ dependencies = [ "serde 1.0.215", "serde_json", "serde_yaml", + "shlex", "syn 2.0.90", "tempfile", "test-r", @@ -2633,7 +2666,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ - "arrayvec", + "arrayvec 0.5.2", "bitflags 1.3.2", "cfg-if", "ryu", diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 0a732e28..bcc15249 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -66,6 +66,8 @@ walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" wit-encoder = "=0.221.2" wit-parser = "=0.221.2" +shlex = "1.3.0" +blake3 = "1.5.5" [dev-dependencies] diff --git a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json index a8186770..58f76945 100644 --- a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json +++ b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json @@ -228,6 +228,13 @@ "command": { "type": "string", "description": "External command to execute" + }, + "mkdirs": { + "type": "array", + "description": "List of directories that should be created before running the command", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -241,14 +248,21 @@ "type": "string", "description": "External command to execute" }, - "inputs": { + "mkdirs": { + "type": "array", + "description": "List of directories that should be created before running the command", + "items": { + "type": "string" + } + }, + "sources": { "type": "array", "description": "Inputs (paths and globs) for the external command", "items": { "type": "string" } }, - "outputs": { + "targets": { "type": "array", "description": "Output (paths and globs) for the external command", "items": { diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 68bc6afd..f0b7a965 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -10,7 +10,6 @@ use crate::model::app::{ ComponentPropertiesExtensions, ProfileName, DEFAULT_CONFIG_FILE_NAME, }; use crate::model::app_raw; -use crate::model::app_raw::ExternalCommand; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; use crate::wit_generate::{ @@ -24,6 +23,7 @@ use colored::Colorize; use glob::{glob_with, MatchOptions}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; +use serde::Serialize; use std::cell::OnceCell; use std::cmp::Ordering; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -336,10 +336,7 @@ async fn gen_rpc( let mut any_changed = false; for component_name in ctx.application.component_names() { let changed = create_generated_wit(ctx, component_name)?; - if changed { - // TODO: if this fails, it won't be retried, add done file for this - update_cargo_toml(ctx, component_name)?; - } + update_cargo_toml(ctx, changed, component_name)?; any_changed |= changed; } if any_changed { @@ -717,7 +714,7 @@ fn load_app_validated( app } -fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { +fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { log_action("Collecting", "sources"); let _indent = LogIndent::new(); @@ -732,12 +729,12 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult let includes = includes_from_yaml_file(source.as_path()); if includes.is_empty() { - ValidatedResult::Ok(vec![source]) + ValidatedResult::Ok(BTreeSet::from([source])) } else { ValidatedResult::from_result(compile_and_collect_globs(source_dir, &includes)) .map(|mut sources| { sources.insert(0, source); - sources + sources.into_iter().collect() }) } } @@ -758,7 +755,10 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult }) .collect(); - ValidatedResult::from_value_and_warns(sources.clone(), non_unique_source_warns) + ValidatedResult::from_value_and_warns( + sources.iter().cloned().collect(), + non_unique_source_warns, + ) } }; @@ -976,11 +976,17 @@ fn create_generated_base_wit( .application .component_source_wit(component_name, ctx.profile()); let component_generated_base_wit = ctx.application.component_generated_base_wit(component_name); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_base_wit); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "base_wit", + }, + )?; if is_up_to_date( ctx.config.skip_up_to_date_checks - || !gen_dir_done_marker.is_done() + || !task_result_marker.is_up_to_date() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_source_wit.clone()], || [component_generated_base_wit.clone()], @@ -1053,12 +1059,17 @@ fn create_generated_base_wit( ); let _indent = LogIndent::new(); - extract_main_interface_as_wit_dep(&component_generated_base_wit)?; + match extract_main_interface_as_wit_dep(&component_generated_base_wit) { + Ok(()) => { + task_result_marker.success()?; + Ok(true) + } + Err(err) => { + task_result_marker.failure()?; + Err(err) + } + } } - - gen_dir_done_marker.mark_as_done()?; - - Ok(true) } } @@ -1070,11 +1081,17 @@ fn create_generated_wit( let component_generated_wit = ctx .application .component_generated_wit(component_name, ctx.profile()); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_wit); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "wit", + }, + )?; if is_up_to_date( ctx.config.skip_up_to_date_checks - || !gen_dir_done_marker.is_done() + || !task_result_marker.is_up_to_date() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_generated_base_wit.clone()], || [component_generated_wit.clone()], @@ -1098,7 +1115,7 @@ fn create_generated_wit( copy_wit_sources(&component_generated_base_wit, &component_generated_wit)?; add_stub_deps(ctx, component_name)?; - gen_dir_done_marker.mark_as_done()?; + task_result_marker.success()?; Ok(true) } @@ -1106,6 +1123,7 @@ fn create_generated_wit( fn update_cargo_toml( ctx: &ApplicationContext, + mut skip_up_to_date_checks: bool, component_name: &ComponentName, ) -> anyhow::Result<()> { let component_source_wit = PathExtra::new( @@ -1120,16 +1138,43 @@ fn update_cargo_toml( })?; let cargo_toml = component_source_wit_parent.join("Cargo.toml"); - if cargo_toml.exists() { - regenerate_cargo_package_component( - &cargo_toml, - &ctx.application - .component_generated_wit(component_name, ctx.profile()), - None, - )? + if !cargo_toml.exists() { + return Ok(()); } - Ok(()) + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "Cargo.toml", + }, + )?; + + skip_up_to_date_checks |= skip_up_to_date_checks || ctx.config.skip_up_to_date_checks; + if !skip_up_to_date_checks && task_result_marker.is_up_to_date() { + log_skipping_up_to_date(format!( + "updating Cargo.toml for {}", + component_name.as_str().log_color_highlight() + )); + return Ok(()); + } + + let result = regenerate_cargo_package_component( + &cargo_toml, + &ctx.application + .component_generated_wit(component_name, ctx.profile()), + None, + ); + match result { + Ok(()) => { + task_result_marker.success()?; + Ok(()) + } + Err(err) => { + task_result_marker.failure()?; + Err(err) + } + } } async fn build_stub( @@ -1165,10 +1210,16 @@ async fn build_stub( let stub_wasm = ctx.application.stub_wasm(component_name); let stub_wit = ctx.application.stub_wit(component_name); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&stub_wit); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "stub", + }, + )?; if is_up_to_date( - ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done(), + ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date(), || stub_sources, || [stub_wit.clone(), stub_wasm.clone()], ) { @@ -1197,13 +1248,19 @@ async fn build_stub( ); fs::create_dir_all(&target_root)?; - commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await?; - - gen_dir_done_marker.mark_as_done()?; - - delete_path("stub temp build dir", &target_root)?; - - Ok(true) + let result = + commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await; + match result { + Ok(()) => { + task_result_marker.success()?; + delete_path("stub temp build dir", &target_root)?; + Ok(true) + } + Err(err) => { + task_result_marker.failure()?; + Err(err) + } + } } } @@ -1295,7 +1352,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { fn execute_external_command( ctx: &ApplicationContext, component_name: &ComponentName, - command: &ExternalCommand, + command: &app_raw::ExternalCommand, ) -> anyhow::Result<()> { let build_dir = command .dir @@ -1311,11 +1368,22 @@ fn execute_external_command( .to_path_buf() }); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ResolvedExternalCommandMarkerHash { + build_dir: &build_dir, + command, + }, + )?; + + let skip_up_to_date_checks = + ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date(); + if !command.sources.is_empty() && !command.targets.is_empty() { let sources = compile_and_collect_globs(&build_dir, &command.sources)?; let targets = compile_and_collect_globs(&build_dir, &command.targets)?; - if is_up_to_date(ctx.config.skip_up_to_date_checks, || sources, || targets) { + if is_up_to_date(skip_up_to_date_checks, || sources, || targets) { log_skipping_up_to_date(format!( "executing external command '{}' in directory {}", command.command.log_color_highlight(), @@ -1334,46 +1402,149 @@ fn execute_external_command( ), ); - let command_tokens = command.command.split(' ').collect::>(); + if !command.mkdirs.is_empty() { + let _ident = LogIndent::new(); + for dir in &command.mkdirs { + let dir = ctx + .application + .component_source_dir(component_name) + .join(dir); + if !std::fs::exists(&dir)? { + log_action( + "Creating", + format!("directory {}", dir.log_color_highlight()), + ); + std::fs::create_dir_all(dir)? + } + } + } + + let command_tokens = shlex::split(&command.command) + .ok_or_else(|| anyhow::anyhow!("Failed to parse external command: {}", command.command))?; if command_tokens.is_empty() { return Err(anyhow!("Empty command!")); } - let result = Command::new(command_tokens[0]) + let result = Command::new(command_tokens[0].clone()) .args(command_tokens.iter().skip(1)) .current_dir(build_dir) .status() .with_context(|| "Failed to execute command".to_string())?; - if !result.success() { - return Err(anyhow!(format!( + if result.success() { + task_result_marker.success()?; + Ok(()) + } else { + task_result_marker.failure()?; + Err(anyhow!(format!( "Command failed with exit code: {}", result .code() .map(|code| code.to_string().log_color_error_highlight().to_string()) .unwrap_or_else(|| "?".to_string()) - ))); + ))) } +} - Ok(()) +trait TaskResultMarkerHashInput { + fn task_kind() -> &'static str; + + fn hash_input(&self) -> anyhow::Result>; } -static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = ".done"; +#[derive(Serialize)] +struct ResolvedExternalCommandMarkerHash<'a> { + build_dir: &'a Path, + command: &'a app_raw::ExternalCommand, +} + +impl TaskResultMarkerHashInput for ResolvedExternalCommandMarkerHash<'_> { + fn task_kind() -> &'static str { + "ResolvedExternalCommandMarkerHash" + } + + fn hash_input(&self) -> anyhow::Result> { + Ok(serde_yaml::to_string(self)?.into_bytes()) + } +} -struct GeneratedDirDoneMarker<'a> { - dir: &'a Path, +struct ComponentGeneratorMarkerHash<'a> { + component_name: &'a ComponentName, + generator_kind: &'a str, } -impl<'a> GeneratedDirDoneMarker<'a> { - fn new(dir: &'a Path) -> Self { - Self { dir } +impl TaskResultMarkerHashInput for ComponentGeneratorMarkerHash<'_> { + fn task_kind() -> &'static str { + "ComponentGeneratorMarkerHash" + } + + fn hash_input(&self) -> anyhow::Result> { + Ok(format!("{}-{}", self.component_name, self.generator_kind).into_bytes()) + } +} + +struct TaskResultMarker { + success_marker_file_path: PathBuf, + failure_marker_file_path: PathBuf, + success_before: bool, + failure_before: bool, +} + +static TASK_RESULT_MARKER_SUCCESS_SUFFIX: &str = "-success"; +static TASK_RESULT_MARKER_FAILURE_SUFFIX: &str = "-failure"; + +impl TaskResultMarker { + fn new(dir: &Path, task: T) -> anyhow::Result { + let mut hasher = blake3::Hasher::new(); + hasher.update(T::task_kind().as_bytes()); + hasher.update(&task.hash_input()?); + let hex_hash = hasher.finalize().to_hex().to_string(); + + let success_marker_file_path = dir.join(format!( + "{}{}", + &hex_hash, TASK_RESULT_MARKER_SUCCESS_SUFFIX + )); + let failure_marker_file_path = dir.join(format!( + "{}{}", + &hex_hash, TASK_RESULT_MARKER_FAILURE_SUFFIX + )); + + let success_marker_exists = success_marker_file_path.exists(); + let failure_marker_exists = failure_marker_file_path.exists(); + + let (success_before, failure_before) = match (success_marker_exists, failure_marker_exists) + { + (true, false) => (true, false), + (false, false) => (false, false), + (_, true) => (false, true), + }; + + if failure_marker_exists || !success_marker_exists { + if success_marker_exists { + fs::remove(&success_marker_file_path)? + } + if failure_marker_exists { + fs::remove(&failure_marker_file_path)? + } + } + + Ok(Self { + success_marker_file_path, + failure_marker_file_path, + success_before, + failure_before, + }) + } + + fn is_up_to_date(&self) -> bool { + !self.failure_before && self.success_before } - fn is_done(&self) -> bool { - self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME).exists() + fn success(&self) -> anyhow::Result<()> { + fs::write_str(&self.success_marker_file_path, "") } - fn mark_as_done(&self) -> anyhow::Result<()> { - fs::write_str(self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME), "") + fn failure(&self) -> anyhow::Result<()> { + fs::write_str(&self.failure_marker_file_path, "") } } diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 047c7466..4d8ac7e7 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -826,6 +826,10 @@ impl Application { } } + pub fn task_result_marker_dir(&self) -> PathBuf { + self.temp_dir().join("task_results") + } + fn component(&self, component_name: &ComponentName) -> &Component { self.components .get(component_name) @@ -1035,12 +1039,17 @@ pub struct Component { impl Component { pub fn source_dir(&self) -> &Path { - self.source.parent().unwrap_or_else(|| { + let parent = self.source.parent().unwrap_or_else(|| { panic!( "Failed to get parent for component, source: {}", self.source.display() ) - }) + }); + if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + } } } diff --git a/wasm-rpc-stubgen/src/model/app_raw.rs b/wasm-rpc-stubgen/src/model/app_raw.rs index f2a740a0..8d1f47b2 100644 --- a/wasm-rpc-stubgen/src/model/app_raw.rs +++ b/wasm-rpc-stubgen/src/model/app_raw.rs @@ -142,6 +142,8 @@ pub struct ExternalCommand { #[serde(default, skip_serializing_if = "Option::is_none")] pub dir: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub mkdirs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub sources: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub targets: Vec, diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs index 3dc7741b..4841c967 100644 --- a/wasm-rpc-stubgen/src/model/template.rs +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -77,6 +77,7 @@ impl Template for app_raw::ExternalCommand { ) -> Result { Ok(app_raw::ExternalCommand { command: self.command.render(env, ctx)?, + mkdirs: self.mkdirs.render(env, ctx)?, dir: self.dir.render(env, ctx)?, sources: self.sources.render(env, ctx)?, targets: self.targets.render(env, ctx)?, diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 56a1fdfb..acf59877 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -186,6 +186,9 @@ impl ResolvedWitApplication { app: &Application, profile: Option<&ProfileName>, ) -> ValidatedResult { + // TODO: Can be removed once we fixed all docs and examples + std::env::set_var("WIT_REQUIRE_F32_F64", "0"); + log_action("Resolving", "application wit directories"); let _indent = LogIndent::new(); @@ -689,6 +692,9 @@ pub struct WitDepsResolver { impl WitDepsResolver { pub fn new(sources: Vec) -> anyhow::Result { + // TODO: Can be removed once we fixed all docs and examples + std::env::set_var("WIT_REQUIRE_F32_F64", "0"); + let mut packages = HashMap::>::new(); for source in &sources { From 4abca3f183dfe994cab8c245d74c9c1df39dd239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sat, 7 Dec 2024 16:32:35 +0100 Subject: [PATCH 2/6] fix extension overrides + error message fix --- wasm-rpc-stubgen/src/model/app.rs | 107 ++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 28 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 4d8ac7e7..71ad6a56 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -445,7 +445,7 @@ impl Application { "Profile {} cannot be used, as the component uses template {} with the following profiles: {}", profile_name.log_color_highlight(), template_name.as_str().log_color_highlight(), - component.profiles.keys().map(|s| s.log_color_highlight()).join(", ") + template.profiles.keys().map(|s| s.log_color_highlight()).join(", ") ) ); } @@ -466,15 +466,27 @@ impl Application { match rendered_template_properties { Ok(rendered_template_properties) => { - let (properties, any_template_overrides) = - rendered_template_properties.merge_with_overrides( - component.component_properties, - ); - Some(ResolvedComponentProperties::Properties { - template_name: Some(template_name), - any_template_overrides, - properties, - }) + match rendered_template_properties.merge_with_overrides( + component.component_properties, + ) { + Ok((properties, any_template_overrides)) => { + Some(ResolvedComponentProperties::Properties { + template_name: Some(template_name), + any_template_overrides, + properties, + }) + } + Err(err) => { + validation.add_error(format!( + "Failed to override template {}, error: {}", + template_name + .as_str() + .log_color_highlight(), + err.to_string().log_color_error_highlight() + )); + None + } + } } Err(err) => { validation.add_error(format!( @@ -490,7 +502,7 @@ impl Application { HashMap::::new(); let mut profiles = HashMap::>::new(); - let mut any_template_render_error = false; + let mut any_template_error = false; for (profile_name, template_component_properties) in &template.profiles @@ -503,7 +515,7 @@ impl Application { ); match rendered_template_properties { Ok(rendered_template_properties) => { - let (properties, any_overrides) = { + let properties_with_overrides = { if let Some(component_properties) = component.profiles.remove(profile_name) { @@ -512,18 +524,32 @@ impl Application { component_properties, ) } else { - (rendered_template_properties, false) + Ok((rendered_template_properties, false)) } }; - any_template_overrides.insert( - profile_name.clone().into(), - any_overrides, - ); - profiles.insert( - profile_name.clone().into(), - properties, - ); + match properties_with_overrides { + Ok((properties, any_overrides)) => { + any_template_overrides.insert( + profile_name.clone().into(), + any_overrides, + ); + profiles.insert( + profile_name.clone().into(), + properties, + ); + } + Err(err) => { + validation.add_error(format!( + "Failed to override template {}, error: {}", + template_name + .as_str() + .log_color_highlight(), + err.to_string().log_color_error_highlight() + )); + any_template_error = true; + } + } } Err(err) => { validation.add_error(format!( @@ -531,12 +557,12 @@ impl Application { template_name.as_str().log_color_highlight(), err.to_string().log_color_error_highlight() )); - any_template_render_error = true + any_template_error = true } } } - (!any_template_render_error).then(|| { + (!any_template_error).then(|| { ResolvedComponentProperties::Profiles { template_name: Some(template_name), any_template_overrides, @@ -1093,7 +1119,10 @@ impl ComponentProperties { ComponentProperties::from_raw(template_properties.render(env, ctx)?) } - fn merge_with_overrides(mut self, overrides: app_raw::ComponentProperties) -> (Self, bool) { + fn merge_with_overrides( + mut self, + overrides: app_raw::ComponentProperties, + ) -> anyhow::Result<(Self, bool)> { let mut any_overrides = false; if let Some(source_wit) = overrides.source_wit { @@ -1129,11 +1158,23 @@ impl ComponentProperties { .insert(custom_command_name, custom_command); } - (self, any_overrides) + if !overrides.extensions.is_empty() { + let extensions_override = + CPE::raw_from_serde_json(serde_json::Value::Object(overrides.extensions))?; + let (extensions, any_extension_overrides) = self + .extensions + .take() + .unwrap_or_default() + .merge_wit_overrides(extensions_override); + self.extensions = Some(extensions); + any_overrides |= any_overrides || any_extension_overrides; + } + + Ok((self, any_overrides)) } } -pub trait ComponentPropertiesExtensions: Sized + Debug + Clone { +pub trait ComponentPropertiesExtensions: Sized + Debug + Clone + Default { type Raw: Debug + Clone; fn raw_from_serde_json(extensions: serde_json::Value) -> serde_json::Result; @@ -1143,9 +1184,11 @@ pub trait ComponentPropertiesExtensions: Sized + Debug + Clone { validation: &mut ValidationBuilder, raw: Self::Raw, ) -> Option; + + fn merge_wit_overrides(self, overrides: Self::Raw) -> (Self, bool); } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ComponentPropertiesExtensionsNone {} @@ -1166,9 +1209,13 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { ) -> Option { Some(raw) } + + fn merge_wit_overrides(self, _overrides: Self::Raw) -> (Self, bool) { + (self, true) + } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ComponentPropertiesExtensionsAny; impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { @@ -1188,4 +1235,8 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { ) -> Option { Some(raw) } + + fn merge_wit_overrides(self, _overrides: Self::Raw) -> (Self, bool) { + (self, true) + } } From 7d07ade2d44ee86d195b9537d8a686029b8043e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 8 Dec 2024 17:59:06 +0100 Subject: [PATCH 3/6] fix mkdirs --- wasm-rpc-stubgen/src/commands/app.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index f0b7a965..4f75403a 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -1405,10 +1405,7 @@ fn execute_external_command( if !command.mkdirs.is_empty() { let _ident = LogIndent::new(); for dir in &command.mkdirs { - let dir = ctx - .application - .component_source_dir(component_name) - .join(dir); + let dir = build_dir.join(dir); if !std::fs::exists(&dir)? { log_action( "Creating", From b571b2421cefee137952903e8968043845509953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 8 Dec 2024 17:59:56 +0100 Subject: [PATCH 4/6] more merge fixes --- wasm-rpc-stubgen/src/model/app.rs | 90 +++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 71ad6a56..d85cf52e 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -467,15 +467,18 @@ impl Application { match rendered_template_properties { Ok(rendered_template_properties) => { match rendered_template_properties.merge_with_overrides( + &source, + &mut validation, component.component_properties, ) { - Ok((properties, any_template_overrides)) => { + Ok((Some(properties), any_template_overrides)) => { Some(ResolvedComponentProperties::Properties { template_name: Some(template_name), any_template_overrides, properties, }) } + Ok((None, _any_template_overrides)) => None, Err(err) => { validation.add_error(format!( "Failed to override template {}, error: {}", @@ -521,15 +524,20 @@ impl Application { { rendered_template_properties .merge_with_overrides( + &source, + &mut validation, component_properties, ) } else { - Ok((rendered_template_properties, false)) + Ok(( + Some(rendered_template_properties), + false, + )) } }; match properties_with_overrides { - Ok((properties, any_overrides)) => { + Ok((Some(properties), any_overrides)) => { any_template_overrides.insert( profile_name.clone().into(), any_overrides, @@ -539,6 +547,9 @@ impl Application { properties, ); } + Ok((None, _any_template_overrides)) => { + any_template_error = true; + } Err(err) => { validation.add_error(format!( "Failed to override template {}, error: {}", @@ -673,8 +684,8 @@ impl Application { for custom_command in properties.custom_commands.keys() { if reserved_commands.contains(custom_command.as_str()) { validation.add_error(format!("Cannot use {} as custom command name, reserved command names: {}", - custom_command.log_color_error_highlight(), - reserved_commands.iter().map(|s| s.log_color_highlight()).join(", ") + custom_command.log_color_error_highlight(), + reserved_commands.iter().map(|s| s.log_color_highlight()).join(", ") )); } } @@ -1121,8 +1132,10 @@ impl ComponentProperties { fn merge_with_overrides( mut self, + source: &Path, + validation: &mut ValidationBuilder, overrides: app_raw::ComponentProperties, - ) -> anyhow::Result<(Self, bool)> { + ) -> anyhow::Result<(Option, bool)> { let mut any_overrides = false; if let Some(source_wit) = overrides.source_wit { @@ -1158,19 +1171,35 @@ impl ComponentProperties { .insert(custom_command_name, custom_command); } - if !overrides.extensions.is_empty() { - let extensions_override = - CPE::raw_from_serde_json(serde_json::Value::Object(overrides.extensions))?; - let (extensions, any_extension_overrides) = self - .extensions - .take() - .unwrap_or_default() - .merge_wit_overrides(extensions_override); - self.extensions = Some(extensions); - any_overrides |= any_overrides || any_extension_overrides; - } + let any_extension_error = { + if !overrides.extensions.is_empty() { + let extensions_override = + CPE::raw_from_serde_json(serde_json::Value::Object(overrides.extensions))?; + let (extensions, any_extension_overrides) = self + .extensions + .take() + .unwrap_or_default() + .merge_wit_overrides(source, validation, extensions_override)?; + + any_overrides |= any_overrides || any_extension_overrides; + + match extensions { + Some(extensions) => { + self.extensions = Some(extensions); + false + } + None => true, + } + } else { + false + } + }; - Ok((self, any_overrides)) + if any_extension_error { + Ok((None, false)) + } else { + Ok((Some(self), any_overrides)) + } } } @@ -1185,7 +1214,12 @@ pub trait ComponentPropertiesExtensions: Sized + Debug + Clone + Default { raw: Self::Raw, ) -> Option; - fn merge_wit_overrides(self, overrides: Self::Raw) -> (Self, bool); + fn merge_wit_overrides( + self, + source: &Path, + validation: &mut ValidationBuilder, + overrides: Self::Raw, + ) -> serde_json::Result<(Option, bool)>; } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -1210,8 +1244,13 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { Some(raw) } - fn merge_wit_overrides(self, _overrides: Self::Raw) -> (Self, bool) { - (self, true) + fn merge_wit_overrides( + self, + _source: &Path, + _validation: &mut ValidationBuilder, + _overrides: Self::Raw, + ) -> serde_json::Result<(Option, bool)> { + Ok((Some(self), false)) } } @@ -1236,7 +1275,12 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { Some(raw) } - fn merge_wit_overrides(self, _overrides: Self::Raw) -> (Self, bool) { - (self, true) + fn merge_wit_overrides( + self, + _source: &Path, + _validation: &mut ValidationBuilder, + _overrides: Self::Raw, + ) -> serde_json::Result<(Option, bool)> { + Ok((Some(self), false)) } } From c9194bad655c87e2b8bae933c2c8a5e421739700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 8 Dec 2024 20:27:40 +0100 Subject: [PATCH 5/6] fix stub interface name usage --- wasm-rpc-stubgen/src/rust.rs | 29 ++++++----------------------- wasm-rpc-stubgen/src/stub.rs | 31 +++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 38885fc7..39dafb12 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -235,10 +235,9 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { }); let remote_function_name = get_remote_function_name( - def, - "drop", interface.interface_name(), interface.resource_name(), + "drop", ); if interface.is_resource() { interface_impls.push(quote! { @@ -329,10 +328,9 @@ fn generate_result_wrapper_get_source( let output_values = get_output_values_source(def, function, mode)?; let remote_function_name = get_remote_function_name( - def, - &function.name, interface.interface_name(), interface.resource_name(), + &function.name, ); Ok(quote! { @@ -390,10 +388,9 @@ fn generate_function_stub_source( let output_values = get_output_values_source(def, function, mode)?; let remote_function_name = get_remote_function_name( - def, - &function.name, owner.interface_name(), owner.resource_name(), + &function.name, ); let rpc = match mode { @@ -591,30 +588,16 @@ fn get_result_type_source( } fn get_remote_function_name( - def: &StubDefinition, - function_name: &str, interface_name: Option<&str>, resource_name: Option<&str>, + function_name: &str, ) -> String { match (interface_name, resource_name) { (Some(remote_interface), None) => { - format!( - "{}:{}/{}.{{{}}}", - def.source_package_name.namespace, - def.source_package_name.name, - remote_interface, - function_name - ) + format!("{}.{{{}}}", remote_interface, function_name) } (Some(remote_interface), Some(resource)) => { - format!( - "{}:{}/{}.{{{}.{}}}", - def.source_package_name.namespace, - def.source_package_name.name, - remote_interface, - resource, - function_name - ) + format!("{}.{{{}.{}}}", remote_interface, resource, function_name) } (None, Some(resource)) => { format!("{}.{}", resource, function_name) diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 87880073..1c3e8834 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -255,9 +255,9 @@ impl StubDefinition { let package = interface.package.map(|package_id| { self.resolve.packages.get(package_id).unwrap_or_else(|| { panic!( - "Missing package for interface, package id: {:?}, interface id: {:?}", - package_id, interface_id, - ) + "Missing package for interface, package id: {:?}, interface id: {:?}", + package_id, interface_id, + ) }) }); @@ -440,6 +440,14 @@ impl StubDefinition { name: String, interface: &Interface, ) -> Vec { + let package = interface + .package + .map(|package_id| self.resolve.packages.get(package_id).unwrap()); + let owner_interface = interface.name.as_ref().map(|name| match package { + Some(package) => package.name.interface_id(name), + None => name.to_string(), + }); + let functions = Self::functions_to_stubs(interface.functions.values()); let (used_types, resource_interfaces) = self.extract_interface_stubs_from_types( @@ -458,7 +466,7 @@ impl StubDefinition { global: false, constructor_params: None, static_functions: vec![], - owner_interface: None, + owner_interface, }); interface_stubs.extend(resource_interfaces); @@ -507,6 +515,14 @@ impl StubDefinition { type_name: String, type_id: TypeId, ) -> InterfaceStub { + let package = owner_interface + .package + .map(|package_id| self.resolve.packages.get(package_id).unwrap()); + let owner_interface_name = owner_interface.name.as_ref().map(|name| match package { + Some(package) => package.name.interface_id(name), + None => name.to_string(), + }); + let functions_by_kind = |kind: FunctionKind| { owner_interface .functions @@ -543,12 +559,7 @@ impl StubDefinition { global: false, constructor_params, static_functions: function_stubs_by_kind(FunctionKind::Static(type_id)), - owner_interface: Some(owner_interface.name.clone().unwrap_or_else(|| { - panic!( - "failed to get interface name for interface: {:?}", - owner_interface - ) - })), + owner_interface: owner_interface_name, } } From 8509022f6a5941e8b9ab8e202f6f99501f19641f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 8 Dec 2024 20:28:31 +0100 Subject: [PATCH 6/6] naming + flag skipping stub dir deletion --- wasm-rpc-stubgen/src/commands/app.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 4f75403a..4c95d921 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -409,7 +409,7 @@ async fn link_rpc( || [linked_wasm.clone()], ) { log_skipping_up_to_date(format!( - "composing wasm rpc dependencies ({}) into {}", + "linking wasm rpc dependencies ({}) into {}", dependencies .iter() .map(|s| s.as_str().log_color_highlight()) @@ -423,7 +423,7 @@ async fn link_rpc( log_action( "Copying", format!( - "(without composing) {} to {}, no wasm rpc dependencies defined", + "(without linking) {} to {}, no wasm rpc dependencies defined", component_wasm.log_color_highlight(), linked_wasm.log_color_highlight(), ), @@ -431,9 +431,9 @@ async fn link_rpc( fs::copy(&component_wasm, &linked_wasm)?; } else { log_action( - "Composing", + "Linking", format!( - "wasm rpc dependencies ({}) into {}", + "WASM RPC dependencies ({}) into {}", dependencies .iter() .map(|s| s.as_str().log_color_highlight()) @@ -1253,7 +1253,18 @@ async fn build_stub( match result { Ok(()) => { task_result_marker.success()?; - delete_path("stub temp build dir", &target_root)?; + + let skip_delete = std::env::var("WASM_RPC_KEEP_STUB_DIR") + .ok() + .map(|flag| { + let flag = flag.to_lowercase(); + flag.starts_with("t") || flag == "1" + }) + .unwrap_or_default(); + + if !skip_delete { + delete_path("stub temp build dir", &target_root)?; + } Ok(true) } Err(err) => {