diff --git a/Cargo.lock b/Cargo.lock index 7cb79f66..7ded79c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,12 +176,24 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "assert2" version = "0.3.15" @@ -511,6 +523,19 @@ dependencies = [ "typenum", ] +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec 0.7.6", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -922,6 +947,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "cookie" version = "0.18.1" @@ -1919,6 +1950,7 @@ version = "0.0.0" dependencies = [ "anyhow", "assert2", + "blake3", "cargo-component", "cargo-component-core", "cargo_toml", @@ -1944,6 +1976,7 @@ dependencies = [ "serde 1.0.215", "serde_json", "serde_yaml", + "shlex", "syn 2.0.90", "tempfile", "test-r", @@ -2633,7 +2666,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ - "arrayvec", + "arrayvec 0.5.2", "bitflags 1.3.2", "cfg-if", "ryu", diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 0a732e28..bcc15249 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -66,6 +66,8 @@ walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" wit-encoder = "=0.221.2" wit-parser = "=0.221.2" +shlex = "1.3.0" +blake3 = "1.5.5" [dev-dependencies] diff --git a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json index a8186770..58f76945 100644 --- a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json +++ b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json @@ -228,6 +228,13 @@ "command": { "type": "string", "description": "External command to execute" + }, + "mkdirs": { + "type": "array", + "description": "List of directories that should be created before running the command", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -241,14 +248,21 @@ "type": "string", "description": "External command to execute" }, - "inputs": { + "mkdirs": { + "type": "array", + "description": "List of directories that should be created before running the command", + "items": { + "type": "string" + } + }, + "sources": { "type": "array", "description": "Inputs (paths and globs) for the external command", "items": { "type": "string" } }, - "outputs": { + "targets": { "type": "array", "description": "Output (paths and globs) for the external command", "items": { diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 68bc6afd..f0b7a965 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -10,7 +10,6 @@ use crate::model::app::{ ComponentPropertiesExtensions, ProfileName, DEFAULT_CONFIG_FILE_NAME, }; use crate::model::app_raw; -use crate::model::app_raw::ExternalCommand; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; use crate::wit_generate::{ @@ -24,6 +23,7 @@ use colored::Colorize; use glob::{glob_with, MatchOptions}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; +use serde::Serialize; use std::cell::OnceCell; use std::cmp::Ordering; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -336,10 +336,7 @@ async fn gen_rpc( let mut any_changed = false; for component_name in ctx.application.component_names() { let changed = create_generated_wit(ctx, component_name)?; - if changed { - // TODO: if this fails, it won't be retried, add done file for this - update_cargo_toml(ctx, component_name)?; - } + update_cargo_toml(ctx, changed, component_name)?; any_changed |= changed; } if any_changed { @@ -717,7 +714,7 @@ fn load_app_validated( app } -fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { +fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { log_action("Collecting", "sources"); let _indent = LogIndent::new(); @@ -732,12 +729,12 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult let includes = includes_from_yaml_file(source.as_path()); if includes.is_empty() { - ValidatedResult::Ok(vec![source]) + ValidatedResult::Ok(BTreeSet::from([source])) } else { ValidatedResult::from_result(compile_and_collect_globs(source_dir, &includes)) .map(|mut sources| { sources.insert(0, source); - sources + sources.into_iter().collect() }) } } @@ -758,7 +755,10 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult }) .collect(); - ValidatedResult::from_value_and_warns(sources.clone(), non_unique_source_warns) + ValidatedResult::from_value_and_warns( + sources.iter().cloned().collect(), + non_unique_source_warns, + ) } }; @@ -976,11 +976,17 @@ fn create_generated_base_wit( .application .component_source_wit(component_name, ctx.profile()); let component_generated_base_wit = ctx.application.component_generated_base_wit(component_name); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_base_wit); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "base_wit", + }, + )?; if is_up_to_date( ctx.config.skip_up_to_date_checks - || !gen_dir_done_marker.is_done() + || !task_result_marker.is_up_to_date() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_source_wit.clone()], || [component_generated_base_wit.clone()], @@ -1053,12 +1059,17 @@ fn create_generated_base_wit( ); let _indent = LogIndent::new(); - extract_main_interface_as_wit_dep(&component_generated_base_wit)?; + match extract_main_interface_as_wit_dep(&component_generated_base_wit) { + Ok(()) => { + task_result_marker.success()?; + Ok(true) + } + Err(err) => { + task_result_marker.failure()?; + Err(err) + } + } } - - gen_dir_done_marker.mark_as_done()?; - - Ok(true) } } @@ -1070,11 +1081,17 @@ fn create_generated_wit( let component_generated_wit = ctx .application .component_generated_wit(component_name, ctx.profile()); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_wit); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "wit", + }, + )?; if is_up_to_date( ctx.config.skip_up_to_date_checks - || !gen_dir_done_marker.is_done() + || !task_result_marker.is_up_to_date() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_generated_base_wit.clone()], || [component_generated_wit.clone()], @@ -1098,7 +1115,7 @@ fn create_generated_wit( copy_wit_sources(&component_generated_base_wit, &component_generated_wit)?; add_stub_deps(ctx, component_name)?; - gen_dir_done_marker.mark_as_done()?; + task_result_marker.success()?; Ok(true) } @@ -1106,6 +1123,7 @@ fn create_generated_wit( fn update_cargo_toml( ctx: &ApplicationContext, + mut skip_up_to_date_checks: bool, component_name: &ComponentName, ) -> anyhow::Result<()> { let component_source_wit = PathExtra::new( @@ -1120,16 +1138,43 @@ fn update_cargo_toml( })?; let cargo_toml = component_source_wit_parent.join("Cargo.toml"); - if cargo_toml.exists() { - regenerate_cargo_package_component( - &cargo_toml, - &ctx.application - .component_generated_wit(component_name, ctx.profile()), - None, - )? + if !cargo_toml.exists() { + return Ok(()); } - Ok(()) + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "Cargo.toml", + }, + )?; + + skip_up_to_date_checks |= skip_up_to_date_checks || ctx.config.skip_up_to_date_checks; + if !skip_up_to_date_checks && task_result_marker.is_up_to_date() { + log_skipping_up_to_date(format!( + "updating Cargo.toml for {}", + component_name.as_str().log_color_highlight() + )); + return Ok(()); + } + + let result = regenerate_cargo_package_component( + &cargo_toml, + &ctx.application + .component_generated_wit(component_name, ctx.profile()), + None, + ); + match result { + Ok(()) => { + task_result_marker.success()?; + Ok(()) + } + Err(err) => { + task_result_marker.failure()?; + Err(err) + } + } } async fn build_stub( @@ -1165,10 +1210,16 @@ async fn build_stub( let stub_wasm = ctx.application.stub_wasm(component_name); let stub_wit = ctx.application.stub_wit(component_name); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&stub_wit); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ComponentGeneratorMarkerHash { + component_name, + generator_kind: "stub", + }, + )?; if is_up_to_date( - ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done(), + ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date(), || stub_sources, || [stub_wit.clone(), stub_wasm.clone()], ) { @@ -1197,13 +1248,19 @@ async fn build_stub( ); fs::create_dir_all(&target_root)?; - commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await?; - - gen_dir_done_marker.mark_as_done()?; - - delete_path("stub temp build dir", &target_root)?; - - Ok(true) + let result = + commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await; + match result { + Ok(()) => { + task_result_marker.success()?; + delete_path("stub temp build dir", &target_root)?; + Ok(true) + } + Err(err) => { + task_result_marker.failure()?; + Err(err) + } + } } } @@ -1295,7 +1352,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { fn execute_external_command( ctx: &ApplicationContext, component_name: &ComponentName, - command: &ExternalCommand, + command: &app_raw::ExternalCommand, ) -> anyhow::Result<()> { let build_dir = command .dir @@ -1311,11 +1368,22 @@ fn execute_external_command( .to_path_buf() }); + let task_result_marker = TaskResultMarker::new( + &ctx.application.task_result_marker_dir(), + ResolvedExternalCommandMarkerHash { + build_dir: &build_dir, + command, + }, + )?; + + let skip_up_to_date_checks = + ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date(); + if !command.sources.is_empty() && !command.targets.is_empty() { let sources = compile_and_collect_globs(&build_dir, &command.sources)?; let targets = compile_and_collect_globs(&build_dir, &command.targets)?; - if is_up_to_date(ctx.config.skip_up_to_date_checks, || sources, || targets) { + if is_up_to_date(skip_up_to_date_checks, || sources, || targets) { log_skipping_up_to_date(format!( "executing external command '{}' in directory {}", command.command.log_color_highlight(), @@ -1334,46 +1402,149 @@ fn execute_external_command( ), ); - let command_tokens = command.command.split(' ').collect::>(); + if !command.mkdirs.is_empty() { + let _ident = LogIndent::new(); + for dir in &command.mkdirs { + let dir = ctx + .application + .component_source_dir(component_name) + .join(dir); + if !std::fs::exists(&dir)? { + log_action( + "Creating", + format!("directory {}", dir.log_color_highlight()), + ); + std::fs::create_dir_all(dir)? + } + } + } + + let command_tokens = shlex::split(&command.command) + .ok_or_else(|| anyhow::anyhow!("Failed to parse external command: {}", command.command))?; if command_tokens.is_empty() { return Err(anyhow!("Empty command!")); } - let result = Command::new(command_tokens[0]) + let result = Command::new(command_tokens[0].clone()) .args(command_tokens.iter().skip(1)) .current_dir(build_dir) .status() .with_context(|| "Failed to execute command".to_string())?; - if !result.success() { - return Err(anyhow!(format!( + if result.success() { + task_result_marker.success()?; + Ok(()) + } else { + task_result_marker.failure()?; + Err(anyhow!(format!( "Command failed with exit code: {}", result .code() .map(|code| code.to_string().log_color_error_highlight().to_string()) .unwrap_or_else(|| "?".to_string()) - ))); + ))) } +} - Ok(()) +trait TaskResultMarkerHashInput { + fn task_kind() -> &'static str; + + fn hash_input(&self) -> anyhow::Result>; } -static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = ".done"; +#[derive(Serialize)] +struct ResolvedExternalCommandMarkerHash<'a> { + build_dir: &'a Path, + command: &'a app_raw::ExternalCommand, +} + +impl TaskResultMarkerHashInput for ResolvedExternalCommandMarkerHash<'_> { + fn task_kind() -> &'static str { + "ResolvedExternalCommandMarkerHash" + } + + fn hash_input(&self) -> anyhow::Result> { + Ok(serde_yaml::to_string(self)?.into_bytes()) + } +} -struct GeneratedDirDoneMarker<'a> { - dir: &'a Path, +struct ComponentGeneratorMarkerHash<'a> { + component_name: &'a ComponentName, + generator_kind: &'a str, } -impl<'a> GeneratedDirDoneMarker<'a> { - fn new(dir: &'a Path) -> Self { - Self { dir } +impl TaskResultMarkerHashInput for ComponentGeneratorMarkerHash<'_> { + fn task_kind() -> &'static str { + "ComponentGeneratorMarkerHash" + } + + fn hash_input(&self) -> anyhow::Result> { + Ok(format!("{}-{}", self.component_name, self.generator_kind).into_bytes()) + } +} + +struct TaskResultMarker { + success_marker_file_path: PathBuf, + failure_marker_file_path: PathBuf, + success_before: bool, + failure_before: bool, +} + +static TASK_RESULT_MARKER_SUCCESS_SUFFIX: &str = "-success"; +static TASK_RESULT_MARKER_FAILURE_SUFFIX: &str = "-failure"; + +impl TaskResultMarker { + fn new(dir: &Path, task: T) -> anyhow::Result { + let mut hasher = blake3::Hasher::new(); + hasher.update(T::task_kind().as_bytes()); + hasher.update(&task.hash_input()?); + let hex_hash = hasher.finalize().to_hex().to_string(); + + let success_marker_file_path = dir.join(format!( + "{}{}", + &hex_hash, TASK_RESULT_MARKER_SUCCESS_SUFFIX + )); + let failure_marker_file_path = dir.join(format!( + "{}{}", + &hex_hash, TASK_RESULT_MARKER_FAILURE_SUFFIX + )); + + let success_marker_exists = success_marker_file_path.exists(); + let failure_marker_exists = failure_marker_file_path.exists(); + + let (success_before, failure_before) = match (success_marker_exists, failure_marker_exists) + { + (true, false) => (true, false), + (false, false) => (false, false), + (_, true) => (false, true), + }; + + if failure_marker_exists || !success_marker_exists { + if success_marker_exists { + fs::remove(&success_marker_file_path)? + } + if failure_marker_exists { + fs::remove(&failure_marker_file_path)? + } + } + + Ok(Self { + success_marker_file_path, + failure_marker_file_path, + success_before, + failure_before, + }) + } + + fn is_up_to_date(&self) -> bool { + !self.failure_before && self.success_before } - fn is_done(&self) -> bool { - self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME).exists() + fn success(&self) -> anyhow::Result<()> { + fs::write_str(&self.success_marker_file_path, "") } - fn mark_as_done(&self) -> anyhow::Result<()> { - fs::write_str(self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME), "") + fn failure(&self) -> anyhow::Result<()> { + fs::write_str(&self.failure_marker_file_path, "") } } diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 047c7466..4d8ac7e7 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -826,6 +826,10 @@ impl Application { } } + pub fn task_result_marker_dir(&self) -> PathBuf { + self.temp_dir().join("task_results") + } + fn component(&self, component_name: &ComponentName) -> &Component { self.components .get(component_name) @@ -1035,12 +1039,17 @@ pub struct Component { impl Component { pub fn source_dir(&self) -> &Path { - self.source.parent().unwrap_or_else(|| { + let parent = self.source.parent().unwrap_or_else(|| { panic!( "Failed to get parent for component, source: {}", self.source.display() ) - }) + }); + if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + } } } diff --git a/wasm-rpc-stubgen/src/model/app_raw.rs b/wasm-rpc-stubgen/src/model/app_raw.rs index f2a740a0..8d1f47b2 100644 --- a/wasm-rpc-stubgen/src/model/app_raw.rs +++ b/wasm-rpc-stubgen/src/model/app_raw.rs @@ -142,6 +142,8 @@ pub struct ExternalCommand { #[serde(default, skip_serializing_if = "Option::is_none")] pub dir: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub mkdirs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub sources: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub targets: Vec, diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs index 3dc7741b..4841c967 100644 --- a/wasm-rpc-stubgen/src/model/template.rs +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -77,6 +77,7 @@ impl Template for app_raw::ExternalCommand { ) -> Result { Ok(app_raw::ExternalCommand { command: self.command.render(env, ctx)?, + mkdirs: self.mkdirs.render(env, ctx)?, dir: self.dir.render(env, ctx)?, sources: self.sources.render(env, ctx)?, targets: self.targets.render(env, ctx)?, diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 56a1fdfb..acf59877 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -186,6 +186,9 @@ impl ResolvedWitApplication { app: &Application, profile: Option<&ProfileName>, ) -> ValidatedResult { + // TODO: Can be removed once we fixed all docs and examples + std::env::set_var("WIT_REQUIRE_F32_F64", "0"); + log_action("Resolving", "application wit directories"); let _indent = LogIndent::new(); @@ -689,6 +692,9 @@ pub struct WitDepsResolver { impl WitDepsResolver { pub fn new(sources: Vec) -> anyhow::Result { + // TODO: Can be removed once we fixed all docs and examples + std::env::set_var("WIT_REQUIRE_F32_F64", "0"); + let mut packages = HashMap::>::new(); for source in &sources {